runnable 0.50.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. extensions/README.md +0 -0
  2. extensions/__init__.py +0 -0
  3. extensions/catalog/README.md +0 -0
  4. extensions/catalog/any_path.py +214 -0
  5. extensions/catalog/file_system.py +52 -0
  6. extensions/catalog/minio.py +72 -0
  7. extensions/catalog/pyproject.toml +14 -0
  8. extensions/catalog/s3.py +11 -0
  9. extensions/job_executor/README.md +0 -0
  10. extensions/job_executor/__init__.py +236 -0
  11. extensions/job_executor/emulate.py +70 -0
  12. extensions/job_executor/k8s.py +553 -0
  13. extensions/job_executor/k8s_job_spec.yaml +37 -0
  14. extensions/job_executor/local.py +35 -0
  15. extensions/job_executor/local_container.py +161 -0
  16. extensions/job_executor/pyproject.toml +16 -0
  17. extensions/nodes/README.md +0 -0
  18. extensions/nodes/__init__.py +0 -0
  19. extensions/nodes/conditional.py +301 -0
  20. extensions/nodes/fail.py +78 -0
  21. extensions/nodes/loop.py +394 -0
  22. extensions/nodes/map.py +477 -0
  23. extensions/nodes/parallel.py +281 -0
  24. extensions/nodes/pyproject.toml +15 -0
  25. extensions/nodes/stub.py +93 -0
  26. extensions/nodes/success.py +78 -0
  27. extensions/nodes/task.py +156 -0
  28. extensions/pipeline_executor/README.md +0 -0
  29. extensions/pipeline_executor/__init__.py +871 -0
  30. extensions/pipeline_executor/argo.py +1266 -0
  31. extensions/pipeline_executor/emulate.py +119 -0
  32. extensions/pipeline_executor/local.py +226 -0
  33. extensions/pipeline_executor/local_container.py +369 -0
  34. extensions/pipeline_executor/mocked.py +159 -0
  35. extensions/pipeline_executor/pyproject.toml +16 -0
  36. extensions/run_log_store/README.md +0 -0
  37. extensions/run_log_store/__init__.py +0 -0
  38. extensions/run_log_store/any_path.py +100 -0
  39. extensions/run_log_store/chunked_fs.py +122 -0
  40. extensions/run_log_store/chunked_minio.py +141 -0
  41. extensions/run_log_store/file_system.py +91 -0
  42. extensions/run_log_store/generic_chunked.py +549 -0
  43. extensions/run_log_store/minio.py +114 -0
  44. extensions/run_log_store/pyproject.toml +15 -0
  45. extensions/secrets/README.md +0 -0
  46. extensions/secrets/dotenv.py +62 -0
  47. extensions/secrets/pyproject.toml +15 -0
  48. runnable/__init__.py +108 -0
  49. runnable/catalog.py +141 -0
  50. runnable/cli.py +484 -0
  51. runnable/context.py +730 -0
  52. runnable/datastore.py +1058 -0
  53. runnable/defaults.py +159 -0
  54. runnable/entrypoints.py +390 -0
  55. runnable/exceptions.py +137 -0
  56. runnable/executor.py +561 -0
  57. runnable/gantt.py +1646 -0
  58. runnable/graph.py +501 -0
  59. runnable/names.py +546 -0
  60. runnable/nodes.py +593 -0
  61. runnable/parameters.py +217 -0
  62. runnable/pickler.py +96 -0
  63. runnable/sdk.py +1277 -0
  64. runnable/secrets.py +92 -0
  65. runnable/tasks.py +1268 -0
  66. runnable/telemetry.py +142 -0
  67. runnable/utils.py +423 -0
  68. runnable-0.50.0.dist-info/METADATA +189 -0
  69. runnable-0.50.0.dist-info/RECORD +72 -0
  70. runnable-0.50.0.dist-info/WHEEL +4 -0
  71. runnable-0.50.0.dist-info/entry_points.txt +53 -0
  72. runnable-0.50.0.dist-info/licenses/LICENSE +201 -0
runnable/gantt.py ADDED
@@ -0,0 +1,1646 @@
1
+ """
2
+ Simplified visualization for Runnable pipeline execution.
3
+
4
+ This module provides lightweight, reusable components that understand
5
+ the composite pipeline structure documented in the run logs.
6
+ """
7
+
8
+ import json
9
+ from dataclasses import dataclass
10
+ from datetime import datetime
11
+ from pathlib import Path
12
+ from typing import Any, Dict, List, Optional, Tuple, Union
13
+
14
+
15
+ @dataclass
16
+ class ParameterInfo:
17
+ """Enhanced parameter representation with categorization."""
18
+
19
+ key_inputs: List[str] # Most important inputs for this step
20
+ step_outputs: List[str] # Parameters generated by this step
21
+ context_params: List[str] # Pipeline/inherited parameters
22
+ iteration_vars: List[str] # Map/loop variables (chunk, etc.)
23
+ all_inputs: List[str] # Complete input list
24
+ all_outputs: List[str] # Complete output list
25
+
26
+
27
+ @dataclass
28
+ class StepInfo:
29
+ """Clean representation of a pipeline step."""
30
+
31
+ name: str
32
+ internal_name: str
33
+ status: str
34
+ step_type: str
35
+ start_time: Optional[datetime]
36
+ end_time: Optional[datetime]
37
+ duration_ms: float
38
+ level: int # 0=top-level, 1=branch, 2=nested
39
+ parent: Optional[str]
40
+ branch: Optional[str]
41
+ command: str
42
+ command_type: str
43
+ input_params: List[str]
44
+ output_params: List[str]
45
+ catalog_ops: Dict[str, List[Dict[str, str]]]
46
+ # Enhanced parameter information
47
+ parameters: Optional["ParameterInfo"] = None
48
+ # Error and attempt information
49
+ error_message: Optional[str] = None
50
+ attempt_count: int = 0
51
+ max_attempts: int = 1
52
+
53
+
54
+ class StepHierarchyParser:
55
+ """Parse internal names to understand pipeline hierarchy."""
56
+
57
+ @staticmethod
58
+ def parse_internal_name(internal_name: str) -> Dict[str, str]:
59
+ """
60
+ Parse internal name into components.
61
+
62
+ Examples:
63
+ - "hello" -> {"step": "hello"}
64
+ - "parallel_step.branch1.hello_stub" -> {
65
+ "composite": "parallel_step",
66
+ "branch": "branch1",
67
+ "step": "hello_stub"
68
+ }
69
+ """
70
+ parts = internal_name.split(".")
71
+
72
+ if len(parts) == 1:
73
+ return {"step": parts[0]}
74
+ elif len(parts) == 2:
75
+ return {"composite": parts[0], "branch": parts[1]}
76
+ elif len(parts) == 3:
77
+ return {"composite": parts[0], "branch": parts[1], "step": parts[2]}
78
+ else:
79
+ # Handle deeper nesting if needed
80
+ return {
81
+ "composite": parts[0],
82
+ "branch": ".".join(parts[1:-1]),
83
+ "step": parts[-1],
84
+ }
85
+
86
+ @staticmethod
87
+ def get_step_level(internal_name: str) -> int:
88
+ """Determine hierarchy level from internal name."""
89
+ parts = internal_name.split(".")
90
+ if len(parts) == 1:
91
+ return 0 # Top-level step
92
+ elif len(parts) == 2:
93
+ return 1 # Branch level (for composite step parent)
94
+ else:
95
+ return 2 # Branch step
96
+
97
+
98
+ class TimelineExtractor:
99
+ """Extract chronological timeline from run log."""
100
+
101
+ def __init__(self, run_log_data: Dict[str, Any]):
102
+ self.run_log_data = run_log_data
103
+ self.dag_nodes = (
104
+ run_log_data.get("run_config", {}).get("dag", {}).get("nodes", {})
105
+ )
106
+
107
+ def parse_time(self, time_str: str) -> Optional[datetime]:
108
+ """Parse ISO timestamp string."""
109
+ try:
110
+ return datetime.fromisoformat(time_str) if time_str else None
111
+ except (ValueError, TypeError):
112
+ return None
113
+
114
+ def get_step_timing(
115
+ self, step_data: Dict[str, Any]
116
+ ) -> Tuple[Optional[datetime], Optional[datetime], float]:
117
+ """Extract timing from step attempts."""
118
+ attempts = step_data.get("attempts", [])
119
+ if not attempts:
120
+ return None, None, 0
121
+
122
+ attempt = attempts[0]
123
+ start = self.parse_time(attempt.get("start_time"))
124
+ end = self.parse_time(attempt.get("end_time"))
125
+
126
+ if start and end:
127
+ duration_ms = (end - start).total_seconds() * 1000
128
+ return start, end, duration_ms
129
+
130
+ return None, None, 0
131
+
132
+ def find_dag_node(self, internal_name: str, clean_name: str) -> Dict[str, Any]:
133
+ """Find DAG node info for command details."""
134
+ # Try direct lookup first
135
+ if clean_name in self.dag_nodes:
136
+ return self.dag_nodes[clean_name]
137
+
138
+ # For composite steps, look in branch structures
139
+ hierarchy = StepHierarchyParser.parse_internal_name(internal_name)
140
+ if "composite" in hierarchy:
141
+ composite_node = self.dag_nodes.get(hierarchy["composite"], {})
142
+ if composite_node.get("is_composite"):
143
+ branches = composite_node.get("branches", {})
144
+ branch_key = hierarchy.get("branch", "")
145
+
146
+ if branch_key in branches:
147
+ branch_nodes = branches[branch_key].get("nodes", {})
148
+ if clean_name in branch_nodes:
149
+ return branch_nodes[clean_name]
150
+ # For map nodes, the branch structure might be different
151
+ elif "branch" in composite_node: # Map node structure
152
+ branch_nodes = composite_node["branch"].get("nodes", {})
153
+ if clean_name in branch_nodes:
154
+ return branch_nodes[clean_name]
155
+
156
+ return {}
157
+
158
+ def _format_parameter_value(self, value: Any, kind: str) -> str:
159
+ """Format parameter value for display."""
160
+ if kind == "metric":
161
+ if isinstance(value, (int, float)):
162
+ return f"{value:.3g}"
163
+ return str(value)
164
+
165
+ if isinstance(value, str):
166
+ # Truncate long strings
167
+ if len(value) > 50:
168
+ return f'"{value[:47]}..."'
169
+ return f'"{value}"'
170
+ elif isinstance(value, (list, tuple)):
171
+ if len(value) > 3:
172
+ preview = ", ".join(str(v) for v in value[:3])
173
+ return f"[{preview}, ...+{len(value)-3}]"
174
+ return str(value)
175
+ elif isinstance(value, dict):
176
+ if len(value) > 2:
177
+ keys = list(value.keys())[:2]
178
+ preview = ", ".join(f'"{k}": {value[k]}' for k in keys)
179
+ return f"{{{preview}, ...+{len(value)-2}}}"
180
+ return str(value)
181
+ else:
182
+ return str(value)
183
+
184
+ def _categorize_parameters(
185
+ self,
186
+ input_params: List[str],
187
+ output_params: List[str],
188
+ step_name: str,
189
+ internal_name: str,
190
+ ) -> ParameterInfo:
191
+ """Categorize parameters for smarter display."""
192
+ # Common pipeline parameter patterns
193
+ pipeline_params = {"integer", "floater", "stringer", "pydantic_param", "chunks"}
194
+
195
+ # Categorize inputs
196
+ key_inputs = []
197
+ context_params = []
198
+ iteration_vars = []
199
+
200
+ for param in input_params:
201
+ param_name = param.split("=")[0]
202
+
203
+ # Check for iteration variables (chunk, index, etc.)
204
+ if param_name in {"chunk", "index", "iteration", "item"}:
205
+ iteration_vars.append(param)
206
+ # Check for step-specific parameters (not empty and not pipeline defaults)
207
+ elif (
208
+ param_name not in pipeline_params
209
+ and "=" in param
210
+ and param.split("=")[1] not in ['""', "''", "[]", "{}"]
211
+ ):
212
+ key_inputs.append(param)
213
+ # Pipeline/context parameters
214
+ elif param_name in pipeline_params:
215
+ context_params.append(param)
216
+ else:
217
+ context_params.append(param)
218
+
219
+ # Limit key inputs to most relevant ones
220
+ if len(key_inputs) > 3:
221
+ key_inputs = key_inputs[:3]
222
+
223
+ # Add iteration vars to key inputs if present
224
+ key_inputs.extend(iteration_vars)
225
+
226
+ # Identify step outputs (parameters generated by this step)
227
+ step_outputs = []
228
+ for param in output_params:
229
+ param_name = param.split("=")[0]
230
+ # Step outputs often have prefixes like "1_processed_python"
231
+ if param_name.startswith(
232
+ f"{step_name.split('.')[-1]}_"
233
+ ) or param_name.startswith(f"{internal_name.split('.')[-1]}_"):
234
+ step_outputs.append(param)
235
+ else:
236
+ step_outputs.append(param)
237
+
238
+ return ParameterInfo(
239
+ key_inputs=key_inputs,
240
+ step_outputs=step_outputs,
241
+ context_params=context_params,
242
+ iteration_vars=iteration_vars,
243
+ all_inputs=input_params,
244
+ all_outputs=output_params,
245
+ )
246
+
247
+ def extract_timeline(self) -> List[StepInfo]:
248
+ """Extract all steps in chronological order."""
249
+ steps = []
250
+
251
+ # Process top-level steps
252
+ for step_name, step_data in self.run_log_data.get("steps", {}).items():
253
+ step_info = self._create_step_info(step_name, step_data)
254
+ steps.append(step_info)
255
+
256
+ # Process branches if they exist
257
+ branches = step_data.get("branches", {})
258
+ for branch_name, branch_data in branches.items():
259
+ # Add branch steps
260
+ for sub_step_name, sub_step_data in branch_data.get(
261
+ "steps", {}
262
+ ).items():
263
+ sub_step_info = self._create_step_info(
264
+ sub_step_name,
265
+ sub_step_data,
266
+ parent=step_name,
267
+ branch=branch_name,
268
+ )
269
+ steps.append(sub_step_info)
270
+
271
+ # Sort by start time for chronological order
272
+ return sorted(steps, key=lambda x: x.start_time or datetime.min)
273
+
274
+ def _create_step_info(
275
+ self,
276
+ step_name: str,
277
+ step_data: Dict[str, Any],
278
+ parent: Optional[str] = None,
279
+ branch: Optional[str] = None,
280
+ ) -> StepInfo:
281
+ """Create StepInfo from raw step data."""
282
+ internal_name = step_data.get("internal_name", step_name)
283
+ clean_name = step_data.get("name", step_name)
284
+
285
+ # Get timing
286
+ start, end, duration = self.get_step_timing(step_data)
287
+
288
+ # Get command info from DAG
289
+ dag_node = self.find_dag_node(internal_name, clean_name)
290
+ command = dag_node.get("command", "")
291
+ command_type = dag_node.get("command_type", "")
292
+
293
+ # Extract parameters with detailed metadata (exclude pickled/object types)
294
+ input_params = []
295
+ output_params = []
296
+ catalog_ops: Dict[str, List[Dict[str, str]]] = {"put": [], "get": []}
297
+
298
+ attempts = step_data.get("attempts", [])
299
+ if attempts:
300
+ attempt = attempts[0]
301
+ input_param_data = attempt.get("input_parameters", {})
302
+ output_param_data = attempt.get("output_parameters", {})
303
+
304
+ # Process input parameters (exclude object/pickled types)
305
+ for name, param in input_param_data.items():
306
+ if isinstance(param, dict):
307
+ kind = param.get("kind", "")
308
+ if kind in ("json", "metric"):
309
+ value = param.get("value", "")
310
+ # Format value for display
311
+ formatted_value = self._format_parameter_value(value, kind)
312
+ input_params.append(f"{name}={formatted_value}")
313
+ # Skip object/pickled parameters entirely
314
+
315
+ # Process output parameters (exclude object/pickled types)
316
+ for name, param in output_param_data.items():
317
+ if isinstance(param, dict):
318
+ kind = param.get("kind", "")
319
+ if kind in ("json", "metric"):
320
+ value = param.get("value", "")
321
+ # Format value for display
322
+ formatted_value = self._format_parameter_value(value, kind)
323
+ output_params.append(f"{name}={formatted_value}")
324
+ # Skip object/pickled parameters entirely
325
+
326
+ # Extract error information and attempt details
327
+ error_message = None
328
+ attempt_count = len(attempts)
329
+ max_attempts = 1 # Default, will be updated from DAG if available
330
+
331
+ # Get max_attempts from DAG node
332
+ max_attempts = dag_node.get("max_attempts", 1)
333
+
334
+ # Extract error message from failed attempts
335
+ if attempts and step_data.get("status") == "FAIL":
336
+ # Find the failed attempt (usually the last one)
337
+ failed_attempt = None
338
+ for attempt in reversed(attempts):
339
+ if attempt.get("status") == "FAIL":
340
+ failed_attempt = attempt
341
+ break
342
+
343
+ if failed_attempt and failed_attempt.get("message"):
344
+ error_message = failed_attempt["message"].strip()
345
+
346
+ # Extract catalog operations with detailed information
347
+ catalog_data = step_data.get("data_catalog", [])
348
+ for item in catalog_data:
349
+ stage = item.get("stage", "")
350
+ if stage in ("put", "get"):
351
+ catalog_info = {
352
+ "name": item.get("name", ""),
353
+ "data_hash": item.get("data_hash", "")[:8] + "..."
354
+ if item.get("data_hash")
355
+ else "", # Show first 8 chars
356
+ "catalog_path": item.get("catalog_relative_path", ""),
357
+ "stage": stage,
358
+ }
359
+ catalog_ops[stage].append(catalog_info)
360
+
361
+ # Create parameter categorization for better display
362
+ parameters = self._categorize_parameters(
363
+ input_params, output_params, clean_name, internal_name
364
+ )
365
+
366
+ return StepInfo(
367
+ name=clean_name,
368
+ internal_name=internal_name,
369
+ status=step_data.get("status", "UNKNOWN"),
370
+ step_type=step_data.get("step_type", "task"),
371
+ start_time=start,
372
+ end_time=end,
373
+ duration_ms=duration,
374
+ level=StepHierarchyParser.get_step_level(internal_name),
375
+ parent=parent,
376
+ branch=branch,
377
+ command=command,
378
+ command_type=command_type,
379
+ input_params=input_params,
380
+ output_params=output_params,
381
+ catalog_ops=catalog_ops,
382
+ parameters=parameters,
383
+ error_message=error_message,
384
+ attempt_count=attempt_count,
385
+ max_attempts=max_attempts,
386
+ )
387
+
388
+
389
+ class SimpleVisualizer:
390
+ """Simple, lightweight pipeline visualizer."""
391
+
392
+ def __init__(self, run_log_path: Union[str, Path]):
393
+ self.run_log_path = Path(run_log_path)
394
+ self.run_log_data = self._load_run_log()
395
+ self.extractor = TimelineExtractor(self.run_log_data)
396
+ self.timeline = self.extractor.extract_timeline()
397
+
398
+ def _load_run_log(self) -> Dict[str, Any]:
399
+ """Load run log JSON."""
400
+ if not self.run_log_path.exists():
401
+ raise FileNotFoundError(f"Run log not found: {self.run_log_path}")
402
+
403
+ with open(self.run_log_path, "r") as f:
404
+ return json.load(f)
405
+
406
+ def print_simple_timeline(self) -> None:
407
+ """Print a clean console timeline."""
408
+ run_id = self.run_log_data.get("run_id", "unknown")
409
+ status = self.run_log_data.get("status", "UNKNOWN")
410
+
411
+ print(f"\n🔄 Pipeline Timeline - {run_id}")
412
+ print(f"Status: {status}")
413
+ print("=" * 80)
414
+
415
+ # Group by composite steps for better display
416
+ current_composite = None
417
+ current_branch = None
418
+
419
+ for step in self.timeline:
420
+ # Skip composite steps themselves (they have no timing)
421
+ if (
422
+ step.step_type in ["parallel", "map", "conditional"]
423
+ and not step.start_time
424
+ ):
425
+ continue
426
+
427
+ # Detect composite/branch changes
428
+ hierarchy = StepHierarchyParser.parse_internal_name(step.internal_name)
429
+ composite = hierarchy.get("composite")
430
+ branch = hierarchy.get("branch")
431
+
432
+ # Show composite header
433
+ if composite and composite != current_composite:
434
+ print(f"\n🔀 {composite} ({self._get_composite_type(composite)})")
435
+ current_composite = composite
436
+ current_branch = None
437
+
438
+ # Show branch header
439
+ if branch and branch != current_branch:
440
+ branch_display = self._format_branch_name(composite or "", branch)
441
+ print(f" ├─ Branch: {branch_display}")
442
+ current_branch = branch
443
+
444
+ # Show step
445
+ indent = (
446
+ " " if step.level == 0 else " " if step.level == 1 else " "
447
+ )
448
+ status_emoji = (
449
+ "✅"
450
+ if step.status == "SUCCESS"
451
+ else "❌"
452
+ if step.status == "FAIL"
453
+ else "⏸️"
454
+ )
455
+
456
+ # Type icon
457
+ type_icons = {
458
+ "task": "⚙️",
459
+ "stub": "📝",
460
+ "success": "✅",
461
+ "fail": "❌",
462
+ "parallel": "🔀",
463
+ "map": "🔁",
464
+ "conditional": "🔀",
465
+ }
466
+ type_icon = type_icons.get(step.step_type, "⚙️")
467
+
468
+ timing = f"({step.duration_ms:.1f}ms)" if step.duration_ms > 0 else ""
469
+
470
+ print(f"{indent}{type_icon} {status_emoji} {step.name} {timing}")
471
+
472
+ # Show error information for failed steps
473
+ if step.status == "FAIL" and step.error_message:
474
+ error_lines = step.error_message.split("\n")
475
+ # Show first line of error, truncated if too long
476
+ error_preview = (
477
+ error_lines[0][:100] + "..."
478
+ if len(error_lines[0]) > 100
479
+ else error_lines[0]
480
+ )
481
+ print(f"{indent} 💥 Error: {error_preview}")
482
+ if step.attempt_count > 1:
483
+ print(
484
+ f"{indent} 🔄 Failed after {step.attempt_count}/{step.max_attempts} attempts"
485
+ )
486
+
487
+ # Show metadata for tasks
488
+ if step.step_type == "task" and (
489
+ step.command
490
+ or step.input_params
491
+ or step.output_params
492
+ or step.catalog_ops["put"]
493
+ or step.catalog_ops["get"]
494
+ ):
495
+ if step.command:
496
+ cmd_short = (
497
+ step.command[:50] + "..."
498
+ if len(step.command) > 50
499
+ else step.command
500
+ )
501
+ print(f"{indent} 📝 {step.command_type.upper()}: {cmd_short}")
502
+
503
+ # Show input parameters - compact horizontal display
504
+ if step.input_params:
505
+ params_display = " • ".join(step.input_params)
506
+ print(f"{indent} 📥 {params_display}")
507
+
508
+ # Show output parameters - compact horizontal display
509
+ if step.output_params:
510
+ params_display = " • ".join(step.output_params)
511
+ print(f"{indent} 📤 {params_display}")
512
+
513
+ # Show catalog operations - compact horizontal display
514
+ if step.catalog_ops.get("put") or step.catalog_ops.get("get"):
515
+ catalog_items = []
516
+ if step.catalog_ops.get("put"):
517
+ catalog_items.extend(
518
+ [f"PUT:{item['name']}" for item in step.catalog_ops["put"]]
519
+ )
520
+ if step.catalog_ops.get("get"):
521
+ catalog_items.extend(
522
+ [f"GET:{item['name']}" for item in step.catalog_ops["get"]]
523
+ )
524
+ if catalog_items:
525
+ catalog_display = " • ".join(catalog_items)
526
+ print(f"{indent} 💾 {catalog_display}")
527
+
528
+ print("=" * 80)
529
+
530
+ def _get_composite_type(self, composite_name: str) -> str:
531
+ """Get composite node type from DAG."""
532
+ dag_nodes = (
533
+ self.run_log_data.get("run_config", {}).get("dag", {}).get("nodes", {})
534
+ )
535
+ node = dag_nodes.get(composite_name, {})
536
+ return node.get("node_type", "composite")
537
+
538
+ def _format_branch_name(self, composite: str, branch: str) -> str:
539
+ """Format branch name based on composite type."""
540
+ # Remove composite prefix if present
541
+ if branch.startswith(f"{composite}."):
542
+ branch_clean = branch[len(f"{composite}.") :]
543
+ else:
544
+ branch_clean = branch
545
+
546
+ # Check if it's a map iteration (numeric)
547
+ if branch_clean.isdigit():
548
+ return f"Iteration {branch_clean}"
549
+
550
+ return branch_clean
551
+
552
+ def print_execution_summary(self) -> None:
553
+ """Print execution summary table."""
554
+ run_id = self.run_log_data.get("run_id", "unknown")
555
+
556
+ print(f"\n📊 Execution Summary - {run_id}")
557
+ print("=" * 80)
558
+
559
+ # Filter to actual executed steps (with timing)
560
+ executed_steps = [step for step in self.timeline if step.start_time]
561
+
562
+ if not executed_steps:
563
+ print("No executed steps found")
564
+ return
565
+
566
+ # Table header
567
+ print(f"{'Step':<30} {'Status':<10} {'Duration':<12} {'Type':<10}")
568
+ print("-" * 80)
569
+
570
+ total_duration = 0
571
+ success_count = 0
572
+
573
+ for step in executed_steps:
574
+ status_emoji = (
575
+ "✅"
576
+ if step.status == "SUCCESS"
577
+ else "❌"
578
+ if step.status == "FAIL"
579
+ else "⏸️"
580
+ )
581
+ duration_text = (
582
+ f"{step.duration_ms:.1f}ms" if step.duration_ms > 0 else "0.0ms"
583
+ )
584
+
585
+ # Truncate long names
586
+ display_name = step.name[:28] + ".." if len(step.name) > 30 else step.name
587
+
588
+ print(
589
+ f"{display_name:<30} {status_emoji}{step.status:<9} {duration_text:<12} {step.step_type:<10}"
590
+ )
591
+
592
+ total_duration += int(step.duration_ms)
593
+ if step.status == "SUCCESS":
594
+ success_count += 1
595
+
596
+ print("-" * 80)
597
+ success_rate = (
598
+ (success_count / len(executed_steps)) * 100 if executed_steps else 0
599
+ )
600
+ overall_status = self.run_log_data.get("status", "UNKNOWN")
601
+ overall_emoji = "✅" if overall_status == "SUCCESS" else "❌"
602
+
603
+ print(
604
+ f"Total Duration: {total_duration:.1f}ms | Success Rate: {success_rate:.1f}% | Status: {overall_emoji} {overall_status}"
605
+ )
606
+
607
+ def generate_html_timeline(
608
+ self, output_path: Optional[Union[str, Path]] = None
609
+ ) -> str:
610
+ """
611
+ Generate an interactive HTML timeline visualization.
612
+
613
+ This creates a lightweight HTML version with:
614
+ - Clean timeline layout
615
+ - Hover tooltips with metadata
616
+ - Expandable composite sections
617
+ - Timing bars proportional to execution duration
618
+ """
619
+ run_id = self.run_log_data.get("run_id", "unknown")
620
+ status = self.run_log_data.get("status", "UNKNOWN")
621
+
622
+ # Calculate total timeline for proportional bars
623
+ executed_steps = [step for step in self.timeline if step.start_time]
624
+ if executed_steps:
625
+ earliest = min(
626
+ step.start_time for step in executed_steps if step.start_time
627
+ )
628
+ latest = max(step.end_time for step in executed_steps if step.end_time)
629
+ total_duration_ms = (
630
+ (latest - earliest).total_seconds() * 1000 if latest and earliest else 1
631
+ )
632
+ else:
633
+ total_duration_ms = 1
634
+
635
+ html_content = f"""<!DOCTYPE html>
636
+ <html lang="en">
637
+ <head>
638
+ <meta charset="UTF-8">
639
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
640
+ <title>Pipeline Timeline - {run_id}</title>
641
+ <style>
642
+ * {{ margin: 0; padding: 0; box-sizing: border-box; }}
643
+
644
+ body {{
645
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, sans-serif;
646
+ background: #f8fafc;
647
+ color: #1e293b;
648
+ line-height: 1.6;
649
+ }}
650
+
651
+ .header {{
652
+ background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
653
+ color: white;
654
+ padding: 2rem 0;
655
+ text-align: center;
656
+ box-shadow: 0 4px 6px rgba(0,0,0,0.1);
657
+ }}
658
+
659
+ .container {{
660
+ max-width: 1600px;
661
+ margin: 2rem auto;
662
+ padding: 0 1rem;
663
+ display: flex;
664
+ gap: 2rem;
665
+ }}
666
+
667
+ .main-content {{
668
+ flex: 1;
669
+ min-width: 0;
670
+ }}
671
+
672
+ .sidebar {{
673
+ width: 400px;
674
+ position: sticky;
675
+ top: 2rem;
676
+ height: fit-content;
677
+ max-height: calc(100vh - 4rem);
678
+ overflow-y: auto;
679
+ background: white;
680
+ border-radius: 12px;
681
+ box-shadow: 0 2px 10px rgba(0,0,0,0.08);
682
+ transition: transform 0.3s ease;
683
+ }}
684
+
685
+ .sidebar.hidden {{
686
+ transform: translateX(100%);
687
+ }}
688
+
689
+ .sidebar-header {{
690
+ background: #f8fafc;
691
+ padding: 1.5rem;
692
+ border-bottom: 1px solid #e2e8f0;
693
+ border-radius: 12px 12px 0 0;
694
+ display: flex;
695
+ justify-content: space-between;
696
+ align-items: center;
697
+ }}
698
+
699
+ .sidebar-content {{
700
+ padding: 1.5rem;
701
+ max-height: 60vh;
702
+ overflow-y: auto;
703
+ }}
704
+
705
+ .sidebar-section {{
706
+ margin-bottom: 1.5rem;
707
+ }}
708
+
709
+ .sidebar-section h4 {{
710
+ color: #374151;
711
+ font-size: 0.875rem;
712
+ font-weight: 600;
713
+ margin-bottom: 0.75rem;
714
+ text-transform: uppercase;
715
+ letter-spacing: 0.05em;
716
+ }}
717
+
718
+ .param-grid {{
719
+ display: grid;
720
+ gap: 0.5rem;
721
+ }}
722
+
723
+ .param-item {{
724
+ padding: 0.5rem 0.75rem;
725
+ border-radius: 6px;
726
+ font-family: monospace;
727
+ font-size: 0.8rem;
728
+ border-left: 3px solid;
729
+ }}
730
+
731
+ .param-item.input {{
732
+ background: #f0fdf4;
733
+ border-left-color: #22c55e;
734
+ color: #166534;
735
+ }}
736
+
737
+ .param-item.output {{
738
+ background: #fef2f2;
739
+ border-left-color: #ef4444;
740
+ color: #991b1b;
741
+ }}
742
+
743
+ .param-item.context {{
744
+ background: #f8fafc;
745
+ border-left-color: #64748b;
746
+ color: #475569;
747
+ }}
748
+
749
+ .param-item.iteration {{
750
+ background: #fefce8;
751
+ border-left-color: #eab308;
752
+ color: #854d0e;
753
+ }}
754
+
755
+ .step-clickable {{
756
+ cursor: pointer;
757
+ transition: background-color 0.2s ease;
758
+ }}
759
+
760
+ .step-clickable:hover {{
761
+ background-color: #f1f5f9 !important;
762
+ }}
763
+
764
+ .step-clickable.selected {{
765
+ background-color: #dbeafe !important;
766
+ border-left: 3px solid #3b82f6;
767
+ }}
768
+
769
+ .close-sidebar {{
770
+ background: none;
771
+ border: none;
772
+ color: #64748b;
773
+ cursor: pointer;
774
+ font-size: 1.2rem;
775
+ padding: 0.25rem;
776
+ }}
777
+
778
+ .close-sidebar:hover {{
779
+ color: #374151;
780
+ }}
781
+
782
+ .timeline-card {{
783
+ background: white;
784
+ border-radius: 12px;
785
+ box-shadow: 0 2px 10px rgba(0,0,0,0.08);
786
+ overflow: hidden;
787
+ margin-bottom: 2rem;
788
+ }}
789
+
790
+ .timeline-header {{
791
+ background: #f8fafc;
792
+ padding: 1.5rem;
793
+ border-bottom: 1px solid #e2e8f0;
794
+ display: flex;
795
+ justify-content: space-between;
796
+ align-items: center;
797
+ }}
798
+
799
+ .timeline-content {{
800
+ padding: 1rem;
801
+ }}
802
+
803
+ .step-row {{
804
+ display: grid;
805
+ grid-template-columns: 300px 1fr 80px;
806
+ align-items: center;
807
+ padding: 0.5rem 0;
808
+ border-bottom: 1px solid #f1f5f9;
809
+ transition: background 0.2s ease;
810
+ gap: 1rem;
811
+ min-height: 40px;
812
+ overflow: visible;
813
+ }}
814
+
815
+ .step-row:hover {{
816
+ background: #f8fafc;
817
+ }}
818
+
819
+ .step-info {{
820
+ display: flex;
821
+ flex-direction: column;
822
+ gap: 0.25rem;
823
+ font-weight: 500;
824
+ min-height: 24px;
825
+ justify-content: flex-start;
826
+ overflow: visible;
827
+ word-wrap: break-word;
828
+ overflow-wrap: break-word;
829
+ }}
830
+
831
+ .step-level-0 {{ padding-left: 0; }}
832
+ .step-level-1 {{ padding-left: 1rem; }}
833
+ .step-level-2 {{ padding-left: 2rem; }}
834
+
835
+ .composite-header {{
836
+ background: #e0f2fe !important;
837
+ border-left: 4px solid #0277bd;
838
+ font-weight: 600;
839
+ color: #01579b;
840
+ }}
841
+
842
+ .branch-header {{
843
+ background: #f3e5f5 !important;
844
+ border-left: 4px solid #7b1fa2;
845
+ font-weight: 600;
846
+ color: #4a148c;
847
+ }}
848
+
849
+ .gantt-container {{
850
+ position: relative;
851
+ height: 30px;
852
+ background: #f8fafc;
853
+ border: 1px solid #e2e8f0;
854
+ border-radius: 4px;
855
+ min-width: 100%;
856
+ overflow: hidden;
857
+ }}
858
+
859
+ .gantt-bar {{
860
+ position: absolute;
861
+ top: 3px;
862
+ height: 24px;
863
+ border-radius: 3px;
864
+ transition: all 0.2s ease;
865
+ cursor: pointer;
866
+ border: 1px solid rgba(255,255,255,0.3);
867
+ }}
868
+
869
+ .gantt-bar:hover {{
870
+ transform: scaleY(1.1);
871
+ z-index: 10;
872
+ box-shadow: 0 2px 8px rgba(0,0,0,0.2);
873
+ }}
874
+
875
+ .time-grid {{
876
+ position: absolute;
877
+ top: 0;
878
+ bottom: 0;
879
+ border-left: 1px solid #e2e8f0;
880
+ opacity: 0.3;
881
+ }}
882
+
883
+ .time-scale {{
884
+ position: relative;
885
+ height: 20px;
886
+ background: #f1f5f9;
887
+ border-bottom: 1px solid #d1d5db;
888
+ font-size: 0.75rem;
889
+ color: #6b7280;
890
+ }}
891
+
892
+ .time-marker {{
893
+ position: absolute;
894
+ top: 0;
895
+ height: 100%;
896
+ display: flex;
897
+ align-items: center;
898
+ padding-left: 4px;
899
+ font-weight: 500;
900
+ }}
901
+
902
+ .timeline-bar:hover {{
903
+ transform: scaleY(1.1);
904
+ z-index: 10;
905
+ }}
906
+
907
+ .bar-success {{ background: linear-gradient(90deg, #22c55e, #16a34a); }}
908
+ .bar-fail {{ background: linear-gradient(90deg, #ef4444, #dc2626); }}
909
+ .bar-unknown {{ background: linear-gradient(90deg, #f59e0b, #d97706); }}
910
+
911
+ .duration-text {{
912
+ font-family: monospace;
913
+ font-size: 0.875rem;
914
+ font-weight: 600;
915
+ }}
916
+
917
+ .duration-fast {{ color: #16a34a; }}
918
+ .duration-medium {{ color: #f59e0b; }}
919
+ .duration-slow {{ color: #dc2626; }}
920
+
921
+ .status-success {{ color: #16a34a; }}
922
+ .status-fail {{ color: #dc2626; }}
923
+ .status-unknown {{ color: #f59e0b; }}
924
+
925
+ .expandable {{
926
+ cursor: pointer;
927
+ user-select: none;
928
+ }}
929
+
930
+ .expandable:hover {{
931
+ background: #e2e8f0 !important;
932
+ }}
933
+
934
+ .step-header.expandable {{
935
+ padding: 0.25rem;
936
+ border-radius: 4px;
937
+ margin: -0.25rem;
938
+ }}
939
+
940
+ .step-header.expandable:hover {{
941
+ background: #f1f5f9 !important;
942
+ }}
943
+
944
+ .collapsible-content {{
945
+ max-height: none;
946
+ overflow: visible;
947
+ transition: max-height 0.3s ease;
948
+ }}
949
+
950
+ .collapsible-content.collapsed {{
951
+ max-height: 0;
952
+ overflow: hidden;
953
+ }}
954
+
955
+ .summary-grid {{
956
+ display: grid;
957
+ grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
958
+ gap: 1.5rem;
959
+ margin-top: 2rem;
960
+ }}
961
+
962
+ .summary-card {{
963
+ background: white;
964
+ padding: 1.5rem;
965
+ border-radius: 8px;
966
+ box-shadow: 0 2px 10px rgba(0,0,0,0.08);
967
+ text-align: center;
968
+ }}
969
+
970
+ .summary-number {{
971
+ font-size: 2rem;
972
+ font-weight: bold;
973
+ margin-bottom: 0.5rem;
974
+ }}
975
+
976
+ .summary-label {{
977
+ color: #64748b;
978
+ font-size: 0.875rem;
979
+ }}
980
+ </style>
981
+ </head>
982
+ <body>
983
+ <div class="header">
984
+ <h1>🔄 Pipeline Timeline Visualization</h1>
985
+ <p>Interactive execution analysis for {run_id}</p>
986
+ </div>
987
+
988
+ <div class="container">
989
+ <div class="main-content">
990
+ <div class="timeline-card">
991
+ <div class="timeline-header">
992
+ <div>
993
+ <h2>Run ID: {run_id}</h2>
994
+ <p>Status: <span class="status-{status.lower()}">{status}</span></p>
995
+ </div>
996
+ <div>
997
+ <span class="duration-text">Total: {total_duration_ms:.1f}ms</span>
998
+ </div>
999
+ </div>
1000
+
1001
+ <div class="timeline-content">
1002
+ {self._generate_html_timeline_rows(total_duration_ms)}
1003
+ </div>
1004
+ </div>
1005
+
1006
+ {self._generate_html_summary()}
1007
+ </div>
1008
+
1009
+ <div class="sidebar hidden" id="parameterSidebar">
1010
+ <div class="sidebar-header">
1011
+ <div>
1012
+ <h3>📊 Step Details</h3>
1013
+ <p id="sidebarStepName" style="color: #64748b; font-size: 0.875rem; margin: 0;">Select a step to view details</p>
1014
+ </div>
1015
+ <button class="close-sidebar" onclick="closeSidebar()">×</button>
1016
+ </div>
1017
+ <div class="sidebar-content" id="sidebarContent">
1018
+ <div class="sidebar-section">
1019
+ <p style="color: #64748b; text-align: center; padding: 2rem;">Click on any step in the timeline to view its detailed parameters and metadata.</p>
1020
+ </div>
1021
+ </div>
1022
+ </div>
1023
+ </div>
1024
+
1025
+ <script>
1026
+ let currentSelectedStep = null;
1027
+
1028
+ // Collapsible sections
1029
+ document.querySelectorAll('.expandable').forEach(element => {{
1030
+ element.addEventListener('click', (e) => {{
1031
+ e.stopPropagation(); // Prevent triggering step selection
1032
+ let content;
1033
+
1034
+ // Handle step metadata (using data-target)
1035
+ if (element.dataset.target) {{
1036
+ content = document.getElementById(element.dataset.target);
1037
+ }} else {{
1038
+ // Handle composite sections (using nextElementSibling)
1039
+ content = element.nextElementSibling;
1040
+ }}
1041
+
1042
+ if (content && content.classList.contains('collapsible-content')) {{
1043
+ content.classList.toggle('collapsed');
1044
+
1045
+ // Update expand/collapse indicator
1046
+ const indicator = element.querySelector('.expand-indicator');
1047
+ if (indicator) {{
1048
+ indicator.textContent = content.classList.contains('collapsed') ? '▶' : '▼';
1049
+ }}
1050
+ }}
1051
+ }});
1052
+ }});
1053
+
1054
+ function showStepDetails(internalName, startTime) {{
1055
+ const sidebar = document.getElementById('parameterSidebar');
1056
+ const stepNameElement = document.getElementById('sidebarStepName');
1057
+ const contentElement = document.getElementById('sidebarContent');
1058
+
1059
+ // Get step data from embedded JSON
1060
+ const dataId = `step-data-${{internalName.replace(/\\./g, '-')}}-${{startTime}}`;
1061
+ const dataElement = document.getElementById(dataId);
1062
+
1063
+ if (!dataElement) {{
1064
+ console.error('Step data not found for:', dataId);
1065
+ return;
1066
+ }}
1067
+
1068
+ try {{
1069
+ const stepData = JSON.parse(dataElement.textContent);
1070
+
1071
+ // Update step selection visual feedback
1072
+ if (currentSelectedStep) {{
1073
+ currentSelectedStep.classList.remove('selected');
1074
+ }}
1075
+
1076
+ const stepRow = event.currentTarget;
1077
+ stepRow.classList.add('selected');
1078
+ currentSelectedStep = stepRow;
1079
+
1080
+ // Update sidebar content
1081
+ stepNameElement.textContent = `${{stepData.name}} (${{stepData.duration_ms.toFixed(1)}}ms)`;
1082
+
1083
+ const perfEmoji = stepData.performance_class === 'fast' ? '🟢' :
1084
+ stepData.performance_class === 'medium' ? '🟡' : '🔴';
1085
+
1086
+ const statusEmoji = stepData.status === 'SUCCESS' ? '✅' :
1087
+ stepData.status === 'FAIL' ? '❌' : '⏸️';
1088
+
1089
+ contentElement.innerHTML = `
1090
+ <div class="sidebar-section">
1091
+ <h4>📋 Step Information</h4>
1092
+ <div style="display: grid; gap: 0.5rem; font-size: 0.875rem;">
1093
+ <div><strong>Status:</strong> ${{statusEmoji}} ${{stepData.status}}</div>
1094
+ <div><strong>Type:</strong> ${{stepData.step_type}}</div>
1095
+ <div><strong>Duration:</strong> ${{perfEmoji}} ${{stepData.duration_ms.toFixed(1)}}ms</div>
1096
+ <div><strong>Time:</strong> ${{stepData.start_time}} → ${{stepData.end_time}}</div>
1097
+ ${{stepData.attempt_count > 1 ? `<div><strong>Attempts:</strong> ${{stepData.attempt_count}}/${{stepData.max_attempts}}</div>` : ''}}
1098
+ </div>
1099
+ </div>
1100
+
1101
+ ${{stepData.error_message ? `
1102
+ <div class="sidebar-section">
1103
+ <h4 style="color: #dc2626;">💥 Error Details</h4>
1104
+ <div class="param-item" style="background: #fef2f2; border-left-color: #dc2626; color: #991b1b; white-space: pre-wrap; font-family: monospace; font-size: 0.75rem; line-height: 1.4;">
1105
+ ${{stepData.error_message}}
1106
+ </div>
1107
+ ${{stepData.attempt_count > 1 ? `
1108
+ <div style="margin-top: 0.5rem; font-size: 0.8rem; color: #6b7280;">
1109
+ ⚠️ Step failed after ${{stepData.attempt_count}} attempt(s)
1110
+ </div>
1111
+ ` : ''}}
1112
+ </div>
1113
+ ` : ''}}
1114
+
1115
+
1116
+ ${{stepData.command ? `
1117
+ <div class="sidebar-section">
1118
+ <h4>🔧 Command</h4>
1119
+ <div class="param-item context">
1120
+ <strong>${{stepData.command_type.toUpperCase()}}:</strong><br>
1121
+ ${{stepData.command}}
1122
+ </div>
1123
+ </div>
1124
+ ` : ''}}
1125
+
1126
+ ${{stepData.parameters.key_inputs.length > 0 ? `
1127
+ <div class="sidebar-section">
1128
+ <h4>🎯 Key Inputs</h4>
1129
+ <div class="param-grid">
1130
+ ${{stepData.parameters.key_inputs.map(param =>
1131
+ `<div class="param-item input">${{param}}</div>`
1132
+ ).join('')}}
1133
+ </div>
1134
+ </div>
1135
+ ` : ''}}
1136
+
1137
+ ${{stepData.parameters.step_outputs.length > 0 ? `
1138
+ <div class="sidebar-section">
1139
+ <h4>📤 Step Outputs</h4>
1140
+ <div class="param-grid">
1141
+ ${{stepData.parameters.step_outputs.map(param =>
1142
+ `<div class="param-item output">${{param}}</div>`
1143
+ ).join('')}}
1144
+ </div>
1145
+ </div>
1146
+ ` : ''}}
1147
+
1148
+ ${{stepData.parameters.iteration_vars.length > 0 ? `
1149
+ <div class="sidebar-section">
1150
+ <h4>🔁 Iteration Variables</h4>
1151
+ <div class="param-grid">
1152
+ ${{stepData.parameters.iteration_vars.map(param =>
1153
+ `<div class="param-item iteration">${{param}}</div>`
1154
+ ).join('')}}
1155
+ </div>
1156
+ </div>
1157
+ ` : ''}}
1158
+
1159
+ ${{(stepData.catalog_ops.put.length > 0 || stepData.catalog_ops.get.length > 0) ? `
1160
+ <div class="sidebar-section">
1161
+ <h4>💾 Data Operations</h4>
1162
+ ${{stepData.catalog_ops.get.length > 0 ? `
1163
+ <div style="margin-bottom: 1rem;">
1164
+ <h5 style="color: #059669; font-size: 0.8rem; font-weight: 600; margin-bottom: 0.5rem;">📥 CATALOG INPUTS</h5>
1165
+ <div class="param-grid">
1166
+ ${{stepData.catalog_ops.get.map(item =>
1167
+ `<div class="param-item input" style="font-family: monospace;">
1168
+ <div style="font-weight: 600; margin-bottom: 0.25rem;">${{item.name}}</div>
1169
+ <div style="font-size: 0.7rem; opacity: 0.8;">Hash: ${{item.data_hash}}</div>
1170
+ <div style="font-size: 0.7rem; opacity: 0.8;">Path: ${{item.catalog_path}}</div>
1171
+ </div>`
1172
+ ).join('')}}
1173
+ </div>
1174
+ </div>
1175
+ ` : ''}}
1176
+ ${{stepData.catalog_ops.put.length > 0 ? `
1177
+ <div>
1178
+ <h5 style="color: #dc2626; font-size: 0.8rem; font-weight: 600; margin-bottom: 0.5rem;">📤 CATALOG OUTPUTS</h5>
1179
+ <div class="param-grid">
1180
+ ${{stepData.catalog_ops.put.map(item =>
1181
+ `<div class="param-item output" style="font-family: monospace;">
1182
+ <div style="font-weight: 600; margin-bottom: 0.25rem;">${{item.name}}</div>
1183
+ <div style="font-size: 0.7rem; opacity: 0.8;">Hash: ${{item.data_hash}}</div>
1184
+ <div style="font-size: 0.7rem; opacity: 0.8;">Path: ${{item.catalog_path}}</div>
1185
+ </div>`
1186
+ ).join('')}}
1187
+ </div>
1188
+ </div>
1189
+ ` : ''}}
1190
+ </div>
1191
+ ` : ''}}
1192
+
1193
+ ${{stepData.parameters.context_params.length > 0 ? `
1194
+ <div class="sidebar-section">
1195
+ <details>
1196
+ <summary style="cursor: pointer; padding: 0.5rem 0; font-weight: 600; color: #374151;">
1197
+ 📊 Pipeline Context (${{stepData.parameters.context_params.length}} params)
1198
+ </summary>
1199
+ <div class="param-grid" style="margin-top: 0.5rem;">
1200
+ ${{stepData.parameters.context_params.map(param =>
1201
+ `<div class="param-item context">${{param}}</div>`
1202
+ ).join('')}}
1203
+ </div>
1204
+ </details>
1205
+ </div>
1206
+ ` : ''}}
1207
+ `;
1208
+
1209
+ // Show sidebar
1210
+ sidebar.classList.remove('hidden');
1211
+
1212
+ }} catch (error) {{
1213
+ console.error('Error parsing step data:', error);
1214
+ contentElement.innerHTML = `<div class="sidebar-section"><p style="color: #ef4444;">Error loading step details</p></div>`;
1215
+ sidebar.classList.remove('hidden');
1216
+ }}
1217
+ }}
1218
+
1219
+ function closeSidebar() {{
1220
+ const sidebar = document.getElementById('parameterSidebar');
1221
+ sidebar.classList.add('hidden');
1222
+
1223
+ // Clear step selection
1224
+ if (currentSelectedStep) {{
1225
+ currentSelectedStep.classList.remove('selected');
1226
+ currentSelectedStep = null;
1227
+ }}
1228
+ }}
1229
+
1230
+ // Close sidebar when clicking outside
1231
+ document.addEventListener('click', (e) => {{
1232
+ const sidebar = document.getElementById('parameterSidebar');
1233
+ const isClickInsideSidebar = sidebar.contains(e.target);
1234
+ const isClickOnStep = e.target.closest('.step-clickable');
1235
+
1236
+ if (!isClickInsideSidebar && !isClickOnStep && !sidebar.classList.contains('hidden')) {{
1237
+ closeSidebar();
1238
+ }}
1239
+ }});
1240
+
1241
+ // Keyboard shortcuts
1242
+ document.addEventListener('keydown', (e) => {{
1243
+ if (e.key === 'Escape') {{
1244
+ closeSidebar();
1245
+ }}
1246
+ }});
1247
+ </script>
1248
+ </body>
1249
+ </html>"""
1250
+
1251
+ # Save to file if path provided
1252
+ if output_path:
1253
+ Path(output_path).write_text(html_content)
1254
+ print(f"HTML timeline saved to: {output_path}")
1255
+
1256
+ return html_content
1257
+
1258
+ def _generate_html_timeline_rows(self, total_duration_ms: float) -> str:
1259
+ """Generate HTML rows for the Gantt chart timeline display."""
1260
+ executed_steps = [
1261
+ step for step in self.timeline if step.start_time and step.end_time
1262
+ ]
1263
+
1264
+ if not executed_steps:
1265
+ return "<div>No executed steps found</div>"
1266
+
1267
+ # Calculate the absolute timeline
1268
+ earliest_start = min(
1269
+ step.start_time for step in executed_steps if step.start_time
1270
+ )
1271
+ latest_end = max(step.end_time for step in executed_steps if step.end_time)
1272
+ total_timeline_ms = (
1273
+ (latest_end - earliest_start).total_seconds() * 1000
1274
+ if latest_end and earliest_start
1275
+ else 1
1276
+ )
1277
+
1278
+ # Generate time scale and Gantt rows
1279
+ time_scale_html = self._generate_time_scale(total_timeline_ms)
1280
+ gantt_rows_html = self._generate_gantt_rows(
1281
+ executed_steps, earliest_start, total_timeline_ms
1282
+ )
1283
+
1284
+ return time_scale_html + "\n" + gantt_rows_html
1285
+
1286
+ def _generate_time_scale(self, total_timeline_ms: float) -> str:
1287
+ """Generate the time scale header for the Gantt chart."""
1288
+ # Create time markers at regular intervals
1289
+ num_markers = 10
1290
+ interval_ms = total_timeline_ms / num_markers
1291
+
1292
+ markers_html = []
1293
+ for i in range(num_markers + 1):
1294
+ time_ms = i * interval_ms
1295
+ position_percent = (time_ms / total_timeline_ms) * 100
1296
+ time_display = (
1297
+ f"{time_ms:.0f}ms" if time_ms < 1000 else f"{time_ms/1000:.1f}s"
1298
+ )
1299
+
1300
+ markers_html.append(f"""
1301
+ <div class="time-marker" style="left: {position_percent:.1f}%;">
1302
+ {time_display}
1303
+ </div>
1304
+ """)
1305
+
1306
+ # Add grid line (except for the first one)
1307
+ if i > 0:
1308
+ markers_html.append(
1309
+ f'<div class="time-grid" style="left: {position_percent:.1f}%;"></div>'
1310
+ )
1311
+
1312
+ return f"""
1313
+ <div class="step-row" style="border-bottom: 2px solid #d1d5db;">
1314
+ <div class="step-info">
1315
+ <strong>Timeline</strong>
1316
+ </div>
1317
+ <div class="time-scale">
1318
+ {"".join(markers_html)}
1319
+ </div>
1320
+ <div></div>
1321
+ </div>
1322
+ """
1323
+
1324
+ def _generate_gantt_rows(
1325
+ self, executed_steps: List, earliest_start, total_timeline_ms: float
1326
+ ) -> str:
1327
+ """Generate HTML rows for the Gantt chart display."""
1328
+ html_parts = []
1329
+
1330
+ # Group by composite steps for better display
1331
+ current_composite = None
1332
+ current_branch = None
1333
+
1334
+ for step in executed_steps:
1335
+ # Calculate timing positions for Gantt chart
1336
+ start_offset_ms = (step.start_time - earliest_start).total_seconds() * 1000
1337
+ start_percent = (start_offset_ms / total_timeline_ms) * 100
1338
+ width_percent = (step.duration_ms / total_timeline_ms) * 100
1339
+
1340
+ # Detect composite/branch changes
1341
+ hierarchy = StepHierarchyParser.parse_internal_name(step.internal_name)
1342
+ composite = hierarchy.get("composite")
1343
+ branch = hierarchy.get("branch")
1344
+
1345
+ # Show composite header
1346
+ if composite and composite != current_composite:
1347
+ composite_type = self._get_composite_type(composite)
1348
+ composite_id = f"composite-{composite.replace(' ', '-')}"
1349
+
1350
+ html_parts.append(f"""
1351
+ <div class="step-row composite-header expandable" data-composite="{composite}">
1352
+ <div class="step-info step-level-0">
1353
+ <div style="display: flex; align-items: center; gap: 0.5rem;">
1354
+ <span class="expand-indicator">▼</span>
1355
+ 🔀 <strong>{composite}</strong> ({composite_type})
1356
+ </div>
1357
+ </div>
1358
+ <div class="gantt-container"></div>
1359
+ <div></div>
1360
+ </div>
1361
+ """)
1362
+
1363
+ # Start collapsible content
1364
+ html_parts.append(
1365
+ f'<div class="collapsible-content" id="{composite_id}">'
1366
+ )
1367
+ current_composite = composite
1368
+ current_branch = None
1369
+
1370
+ # Show branch header for parallel/map steps
1371
+ if branch and branch != current_branch:
1372
+ branch_display = self._format_branch_name(composite or "", branch)
1373
+
1374
+ html_parts.append(f"""
1375
+ <div class="step-row branch-header">
1376
+ <div class="step-info step-level-1">
1377
+ <div style="display: flex; align-items: center; gap: 0.5rem;">
1378
+ 🌿 <strong>Branch: {branch_display}</strong>
1379
+ </div>
1380
+ </div>
1381
+ <div class="gantt-container"></div>
1382
+ <div></div>
1383
+ </div>
1384
+ """)
1385
+ current_branch = branch
1386
+
1387
+ # Status styling
1388
+ status_class = step.status.lower()
1389
+ bar_class = (
1390
+ f"bar-{status_class}"
1391
+ if status_class in ["success", "fail"]
1392
+ else "bar-unknown"
1393
+ )
1394
+
1395
+ # Type icon and status emoji
1396
+ type_icons = {
1397
+ "task": "⚙️",
1398
+ "stub": "📝",
1399
+ "success": "✅",
1400
+ "fail": "❌",
1401
+ "parallel": "🔀",
1402
+ "map": "🔁",
1403
+ "conditional": "🔀",
1404
+ }
1405
+ type_icon = type_icons.get(step.step_type, "⚙️")
1406
+ status_emoji = (
1407
+ "✅"
1408
+ if step.status == "SUCCESS"
1409
+ else "❌"
1410
+ if step.status == "FAIL"
1411
+ else "⏸️"
1412
+ )
1413
+
1414
+ # Build parameter display - compact horizontal format
1415
+ param_info = []
1416
+
1417
+ if step.input_params:
1418
+ params_text = " • ".join(step.input_params)
1419
+ param_info.append(
1420
+ f'<div style="color: #059669; font-size: 0.7rem; margin-top: 0.2rem; font-family: monospace; word-break: break-all; line-height: 1.3;">📥 {params_text}</div>'
1421
+ )
1422
+
1423
+ if step.output_params:
1424
+ params_text = " • ".join(step.output_params)
1425
+ param_info.append(
1426
+ f'<div style="color: #dc2626; font-size: 0.7rem; margin-top: 0.2rem; font-family: monospace; word-break: break-all; line-height: 1.3;">📤 {params_text}</div>'
1427
+ )
1428
+
1429
+ if step.catalog_ops.get("put") or step.catalog_ops.get("get"):
1430
+ catalog_items = []
1431
+ if step.catalog_ops.get("put"):
1432
+ catalog_items.extend(
1433
+ [f"PUT:{item['name']}" for item in step.catalog_ops["put"]]
1434
+ )
1435
+ if step.catalog_ops.get("get"):
1436
+ catalog_items.extend(
1437
+ [f"GET:{item['name']}" for item in step.catalog_ops["get"]]
1438
+ )
1439
+ if catalog_items:
1440
+ catalog_text = " • ".join(catalog_items)
1441
+ param_info.append(
1442
+ f'<div style="color: #7c3aed; font-size: 0.7rem; margin-top: 0.2rem; font-family: monospace; word-break: break-all; line-height: 1.3;">💾 {catalog_text}</div>'
1443
+ )
1444
+
1445
+ # Generate sidebar data for this step
1446
+ sidebar_data = self._generate_step_sidebar_data(step)
1447
+
1448
+ html_parts.append(f"""
1449
+ <div class="step-row step-clickable" onclick="showStepDetails('{step.internal_name}', '{step.start_time.isoformat()}')">
1450
+ <div class="step-info step-level-{step.level}">
1451
+ <div style="display: flex; align-items: center; gap: 0.5rem;">
1452
+ {type_icon} {status_emoji} <strong>{step.name}</strong>
1453
+ <span style="font-size: 0.7rem; color: #64748b;">📊 details</span>
1454
+ </div>
1455
+ {f'<div style="color: #64748b; font-size: 0.75rem; margin-top: 0.25rem;"><strong>{step.command_type.upper()}:</strong> {step.command[:40]}{"..." if len(step.command) > 40 else ""}</div>' if step.command else ''}
1456
+ {'<div style="color: #059669; font-size: 0.65rem; margin-top: 0.25rem;">🎯 ' + (', '.join(step.parameters.key_inputs[:2]) if step.parameters and step.parameters.key_inputs else 'No key inputs') + ('...' if step.parameters and len(step.parameters.key_inputs) > 2 else '') + '</div>' if step.parameters else ''}
1457
+ </div>
1458
+ <div class="gantt-container">
1459
+ <div class="gantt-bar {bar_class}"
1460
+ style="left: {start_percent:.2f}%; width: {max(width_percent, 0.5):.2f}%;"
1461
+ title="{step.name}: {step.duration_ms:.1f}ms">
1462
+ </div>
1463
+ </div>
1464
+ <div style="font-family: monospace; font-size: 0.75rem; color: #6b7280;">
1465
+ {step.duration_ms:.1f}ms
1466
+ </div>
1467
+ </div>
1468
+
1469
+ <!-- Hidden step data for sidebar -->
1470
+ <script type="application/json" id="step-data-{step.internal_name.replace('.', '-')}-{step.start_time.isoformat()}">{sidebar_data}</script>
1471
+ """)
1472
+
1473
+ # Close any open composite sections
1474
+ if current_composite:
1475
+ html_parts.append("</div>") # Close collapsible-content
1476
+
1477
+ return "\n".join(html_parts)
1478
+
1479
+ def _generate_step_sidebar_data(self, step: StepInfo) -> str:
1480
+ """Generate JSON data for step sidebar display."""
1481
+ import json
1482
+
1483
+ if not step.parameters:
1484
+ return json.dumps({"error": "No parameter data available"})
1485
+
1486
+ # Performance classification
1487
+ perf_class = "fast"
1488
+ if step.duration_ms > 1000:
1489
+ perf_class = "slow"
1490
+ elif step.duration_ms > 100:
1491
+ perf_class = "medium"
1492
+
1493
+ data = {
1494
+ "name": step.name,
1495
+ "internal_name": step.internal_name,
1496
+ "status": step.status,
1497
+ "step_type": step.step_type,
1498
+ "command": step.command,
1499
+ "command_type": step.command_type,
1500
+ "duration_ms": step.duration_ms,
1501
+ "performance_class": perf_class,
1502
+ "start_time": step.start_time.strftime("%H:%M:%S.%f")[:-3]
1503
+ if step.start_time
1504
+ else None,
1505
+ "end_time": step.end_time.strftime("%H:%M:%S.%f")[:-3]
1506
+ if step.end_time
1507
+ else None,
1508
+ "parameters": {
1509
+ "key_inputs": step.parameters.key_inputs,
1510
+ "step_outputs": step.parameters.step_outputs,
1511
+ "context_params": step.parameters.context_params,
1512
+ "iteration_vars": step.parameters.iteration_vars,
1513
+ },
1514
+ "catalog_ops": step.catalog_ops,
1515
+ "error_message": step.error_message,
1516
+ "attempt_count": step.attempt_count,
1517
+ "max_attempts": step.max_attempts,
1518
+ }
1519
+
1520
+ return json.dumps(data)
1521
+
1522
+ def _generate_html_summary(self) -> str:
1523
+ """Generate HTML summary cards."""
1524
+ executed_steps = [step for step in self.timeline if step.start_time]
1525
+ total_duration = sum(step.duration_ms for step in executed_steps)
1526
+ success_count = sum(1 for step in executed_steps if step.status == "SUCCESS")
1527
+ success_rate = (
1528
+ (success_count / len(executed_steps)) * 100 if executed_steps else 0
1529
+ )
1530
+
1531
+ # Find slowest step
1532
+ slowest_step = (
1533
+ max(executed_steps, key=lambda x: x.duration_ms) if executed_steps else None
1534
+ )
1535
+
1536
+ return f"""
1537
+ <div class="summary-grid">
1538
+ <div class="summary-card">
1539
+ <div class="summary-number status-success">{len(executed_steps)}</div>
1540
+ <div class="summary-label">Total Steps</div>
1541
+ </div>
1542
+ <div class="summary-card">
1543
+ <div class="summary-number duration-medium">{total_duration:.1f}ms</div>
1544
+ <div class="summary-label">Total Duration</div>
1545
+ </div>
1546
+ <div class="summary-card">
1547
+ <div class="summary-number {'status-success' if success_rate == 100 else 'status-fail'}">{success_rate:.1f}%</div>
1548
+ <div class="summary-label">Success Rate</div>
1549
+ </div>
1550
+ <div class="summary-card">
1551
+ <div class="summary-number duration-slow">{'%.1fms' % slowest_step.duration_ms if slowest_step else 'N/A'}</div>
1552
+ <div class="summary-label">Slowest Step<br><small>{slowest_step.name if slowest_step else 'N/A'}</small></div>
1553
+ </div>
1554
+ </div>
1555
+ """
1556
+
1557
+
1558
+ def visualize_simple(
1559
+ run_id: str, show_summary: bool = False, output_html: Optional[str] = None
1560
+ ) -> None:
1561
+ """
1562
+ Simple visualization of a pipeline run.
1563
+
1564
+ Args:
1565
+ run_id: Run ID to visualize
1566
+ show_summary: Whether to show execution summary (deprecated, timeline has enough info)
1567
+ output_html: Optional path to save HTML timeline
1568
+ """
1569
+ # Find run log file
1570
+ run_log_dir = Path(".run_log_store")
1571
+ log_file = run_log_dir / f"{run_id}.json"
1572
+
1573
+ if not log_file.exists():
1574
+ # Try partial match
1575
+ matching_files = [f for f in run_log_dir.glob("*.json") if run_id in f.stem]
1576
+ if matching_files:
1577
+ log_file = matching_files[0]
1578
+ else:
1579
+ print(f"❌ Run log not found for: {run_id}")
1580
+ return
1581
+
1582
+ print(f"📊 Visualizing: {log_file.stem}")
1583
+
1584
+ viz = SimpleVisualizer(log_file)
1585
+ viz.print_simple_timeline()
1586
+
1587
+ if show_summary:
1588
+ viz.print_execution_summary()
1589
+
1590
+ # Generate HTML if requested
1591
+ if output_html:
1592
+ viz.generate_html_timeline(output_html)
1593
+
1594
+
1595
+ def generate_html_timeline(
1596
+ run_id: str, output_file: str, open_browser: bool = True
1597
+ ) -> None:
1598
+ """
1599
+ Generate HTML timeline for a specific run ID.
1600
+
1601
+ Args:
1602
+ run_id: The run ID to visualize
1603
+ output_file: Output HTML file path
1604
+ open_browser: Whether to open the result in browser
1605
+ """
1606
+ from pathlib import Path
1607
+
1608
+ # Find run log file
1609
+ run_log_dir = Path(".run_log_store")
1610
+ log_file = run_log_dir / f"{run_id}.json"
1611
+
1612
+ if not log_file.exists():
1613
+ # Try partial match
1614
+ matching_files = [f for f in run_log_dir.glob("*.json") if run_id in f.stem]
1615
+ if matching_files:
1616
+ log_file = matching_files[0]
1617
+ else:
1618
+ print(f"❌ Run log not found for: {run_id}")
1619
+ return
1620
+
1621
+ print(f"🌐 Generating HTML timeline for: {log_file.stem}")
1622
+
1623
+ # Create visualizer and generate HTML
1624
+ viz = SimpleVisualizer(log_file)
1625
+ viz.generate_html_timeline(output_file)
1626
+
1627
+ if open_browser:
1628
+ import webbrowser
1629
+
1630
+ file_path = Path(output_file).absolute()
1631
+ print(f"🌐 Opening timeline in browser: {file_path.name}")
1632
+ webbrowser.open(file_path.as_uri())
1633
+
1634
+
1635
+ if __name__ == "__main__":
1636
+ import sys
1637
+
1638
+ if len(sys.argv) > 1:
1639
+ if len(sys.argv) > 2 and sys.argv[2].endswith(".html"):
1640
+ # Generate HTML: python viz_simple.py <run_id> <output.html>
1641
+ generate_html_timeline(sys.argv[1], sys.argv[2])
1642
+ else:
1643
+ # Console visualization: python viz_simple.py <run_id>
1644
+ visualize_simple(sys.argv[1])
1645
+ else:
1646
+ print("Usage: python viz_simple.py <run_id> [output.html]")