runnable 0.34.0a1__py3-none-any.whl → 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of runnable might be problematic. Click here for more details.

Files changed (49) hide show
  1. extensions/catalog/any_path.py +13 -2
  2. extensions/job_executor/__init__.py +7 -5
  3. extensions/job_executor/emulate.py +106 -0
  4. extensions/job_executor/k8s.py +8 -8
  5. extensions/job_executor/local_container.py +13 -14
  6. extensions/nodes/__init__.py +0 -0
  7. extensions/nodes/conditional.py +243 -0
  8. extensions/nodes/fail.py +72 -0
  9. extensions/nodes/map.py +350 -0
  10. extensions/nodes/parallel.py +159 -0
  11. extensions/nodes/stub.py +89 -0
  12. extensions/nodes/success.py +72 -0
  13. extensions/nodes/task.py +92 -0
  14. extensions/pipeline_executor/__init__.py +27 -27
  15. extensions/pipeline_executor/argo.py +52 -46
  16. extensions/pipeline_executor/emulate.py +112 -0
  17. extensions/pipeline_executor/local.py +4 -4
  18. extensions/pipeline_executor/local_container.py +19 -79
  19. extensions/pipeline_executor/mocked.py +5 -9
  20. extensions/pipeline_executor/retry.py +6 -10
  21. runnable/__init__.py +2 -11
  22. runnable/catalog.py +6 -23
  23. runnable/cli.py +145 -48
  24. runnable/context.py +520 -28
  25. runnable/datastore.py +51 -54
  26. runnable/defaults.py +12 -34
  27. runnable/entrypoints.py +82 -440
  28. runnable/exceptions.py +35 -34
  29. runnable/executor.py +13 -20
  30. runnable/gantt.py +1141 -0
  31. runnable/graph.py +1 -1
  32. runnable/names.py +1 -1
  33. runnable/nodes.py +20 -16
  34. runnable/parameters.py +108 -51
  35. runnable/sdk.py +125 -204
  36. runnable/tasks.py +62 -85
  37. runnable/utils.py +6 -268
  38. runnable-1.0.0.dist-info/METADATA +122 -0
  39. runnable-1.0.0.dist-info/RECORD +73 -0
  40. {runnable-0.34.0a1.dist-info → runnable-1.0.0.dist-info}/entry_points.txt +9 -8
  41. extensions/nodes/nodes.py +0 -778
  42. extensions/nodes/torch.py +0 -273
  43. extensions/nodes/torch_config.py +0 -76
  44. extensions/tasks/torch.py +0 -286
  45. extensions/tasks/torch_config.py +0 -76
  46. runnable-0.34.0a1.dist-info/METADATA +0 -267
  47. runnable-0.34.0a1.dist-info/RECORD +0 -67
  48. {runnable-0.34.0a1.dist-info → runnable-1.0.0.dist-info}/WHEEL +0 -0
  49. {runnable-0.34.0a1.dist-info → runnable-1.0.0.dist-info}/licenses/LICENSE +0 -0
runnable/gantt.py ADDED
@@ -0,0 +1,1141 @@
1
+ """
2
+ Simplified visualization for Runnable pipeline execution.
3
+
4
+ This module provides lightweight, reusable components that understand
5
+ the composite pipeline structure documented in the run logs.
6
+ """
7
+
8
+ import json
9
+ from dataclasses import dataclass
10
+ from datetime import datetime
11
+ from pathlib import Path
12
+ from typing import Any, Dict, List, Optional, Tuple, Union
13
+
14
+
15
+ @dataclass
16
+ class StepInfo:
17
+ """Clean representation of a pipeline step."""
18
+
19
+ name: str
20
+ internal_name: str
21
+ status: str
22
+ step_type: str
23
+ start_time: Optional[datetime]
24
+ end_time: Optional[datetime]
25
+ duration_ms: float
26
+ level: int # 0=top-level, 1=branch, 2=nested
27
+ parent: Optional[str]
28
+ branch: Optional[str]
29
+ command: str
30
+ command_type: str
31
+ input_params: List[str]
32
+ output_params: List[str]
33
+ catalog_ops: Dict[str, List[str]]
34
+
35
+
36
+ class StepHierarchyParser:
37
+ """Parse internal names to understand pipeline hierarchy."""
38
+
39
+ @staticmethod
40
+ def parse_internal_name(internal_name: str) -> Dict[str, str]:
41
+ """
42
+ Parse internal name into components.
43
+
44
+ Examples:
45
+ - "hello" -> {"step": "hello"}
46
+ - "parallel_step.branch1.hello_stub" -> {
47
+ "composite": "parallel_step",
48
+ "branch": "branch1",
49
+ "step": "hello_stub"
50
+ }
51
+ """
52
+ parts = internal_name.split(".")
53
+
54
+ if len(parts) == 1:
55
+ return {"step": parts[0]}
56
+ elif len(parts) == 2:
57
+ return {"composite": parts[0], "branch": parts[1]}
58
+ elif len(parts) == 3:
59
+ return {"composite": parts[0], "branch": parts[1], "step": parts[2]}
60
+ else:
61
+ # Handle deeper nesting if needed
62
+ return {
63
+ "composite": parts[0],
64
+ "branch": ".".join(parts[1:-1]),
65
+ "step": parts[-1],
66
+ }
67
+
68
+ @staticmethod
69
+ def get_step_level(internal_name: str) -> int:
70
+ """Determine hierarchy level from internal name."""
71
+ parts = internal_name.split(".")
72
+ if len(parts) == 1:
73
+ return 0 # Top-level step
74
+ elif len(parts) == 2:
75
+ return 1 # Branch level (for composite step parent)
76
+ else:
77
+ return 2 # Branch step
78
+
79
+
80
+ class TimelineExtractor:
81
+ """Extract chronological timeline from run log."""
82
+
83
+ def __init__(self, run_log_data: Dict[str, Any]):
84
+ self.run_log_data = run_log_data
85
+ self.dag_nodes = (
86
+ run_log_data.get("run_config", {}).get("dag", {}).get("nodes", {})
87
+ )
88
+
89
+ def parse_time(self, time_str: str) -> Optional[datetime]:
90
+ """Parse ISO timestamp string."""
91
+ try:
92
+ return datetime.fromisoformat(time_str) if time_str else None
93
+ except (ValueError, TypeError):
94
+ return None
95
+
96
+ def get_step_timing(
97
+ self, step_data: Dict[str, Any]
98
+ ) -> Tuple[Optional[datetime], Optional[datetime], float]:
99
+ """Extract timing from step attempts."""
100
+ attempts = step_data.get("attempts", [])
101
+ if not attempts:
102
+ return None, None, 0
103
+
104
+ attempt = attempts[0]
105
+ start = self.parse_time(attempt.get("start_time"))
106
+ end = self.parse_time(attempt.get("end_time"))
107
+
108
+ if start and end:
109
+ duration_ms = (end - start).total_seconds() * 1000
110
+ return start, end, duration_ms
111
+
112
+ return None, None, 0
113
+
114
+ def find_dag_node(self, internal_name: str, clean_name: str) -> Dict[str, Any]:
115
+ """Find DAG node info for command details."""
116
+ # Try direct lookup first
117
+ if clean_name in self.dag_nodes:
118
+ return self.dag_nodes[clean_name]
119
+
120
+ # For composite steps, look in branch structures
121
+ hierarchy = StepHierarchyParser.parse_internal_name(internal_name)
122
+ if "composite" in hierarchy:
123
+ composite_node = self.dag_nodes.get(hierarchy["composite"], {})
124
+ if composite_node.get("is_composite"):
125
+ branches = composite_node.get("branches", {})
126
+ branch_key = hierarchy.get("branch", "")
127
+
128
+ if branch_key in branches:
129
+ branch_nodes = branches[branch_key].get("nodes", {})
130
+ if clean_name in branch_nodes:
131
+ return branch_nodes[clean_name]
132
+ # For map nodes, the branch structure might be different
133
+ elif "branch" in composite_node: # Map node structure
134
+ branch_nodes = composite_node["branch"].get("nodes", {})
135
+ if clean_name in branch_nodes:
136
+ return branch_nodes[clean_name]
137
+
138
+ return {}
139
+
140
+ def _format_parameter_value(self, value: Any, kind: str) -> str:
141
+ """Format parameter value for display."""
142
+ if kind == "metric":
143
+ if isinstance(value, (int, float)):
144
+ return f"{value:.3g}"
145
+ return str(value)
146
+
147
+ if isinstance(value, str):
148
+ # Truncate long strings
149
+ if len(value) > 50:
150
+ return f'"{value[:47]}..."'
151
+ return f'"{value}"'
152
+ elif isinstance(value, (list, tuple)):
153
+ if len(value) > 3:
154
+ preview = ", ".join(str(v) for v in value[:3])
155
+ return f"[{preview}, ...+{len(value)-3}]"
156
+ return str(value)
157
+ elif isinstance(value, dict):
158
+ if len(value) > 2:
159
+ keys = list(value.keys())[:2]
160
+ preview = ", ".join(f'"{k}": {value[k]}' for k in keys)
161
+ return f"{{{preview}, ...+{len(value)-2}}}"
162
+ return str(value)
163
+ else:
164
+ return str(value)
165
+
166
+ def extract_timeline(self) -> List[StepInfo]:
167
+ """Extract all steps in chronological order."""
168
+ steps = []
169
+
170
+ # Process top-level steps
171
+ for step_name, step_data in self.run_log_data.get("steps", {}).items():
172
+ step_info = self._create_step_info(step_name, step_data)
173
+ steps.append(step_info)
174
+
175
+ # Process branches if they exist
176
+ branches = step_data.get("branches", {})
177
+ for branch_name, branch_data in branches.items():
178
+ # Add branch steps
179
+ for sub_step_name, sub_step_data in branch_data.get(
180
+ "steps", {}
181
+ ).items():
182
+ sub_step_info = self._create_step_info(
183
+ sub_step_name,
184
+ sub_step_data,
185
+ parent=step_name,
186
+ branch=branch_name,
187
+ )
188
+ steps.append(sub_step_info)
189
+
190
+ # Sort by start time for chronological order
191
+ return sorted(steps, key=lambda x: x.start_time or datetime.min)
192
+
193
+ def _create_step_info(
194
+ self,
195
+ step_name: str,
196
+ step_data: Dict[str, Any],
197
+ parent: Optional[str] = None,
198
+ branch: Optional[str] = None,
199
+ ) -> StepInfo:
200
+ """Create StepInfo from raw step data."""
201
+ internal_name = step_data.get("internal_name", step_name)
202
+ clean_name = step_data.get("name", step_name)
203
+
204
+ # Get timing
205
+ start, end, duration = self.get_step_timing(step_data)
206
+
207
+ # Get command info from DAG
208
+ dag_node = self.find_dag_node(internal_name, clean_name)
209
+ command = dag_node.get("command", "")
210
+ command_type = dag_node.get("command_type", "")
211
+
212
+ # Extract parameters with detailed metadata (exclude pickled/object types)
213
+ input_params = []
214
+ output_params = []
215
+ catalog_ops: Dict[str, List[str]] = {"put": [], "get": []}
216
+
217
+ attempts = step_data.get("attempts", [])
218
+ if attempts:
219
+ attempt = attempts[0]
220
+ input_param_data = attempt.get("input_parameters", {})
221
+ output_param_data = attempt.get("output_parameters", {})
222
+
223
+ # Process input parameters (exclude object/pickled types)
224
+ for name, param in input_param_data.items():
225
+ if isinstance(param, dict):
226
+ kind = param.get("kind", "")
227
+ if kind in ("json", "metric"):
228
+ value = param.get("value", "")
229
+ # Format value for display
230
+ formatted_value = self._format_parameter_value(value, kind)
231
+ input_params.append(f"{name}={formatted_value}")
232
+ # Skip object/pickled parameters entirely
233
+
234
+ # Process output parameters (exclude object/pickled types)
235
+ for name, param in output_param_data.items():
236
+ if isinstance(param, dict):
237
+ kind = param.get("kind", "")
238
+ if kind in ("json", "metric"):
239
+ value = param.get("value", "")
240
+ # Format value for display
241
+ formatted_value = self._format_parameter_value(value, kind)
242
+ output_params.append(f"{name}={formatted_value}")
243
+ # Skip object/pickled parameters entirely
244
+
245
+ # Extract catalog operations
246
+ catalog_data = step_data.get("data_catalog", [])
247
+ for item in catalog_data:
248
+ stage = item.get("stage", "")
249
+ name = item.get("name", "")
250
+ if stage == "put":
251
+ catalog_ops["put"].append(name)
252
+ elif stage == "get":
253
+ catalog_ops["get"].append(name)
254
+
255
+ return StepInfo(
256
+ name=clean_name,
257
+ internal_name=internal_name,
258
+ status=step_data.get("status", "UNKNOWN"),
259
+ step_type=step_data.get("step_type", "task"),
260
+ start_time=start,
261
+ end_time=end,
262
+ duration_ms=duration,
263
+ level=StepHierarchyParser.get_step_level(internal_name),
264
+ parent=parent,
265
+ branch=branch,
266
+ command=command,
267
+ command_type=command_type,
268
+ input_params=input_params,
269
+ output_params=output_params,
270
+ catalog_ops=catalog_ops,
271
+ )
272
+
273
+
274
+ class SimpleVisualizer:
275
+ """Simple, lightweight pipeline visualizer."""
276
+
277
+ def __init__(self, run_log_path: Union[str, Path]):
278
+ self.run_log_path = Path(run_log_path)
279
+ self.run_log_data = self._load_run_log()
280
+ self.extractor = TimelineExtractor(self.run_log_data)
281
+ self.timeline = self.extractor.extract_timeline()
282
+
283
+ def _load_run_log(self) -> Dict[str, Any]:
284
+ """Load run log JSON."""
285
+ if not self.run_log_path.exists():
286
+ raise FileNotFoundError(f"Run log not found: {self.run_log_path}")
287
+
288
+ with open(self.run_log_path, "r") as f:
289
+ return json.load(f)
290
+
291
+ def print_simple_timeline(self) -> None:
292
+ """Print a clean console timeline."""
293
+ run_id = self.run_log_data.get("run_id", "unknown")
294
+ status = self.run_log_data.get("status", "UNKNOWN")
295
+
296
+ print(f"\n🔄 Pipeline Timeline - {run_id}")
297
+ print(f"Status: {status}")
298
+ print("=" * 80)
299
+
300
+ # Group by composite steps for better display
301
+ current_composite = None
302
+ current_branch = None
303
+
304
+ for step in self.timeline:
305
+ # Skip composite steps themselves (they have no timing)
306
+ if (
307
+ step.step_type in ["parallel", "map", "conditional"]
308
+ and not step.start_time
309
+ ):
310
+ continue
311
+
312
+ # Detect composite/branch changes
313
+ hierarchy = StepHierarchyParser.parse_internal_name(step.internal_name)
314
+ composite = hierarchy.get("composite")
315
+ branch = hierarchy.get("branch")
316
+
317
+ # Show composite header
318
+ if composite and composite != current_composite:
319
+ print(f"\n🔀 {composite} ({self._get_composite_type(composite)})")
320
+ current_composite = composite
321
+ current_branch = None
322
+
323
+ # Show branch header
324
+ if branch and branch != current_branch:
325
+ branch_display = self._format_branch_name(composite or "", branch)
326
+ print(f" ├─ Branch: {branch_display}")
327
+ current_branch = branch
328
+
329
+ # Show step
330
+ indent = (
331
+ " " if step.level == 0 else " " if step.level == 1 else " "
332
+ )
333
+ status_emoji = (
334
+ "✅"
335
+ if step.status == "SUCCESS"
336
+ else "❌"
337
+ if step.status == "FAIL"
338
+ else "⏸️"
339
+ )
340
+
341
+ # Type icon
342
+ type_icons = {
343
+ "task": "⚙️",
344
+ "stub": "📝",
345
+ "success": "✅",
346
+ "fail": "❌",
347
+ "parallel": "🔀",
348
+ "map": "🔁",
349
+ "conditional": "🔀",
350
+ }
351
+ type_icon = type_icons.get(step.step_type, "⚙️")
352
+
353
+ timing = f"({step.duration_ms:.1f}ms)" if step.duration_ms > 0 else ""
354
+
355
+ print(f"{indent}{type_icon} {status_emoji} {step.name} {timing}")
356
+
357
+ # Show metadata for tasks
358
+ if step.step_type == "task" and (
359
+ step.command
360
+ or step.input_params
361
+ or step.output_params
362
+ or step.catalog_ops["put"]
363
+ or step.catalog_ops["get"]
364
+ ):
365
+ if step.command:
366
+ cmd_short = (
367
+ step.command[:50] + "..."
368
+ if len(step.command) > 50
369
+ else step.command
370
+ )
371
+ print(f"{indent} 📝 {step.command_type.upper()}: {cmd_short}")
372
+
373
+ # Show input parameters - compact horizontal display
374
+ if step.input_params:
375
+ params_display = " • ".join(step.input_params)
376
+ print(f"{indent} 📥 {params_display}")
377
+
378
+ # Show output parameters - compact horizontal display
379
+ if step.output_params:
380
+ params_display = " • ".join(step.output_params)
381
+ print(f"{indent} 📤 {params_display}")
382
+
383
+ # Show catalog operations - compact horizontal display
384
+ if step.catalog_ops.get("put") or step.catalog_ops.get("get"):
385
+ catalog_items = []
386
+ if step.catalog_ops.get("put"):
387
+ catalog_items.extend(
388
+ [f"PUT:{item}" for item in step.catalog_ops["put"]]
389
+ )
390
+ if step.catalog_ops.get("get"):
391
+ catalog_items.extend(
392
+ [f"GET:{item}" for item in step.catalog_ops["get"]]
393
+ )
394
+ if catalog_items:
395
+ catalog_display = " • ".join(catalog_items)
396
+ print(f"{indent} 💾 {catalog_display}")
397
+
398
+ print("=" * 80)
399
+
400
+ def _get_composite_type(self, composite_name: str) -> str:
401
+ """Get composite node type from DAG."""
402
+ dag_nodes = (
403
+ self.run_log_data.get("run_config", {}).get("dag", {}).get("nodes", {})
404
+ )
405
+ node = dag_nodes.get(composite_name, {})
406
+ return node.get("node_type", "composite")
407
+
408
+ def _format_branch_name(self, composite: str, branch: str) -> str:
409
+ """Format branch name based on composite type."""
410
+ # Remove composite prefix if present
411
+ if branch.startswith(f"{composite}."):
412
+ branch_clean = branch[len(f"{composite}.") :]
413
+ else:
414
+ branch_clean = branch
415
+
416
+ # Check if it's a map iteration (numeric)
417
+ if branch_clean.isdigit():
418
+ return f"Iteration {branch_clean}"
419
+
420
+ return branch_clean
421
+
422
+ def print_execution_summary(self) -> None:
423
+ """Print execution summary table."""
424
+ run_id = self.run_log_data.get("run_id", "unknown")
425
+
426
+ print(f"\n📊 Execution Summary - {run_id}")
427
+ print("=" * 80)
428
+
429
+ # Filter to actual executed steps (with timing)
430
+ executed_steps = [step for step in self.timeline if step.start_time]
431
+
432
+ if not executed_steps:
433
+ print("No executed steps found")
434
+ return
435
+
436
+ # Table header
437
+ print(f"{'Step':<30} {'Status':<10} {'Duration':<12} {'Type':<10}")
438
+ print("-" * 80)
439
+
440
+ total_duration = 0
441
+ success_count = 0
442
+
443
+ for step in executed_steps:
444
+ status_emoji = (
445
+ "✅"
446
+ if step.status == "SUCCESS"
447
+ else "❌"
448
+ if step.status == "FAIL"
449
+ else "⏸️"
450
+ )
451
+ duration_text = (
452
+ f"{step.duration_ms:.1f}ms" if step.duration_ms > 0 else "0.0ms"
453
+ )
454
+
455
+ # Truncate long names
456
+ display_name = step.name[:28] + ".." if len(step.name) > 30 else step.name
457
+
458
+ print(
459
+ f"{display_name:<30} {status_emoji}{step.status:<9} {duration_text:<12} {step.step_type:<10}"
460
+ )
461
+
462
+ total_duration += int(step.duration_ms)
463
+ if step.status == "SUCCESS":
464
+ success_count += 1
465
+
466
+ print("-" * 80)
467
+ success_rate = (
468
+ (success_count / len(executed_steps)) * 100 if executed_steps else 0
469
+ )
470
+ overall_status = self.run_log_data.get("status", "UNKNOWN")
471
+ overall_emoji = "✅" if overall_status == "SUCCESS" else "❌"
472
+
473
+ print(
474
+ f"Total Duration: {total_duration:.1f}ms | Success Rate: {success_rate:.1f}% | Status: {overall_emoji} {overall_status}"
475
+ )
476
+
477
+ def generate_html_timeline(
478
+ self, output_path: Optional[Union[str, Path]] = None
479
+ ) -> str:
480
+ """
481
+ Generate an interactive HTML timeline visualization.
482
+
483
+ This creates a lightweight HTML version with:
484
+ - Clean timeline layout
485
+ - Hover tooltips with metadata
486
+ - Expandable composite sections
487
+ - Timing bars proportional to execution duration
488
+ """
489
+ run_id = self.run_log_data.get("run_id", "unknown")
490
+ status = self.run_log_data.get("status", "UNKNOWN")
491
+
492
+ # Calculate total timeline for proportional bars
493
+ executed_steps = [step for step in self.timeline if step.start_time]
494
+ if executed_steps:
495
+ earliest = min(
496
+ step.start_time for step in executed_steps if step.start_time
497
+ )
498
+ latest = max(step.end_time for step in executed_steps if step.end_time)
499
+ total_duration_ms = (
500
+ (latest - earliest).total_seconds() * 1000 if latest and earliest else 1
501
+ )
502
+ else:
503
+ total_duration_ms = 1
504
+
505
+ html_content = f"""<!DOCTYPE html>
506
+ <html lang="en">
507
+ <head>
508
+ <meta charset="UTF-8">
509
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
510
+ <title>Pipeline Timeline - {run_id}</title>
511
+ <style>
512
+ * {{ margin: 0; padding: 0; box-sizing: border-box; }}
513
+
514
+ body {{
515
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, sans-serif;
516
+ background: #f8fafc;
517
+ color: #1e293b;
518
+ line-height: 1.6;
519
+ }}
520
+
521
+ .header {{
522
+ background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
523
+ color: white;
524
+ padding: 2rem 0;
525
+ text-align: center;
526
+ box-shadow: 0 4px 6px rgba(0,0,0,0.1);
527
+ }}
528
+
529
+ .container {{
530
+ max-width: 1400px;
531
+ margin: 2rem auto;
532
+ padding: 0 1rem;
533
+ }}
534
+
535
+ .timeline-card {{
536
+ background: white;
537
+ border-radius: 12px;
538
+ box-shadow: 0 2px 10px rgba(0,0,0,0.08);
539
+ overflow: hidden;
540
+ margin-bottom: 2rem;
541
+ }}
542
+
543
+ .timeline-header {{
544
+ background: #f8fafc;
545
+ padding: 1.5rem;
546
+ border-bottom: 1px solid #e2e8f0;
547
+ display: flex;
548
+ justify-content: space-between;
549
+ align-items: center;
550
+ }}
551
+
552
+ .timeline-content {{
553
+ padding: 1rem;
554
+ }}
555
+
556
+ .step-row {{
557
+ display: grid;
558
+ grid-template-columns: 300px 1fr 80px;
559
+ align-items: center;
560
+ padding: 0.5rem 0;
561
+ border-bottom: 1px solid #f1f5f9;
562
+ transition: background 0.2s ease;
563
+ gap: 1rem;
564
+ min-height: 40px;
565
+ overflow: visible;
566
+ }}
567
+
568
+ .step-row:hover {{
569
+ background: #f8fafc;
570
+ }}
571
+
572
+ .step-info {{
573
+ display: flex;
574
+ flex-direction: column;
575
+ gap: 0.25rem;
576
+ font-weight: 500;
577
+ min-height: 24px;
578
+ justify-content: flex-start;
579
+ overflow: visible;
580
+ word-wrap: break-word;
581
+ overflow-wrap: break-word;
582
+ }}
583
+
584
+ .step-level-0 {{ padding-left: 0; }}
585
+ .step-level-1 {{ padding-left: 1rem; }}
586
+ .step-level-2 {{ padding-left: 2rem; }}
587
+
588
+ .composite-header {{
589
+ background: #e0f2fe !important;
590
+ border-left: 4px solid #0277bd;
591
+ font-weight: 600;
592
+ color: #01579b;
593
+ }}
594
+
595
+ .branch-header {{
596
+ background: #f3e5f5 !important;
597
+ border-left: 4px solid #7b1fa2;
598
+ font-weight: 600;
599
+ color: #4a148c;
600
+ }}
601
+
602
+ .gantt-container {{
603
+ position: relative;
604
+ height: 30px;
605
+ background: #f8fafc;
606
+ border: 1px solid #e2e8f0;
607
+ border-radius: 4px;
608
+ min-width: 100%;
609
+ overflow: hidden;
610
+ }}
611
+
612
+ .gantt-bar {{
613
+ position: absolute;
614
+ top: 3px;
615
+ height: 24px;
616
+ border-radius: 3px;
617
+ transition: all 0.2s ease;
618
+ cursor: pointer;
619
+ border: 1px solid rgba(255,255,255,0.3);
620
+ }}
621
+
622
+ .gantt-bar:hover {{
623
+ transform: scaleY(1.1);
624
+ z-index: 10;
625
+ box-shadow: 0 2px 8px rgba(0,0,0,0.2);
626
+ }}
627
+
628
+ .time-grid {{
629
+ position: absolute;
630
+ top: 0;
631
+ bottom: 0;
632
+ border-left: 1px solid #e2e8f0;
633
+ opacity: 0.3;
634
+ }}
635
+
636
+ .time-scale {{
637
+ position: relative;
638
+ height: 20px;
639
+ background: #f1f5f9;
640
+ border-bottom: 1px solid #d1d5db;
641
+ font-size: 0.75rem;
642
+ color: #6b7280;
643
+ }}
644
+
645
+ .time-marker {{
646
+ position: absolute;
647
+ top: 0;
648
+ height: 100%;
649
+ display: flex;
650
+ align-items: center;
651
+ padding-left: 4px;
652
+ font-weight: 500;
653
+ }}
654
+
655
+ .timeline-bar:hover {{
656
+ transform: scaleY(1.1);
657
+ z-index: 10;
658
+ }}
659
+
660
+ .bar-success {{ background: linear-gradient(90deg, #22c55e, #16a34a); }}
661
+ .bar-fail {{ background: linear-gradient(90deg, #ef4444, #dc2626); }}
662
+ .bar-unknown {{ background: linear-gradient(90deg, #f59e0b, #d97706); }}
663
+
664
+ .duration-text {{
665
+ font-family: monospace;
666
+ font-size: 0.875rem;
667
+ font-weight: 600;
668
+ }}
669
+
670
+ .duration-fast {{ color: #16a34a; }}
671
+ .duration-medium {{ color: #f59e0b; }}
672
+ .duration-slow {{ color: #dc2626; }}
673
+
674
+ .status-success {{ color: #16a34a; }}
675
+ .status-fail {{ color: #dc2626; }}
676
+ .status-unknown {{ color: #f59e0b; }}
677
+
678
+ .expandable {{
679
+ cursor: pointer;
680
+ user-select: none;
681
+ }}
682
+
683
+ .expandable:hover {{
684
+ background: #e2e8f0 !important;
685
+ }}
686
+
687
+ .step-header.expandable {{
688
+ padding: 0.25rem;
689
+ border-radius: 4px;
690
+ margin: -0.25rem;
691
+ }}
692
+
693
+ .step-header.expandable:hover {{
694
+ background: #f1f5f9 !important;
695
+ }}
696
+
697
+ .collapsible-content {{
698
+ max-height: none;
699
+ overflow: visible;
700
+ transition: max-height 0.3s ease;
701
+ }}
702
+
703
+ .collapsible-content.collapsed {{
704
+ max-height: 0;
705
+ overflow: hidden;
706
+ }}
707
+
708
+ .summary-grid {{
709
+ display: grid;
710
+ grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
711
+ gap: 1.5rem;
712
+ margin-top: 2rem;
713
+ }}
714
+
715
+ .summary-card {{
716
+ background: white;
717
+ padding: 1.5rem;
718
+ border-radius: 8px;
719
+ box-shadow: 0 2px 10px rgba(0,0,0,0.08);
720
+ text-align: center;
721
+ }}
722
+
723
+ .summary-number {{
724
+ font-size: 2rem;
725
+ font-weight: bold;
726
+ margin-bottom: 0.5rem;
727
+ }}
728
+
729
+ .summary-label {{
730
+ color: #64748b;
731
+ font-size: 0.875rem;
732
+ }}
733
+ </style>
734
+ </head>
735
+ <body>
736
+ <div class="header">
737
+ <h1>🔄 Pipeline Timeline Visualization</h1>
738
+ <p>Interactive execution analysis for {run_id}</p>
739
+ </div>
740
+
741
+ <div class="container">
742
+ <div class="timeline-card">
743
+ <div class="timeline-header">
744
+ <div>
745
+ <h2>Run ID: {run_id}</h2>
746
+ <p>Status: <span class="status-{status.lower()}">{status}</span></p>
747
+ </div>
748
+ <div>
749
+ <span class="duration-text">Total: {total_duration_ms:.1f}ms</span>
750
+ </div>
751
+ </div>
752
+
753
+ <div class="timeline-content">
754
+ {self._generate_html_timeline_rows(total_duration_ms)}
755
+ </div>
756
+ </div>
757
+
758
+ {self._generate_html_summary()}
759
+ </div>
760
+
761
+ <script>
762
+ // Collapsible sections
763
+ document.querySelectorAll('.expandable').forEach(element => {{
764
+ element.addEventListener('click', () => {{
765
+ let content;
766
+
767
+ // Handle step metadata (using data-target)
768
+ if (element.dataset.target) {{
769
+ content = document.getElementById(element.dataset.target);
770
+ }} else {{
771
+ // Handle composite sections (using nextElementSibling)
772
+ content = element.nextElementSibling;
773
+ }}
774
+
775
+ if (content && content.classList.contains('collapsible-content')) {{
776
+ content.classList.toggle('collapsed');
777
+
778
+ // Update expand/collapse indicator
779
+ const indicator = element.querySelector('.expand-indicator');
780
+ if (indicator) {{
781
+ indicator.textContent = content.classList.contains('collapsed') ? '▶' : '▼';
782
+ }}
783
+ }}
784
+ }});
785
+ }});
786
+ </script>
787
+ </body>
788
+ </html>"""
789
+
790
+ # Save to file if path provided
791
+ if output_path:
792
+ Path(output_path).write_text(html_content)
793
+ print(f"HTML timeline saved to: {output_path}")
794
+
795
+ return html_content
796
+
797
+ def _generate_html_timeline_rows(self, total_duration_ms: float) -> str:
798
+ """Generate HTML rows for the Gantt chart timeline display."""
799
+ executed_steps = [
800
+ step for step in self.timeline if step.start_time and step.end_time
801
+ ]
802
+
803
+ if not executed_steps:
804
+ return "<div>No executed steps found</div>"
805
+
806
+ # Calculate the absolute timeline
807
+ earliest_start = min(
808
+ step.start_time for step in executed_steps if step.start_time
809
+ )
810
+ latest_end = max(step.end_time for step in executed_steps if step.end_time)
811
+ total_timeline_ms = (
812
+ (latest_end - earliest_start).total_seconds() * 1000
813
+ if latest_end and earliest_start
814
+ else 1
815
+ )
816
+
817
+ # Generate time scale and Gantt rows
818
+ time_scale_html = self._generate_time_scale(total_timeline_ms)
819
+ gantt_rows_html = self._generate_gantt_rows(
820
+ executed_steps, earliest_start, total_timeline_ms
821
+ )
822
+
823
+ return time_scale_html + "\n" + gantt_rows_html
824
+
825
+ def _generate_time_scale(self, total_timeline_ms: float) -> str:
826
+ """Generate the time scale header for the Gantt chart."""
827
+ # Create time markers at regular intervals
828
+ num_markers = 10
829
+ interval_ms = total_timeline_ms / num_markers
830
+
831
+ markers_html = []
832
+ for i in range(num_markers + 1):
833
+ time_ms = i * interval_ms
834
+ position_percent = (time_ms / total_timeline_ms) * 100
835
+ time_display = (
836
+ f"{time_ms:.0f}ms" if time_ms < 1000 else f"{time_ms/1000:.1f}s"
837
+ )
838
+
839
+ markers_html.append(f"""
840
+ <div class="time-marker" style="left: {position_percent:.1f}%;">
841
+ {time_display}
842
+ </div>
843
+ """)
844
+
845
+ # Add grid line (except for the first one)
846
+ if i > 0:
847
+ markers_html.append(
848
+ f'<div class="time-grid" style="left: {position_percent:.1f}%;"></div>'
849
+ )
850
+
851
+ return f"""
852
+ <div class="step-row" style="border-bottom: 2px solid #d1d5db;">
853
+ <div class="step-info">
854
+ <strong>Timeline</strong>
855
+ </div>
856
+ <div class="time-scale">
857
+ {"".join(markers_html)}
858
+ </div>
859
+ <div></div>
860
+ </div>
861
+ """
862
+
863
+ def _generate_gantt_rows(
864
+ self, executed_steps: List, earliest_start, total_timeline_ms: float
865
+ ) -> str:
866
+ """Generate HTML rows for the Gantt chart display."""
867
+ html_parts = []
868
+
869
+ # Group by composite steps for better display
870
+ current_composite = None
871
+ current_branch = None
872
+
873
+ for step in executed_steps:
874
+ # Calculate timing positions for Gantt chart
875
+ start_offset_ms = (step.start_time - earliest_start).total_seconds() * 1000
876
+ start_percent = (start_offset_ms / total_timeline_ms) * 100
877
+ width_percent = (step.duration_ms / total_timeline_ms) * 100
878
+
879
+ # Detect composite/branch changes
880
+ hierarchy = StepHierarchyParser.parse_internal_name(step.internal_name)
881
+ composite = hierarchy.get("composite")
882
+ branch = hierarchy.get("branch")
883
+
884
+ # Show composite header
885
+ if composite and composite != current_composite:
886
+ composite_type = self._get_composite_type(composite)
887
+ composite_id = f"composite-{composite.replace(' ', '-')}"
888
+
889
+ html_parts.append(f"""
890
+ <div class="step-row composite-header expandable" data-composite="{composite}">
891
+ <div class="step-info step-level-0">
892
+ <div style="display: flex; align-items: center; gap: 0.5rem;">
893
+ <span class="expand-indicator">▼</span>
894
+ 🔀 <strong>{composite}</strong> ({composite_type})
895
+ </div>
896
+ </div>
897
+ <div class="gantt-container"></div>
898
+ <div></div>
899
+ </div>
900
+ """)
901
+
902
+ # Start collapsible content
903
+ html_parts.append(
904
+ f'<div class="collapsible-content" id="{composite_id}">'
905
+ )
906
+ current_composite = composite
907
+ current_branch = None
908
+
909
+ # Show branch header for parallel/map steps
910
+ if branch and branch != current_branch:
911
+ branch_display = self._format_branch_name(composite or "", branch)
912
+
913
+ html_parts.append(f"""
914
+ <div class="step-row branch-header">
915
+ <div class="step-info step-level-1">
916
+ <div style="display: flex; align-items: center; gap: 0.5rem;">
917
+ 🌿 <strong>Branch: {branch_display}</strong>
918
+ </div>
919
+ </div>
920
+ <div class="gantt-container"></div>
921
+ <div></div>
922
+ </div>
923
+ """)
924
+ current_branch = branch
925
+
926
+ # Status styling
927
+ status_class = step.status.lower()
928
+ bar_class = (
929
+ f"bar-{status_class}"
930
+ if status_class in ["success", "fail"]
931
+ else "bar-unknown"
932
+ )
933
+
934
+ # Type icon and status emoji
935
+ type_icons = {
936
+ "task": "⚙️",
937
+ "stub": "📝",
938
+ "success": "✅",
939
+ "fail": "❌",
940
+ "parallel": "🔀",
941
+ "map": "🔁",
942
+ "conditional": "🔀",
943
+ }
944
+ type_icon = type_icons.get(step.step_type, "⚙️")
945
+ status_emoji = (
946
+ "✅"
947
+ if step.status == "SUCCESS"
948
+ else "❌"
949
+ if step.status == "FAIL"
950
+ else "⏸️"
951
+ )
952
+
953
+ # Build parameter display - compact horizontal format
954
+ param_info = []
955
+
956
+ if step.input_params:
957
+ params_text = " • ".join(step.input_params)
958
+ param_info.append(
959
+ f'<div style="color: #059669; font-size: 0.7rem; margin-top: 0.2rem; font-family: monospace; word-break: break-all; line-height: 1.3;">📥 {params_text}</div>'
960
+ )
961
+
962
+ if step.output_params:
963
+ params_text = " • ".join(step.output_params)
964
+ param_info.append(
965
+ f'<div style="color: #dc2626; font-size: 0.7rem; margin-top: 0.2rem; font-family: monospace; word-break: break-all; line-height: 1.3;">📤 {params_text}</div>'
966
+ )
967
+
968
+ if step.catalog_ops.get("put") or step.catalog_ops.get("get"):
969
+ catalog_items = []
970
+ if step.catalog_ops.get("put"):
971
+ catalog_items.extend(
972
+ [f"PUT:{item}" for item in step.catalog_ops["put"]]
973
+ )
974
+ if step.catalog_ops.get("get"):
975
+ catalog_items.extend(
976
+ [f"GET:{item}" for item in step.catalog_ops["get"]]
977
+ )
978
+ if catalog_items:
979
+ catalog_text = " • ".join(catalog_items)
980
+ param_info.append(
981
+ f'<div style="color: #7c3aed; font-size: 0.7rem; margin-top: 0.2rem; font-family: monospace; word-break: break-all; line-height: 1.3;">💾 {catalog_text}</div>'
982
+ )
983
+
984
+ # Create unique ID for this step's metadata
985
+ step_id = f"step-{step.internal_name.replace('.', '-')}-{step.start_time.isoformat()}"
986
+
987
+ html_parts.append(f"""
988
+ <div class="step-row">
989
+ <div class="step-info step-level-{step.level}">
990
+ <div style="display: flex; align-items: center; gap: 0.5rem;" class="step-header expandable" data-target="{step_id}">
991
+ <span class="expand-indicator" style="font-size: 0.8rem; color: #6b7280;">{'▼' if param_info else ''}</span>
992
+ {type_icon} {status_emoji} <strong>{step.name}</strong>
993
+ </div>
994
+ {f'<div style="color: #64748b; font-size: 0.75rem; margin-top: 0.25rem;"><strong>{step.command_type.upper()}:</strong> {step.command[:40]}{"..." if len(step.command) > 40 else ""}</div>' if step.command else ''}
995
+ <div class="step-metadata collapsible-content collapsed" id="{step_id}">
996
+ {''.join(param_info)}
997
+ </div>
998
+ </div>
999
+ <div class="gantt-container">
1000
+ <div class="gantt-bar {bar_class}"
1001
+ style="left: {start_percent:.2f}%; width: {max(width_percent, 0.5):.2f}%;"
1002
+ title="{step.name}: {step.duration_ms:.1f}ms">
1003
+ </div>
1004
+ </div>
1005
+ <div style="font-family: monospace; font-size: 0.75rem; color: #6b7280;">
1006
+ {step.duration_ms:.1f}ms
1007
+ </div>
1008
+ </div>
1009
+ """)
1010
+
1011
+ # Close any open composite sections
1012
+ if current_composite:
1013
+ html_parts.append("</div>") # Close collapsible-content
1014
+
1015
+ return "\n".join(html_parts)
1016
+
1017
+ def _generate_html_summary(self) -> str:
1018
+ """Generate HTML summary cards."""
1019
+ executed_steps = [step for step in self.timeline if step.start_time]
1020
+ total_duration = sum(step.duration_ms for step in executed_steps)
1021
+ success_count = sum(1 for step in executed_steps if step.status == "SUCCESS")
1022
+ success_rate = (
1023
+ (success_count / len(executed_steps)) * 100 if executed_steps else 0
1024
+ )
1025
+
1026
+ # Find slowest step
1027
+ slowest_step = (
1028
+ max(executed_steps, key=lambda x: x.duration_ms) if executed_steps else None
1029
+ )
1030
+
1031
+ return f"""
1032
+ <div class="summary-grid">
1033
+ <div class="summary-card">
1034
+ <div class="summary-number status-success">{len(executed_steps)}</div>
1035
+ <div class="summary-label">Total Steps</div>
1036
+ </div>
1037
+ <div class="summary-card">
1038
+ <div class="summary-number duration-medium">{total_duration:.1f}ms</div>
1039
+ <div class="summary-label">Total Duration</div>
1040
+ </div>
1041
+ <div class="summary-card">
1042
+ <div class="summary-number {'status-success' if success_rate == 100 else 'status-fail'}">{success_rate:.1f}%</div>
1043
+ <div class="summary-label">Success Rate</div>
1044
+ </div>
1045
+ <div class="summary-card">
1046
+ <div class="summary-number duration-slow">{'%.1fms' % slowest_step.duration_ms if slowest_step else 'N/A'}</div>
1047
+ <div class="summary-label">Slowest Step<br><small>{slowest_step.name if slowest_step else 'N/A'}</small></div>
1048
+ </div>
1049
+ </div>
1050
+ """
1051
+
1052
+
1053
+ def visualize_simple(
1054
+ run_id: str, show_summary: bool = False, output_html: Optional[str] = None
1055
+ ) -> None:
1056
+ """
1057
+ Simple visualization of a pipeline run.
1058
+
1059
+ Args:
1060
+ run_id: Run ID to visualize
1061
+ show_summary: Whether to show execution summary (deprecated, timeline has enough info)
1062
+ output_html: Optional path to save HTML timeline
1063
+ """
1064
+ # Find run log file
1065
+ run_log_dir = Path(".run_log_store")
1066
+ log_file = run_log_dir / f"{run_id}.json"
1067
+
1068
+ if not log_file.exists():
1069
+ # Try partial match
1070
+ matching_files = [f for f in run_log_dir.glob("*.json") if run_id in f.stem]
1071
+ if matching_files:
1072
+ log_file = matching_files[0]
1073
+ else:
1074
+ print(f"❌ Run log not found for: {run_id}")
1075
+ return
1076
+
1077
+ print(f"📊 Visualizing: {log_file.stem}")
1078
+
1079
+ viz = SimpleVisualizer(log_file)
1080
+ viz.print_simple_timeline()
1081
+
1082
+ if show_summary:
1083
+ viz.print_execution_summary()
1084
+
1085
+ # Generate HTML if requested
1086
+ if output_html:
1087
+ viz.generate_html_timeline(output_html)
1088
+
1089
+
1090
+ def generate_html_timeline(
1091
+ run_id: str, output_file: str, open_browser: bool = True
1092
+ ) -> None:
1093
+ """
1094
+ Generate HTML timeline for a specific run ID.
1095
+
1096
+ Args:
1097
+ run_id: The run ID to visualize
1098
+ output_file: Output HTML file path
1099
+ open_browser: Whether to open the result in browser
1100
+ """
1101
+ from pathlib import Path
1102
+
1103
+ # Find run log file
1104
+ run_log_dir = Path(".run_log_store")
1105
+ log_file = run_log_dir / f"{run_id}.json"
1106
+
1107
+ if not log_file.exists():
1108
+ # Try partial match
1109
+ matching_files = [f for f in run_log_dir.glob("*.json") if run_id in f.stem]
1110
+ if matching_files:
1111
+ log_file = matching_files[0]
1112
+ else:
1113
+ print(f"❌ Run log not found for: {run_id}")
1114
+ return
1115
+
1116
+ print(f"🌐 Generating HTML timeline for: {log_file.stem}")
1117
+
1118
+ # Create visualizer and generate HTML
1119
+ viz = SimpleVisualizer(log_file)
1120
+ viz.generate_html_timeline(output_file)
1121
+
1122
+ if open_browser:
1123
+ import webbrowser
1124
+
1125
+ file_path = Path(output_file).absolute()
1126
+ print(f"🌐 Opening timeline in browser: {file_path.name}")
1127
+ webbrowser.open(file_path.as_uri())
1128
+
1129
+
1130
+ if __name__ == "__main__":
1131
+ import sys
1132
+
1133
+ if len(sys.argv) > 1:
1134
+ if len(sys.argv) > 2 and sys.argv[2].endswith(".html"):
1135
+ # Generate HTML: python viz_simple.py <run_id> <output.html>
1136
+ generate_html_timeline(sys.argv[1], sys.argv[2])
1137
+ else:
1138
+ # Console visualization: python viz_simple.py <run_id>
1139
+ visualize_simple(sys.argv[1])
1140
+ else:
1141
+ print("Usage: python viz_simple.py <run_id> [output.html]")