furu 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,931 @@
1
+ """Filesystem scanner for discovering and parsing Furu experiment state."""
2
+
3
+ import datetime as _dt
4
+ from collections import defaultdict
5
+ from collections.abc import Iterator
6
+ from pathlib import Path
7
+ from typing import cast
8
+
9
+ from ..config import FURU_CONFIG
10
+ from ..storage import MetadataManager, MigrationManager, MigrationRecord, StateAttempt
11
+ from ..storage.state import StateManager, _FuruState
12
+ from .api.models import (
13
+ ChildExperiment,
14
+ DAGEdge,
15
+ DAGExperiment,
16
+ DAGNode,
17
+ DashboardStats,
18
+ ExperimentDAG,
19
+ ExperimentDetail,
20
+ ExperimentRelationships,
21
+ ExperimentSummary,
22
+ JsonDict,
23
+ ParentExperiment,
24
+ StatusCount,
25
+ )
26
+
27
+
28
+ def _iter_roots() -> Iterator[Path]:
29
+ """Iterate over all existing Furu storage roots."""
30
+ for version_controlled in (False, True):
31
+ root = FURU_CONFIG.get_root(version_controlled)
32
+ if root.exists():
33
+ yield root
34
+
35
+
36
+ def _parse_namespace_from_path(experiment_dir: Path, root: Path) -> tuple[str, str]:
37
+ """
38
+ Parse namespace and furu_hash from experiment directory path.
39
+
40
+ Example: /data/my_project/pipelines/TrainModel/abc123 -> ("my_project.pipelines.TrainModel", "abc123")
41
+ """
42
+ relative = experiment_dir.relative_to(root)
43
+ parts = relative.parts
44
+ if len(parts) < 2: # TODO: Maybe this should throw?
45
+ return str(relative), ""
46
+ furu_hash = parts[-1]
47
+ namespace = ".".join(parts[:-1])
48
+ return namespace, furu_hash
49
+
50
+
51
+ def _alias_key(migration: MigrationRecord) -> tuple[str, str, str]:
52
+ return (migration.from_namespace, migration.from_hash, migration.from_root)
53
+
54
+
55
+ def _collect_aliases() -> dict[tuple[str, str, str], list[MigrationRecord]]:
56
+ aliases: dict[tuple[str, str, str], list[MigrationRecord]] = defaultdict(list)
57
+ for root in _iter_roots():
58
+ for experiment_dir in _find_experiment_dirs(root):
59
+ migration = MigrationManager.read_migration(experiment_dir)
60
+ if migration is None or migration.kind != "alias":
61
+ continue
62
+ if migration.overwritten_at is not None:
63
+ continue
64
+ aliases[_alias_key(migration)].append(migration)
65
+ return aliases
66
+
67
+
68
+ def _alias_reference(
69
+ aliases: dict[tuple[str, str, str], list[MigrationRecord]],
70
+ ) -> dict[str, dict[str, list[str]]]:
71
+ ref: dict[str, dict[str, list[str]]] = {}
72
+ for key, records in aliases.items():
73
+ from_namespace, from_hash, _from_root = key
74
+ namespace_map = ref.setdefault(from_namespace, {})
75
+ namespace_map[from_hash] = [record.to_hash for record in records]
76
+ return ref
77
+
78
+
79
+ def _get_class_name(namespace: str) -> str:
80
+ """Extract class name from namespace (last component)."""
81
+ parts = namespace.split(".")
82
+ return parts[-1] if parts else namespace
83
+
84
+
85
+ def _migration_kind(migration: MigrationRecord | None) -> str | None:
86
+ if migration is None:
87
+ return None
88
+ if migration.kind == "migrated":
89
+ return None
90
+ return migration.kind
91
+
92
+
93
+ def _override_summary_attempts(
94
+ summary: ExperimentSummary, state: _FuruState
95
+ ) -> ExperimentSummary:
96
+ attempt = state.attempt
97
+ return summary.model_copy(
98
+ update={
99
+ "attempt_status": attempt.status if attempt else None,
100
+ "attempt_number": attempt.number if attempt else None,
101
+ "backend": attempt.backend if attempt else None,
102
+ "hostname": attempt.owner.hostname if attempt else None,
103
+ "user": attempt.owner.user if attempt else None,
104
+ "started_at": attempt.started_at if attempt else None,
105
+ }
106
+ )
107
+
108
+
109
+ def _state_to_summary(
110
+ state: _FuruState,
111
+ namespace: str,
112
+ furu_hash: str,
113
+ migration: MigrationRecord | None = None,
114
+ original_status: str | None = None,
115
+ original_namespace: str | None = None,
116
+ original_hash: str | None = None,
117
+ ) -> ExperimentSummary:
118
+ """Convert a Furu state to an experiment summary."""
119
+ attempt = state.attempt
120
+ return ExperimentSummary(
121
+ namespace=namespace,
122
+ furu_hash=furu_hash,
123
+ class_name=_get_class_name(namespace),
124
+ result_status=state.result.status,
125
+ attempt_status=attempt.status if attempt else None,
126
+ attempt_number=attempt.number if attempt else None,
127
+ updated_at=state.updated_at,
128
+ started_at=attempt.started_at if attempt else None,
129
+ # Additional fields for filtering
130
+ backend=attempt.backend if attempt else None,
131
+ hostname=attempt.owner.hostname if attempt else None,
132
+ user=attempt.owner.user if attempt else None,
133
+ migration_kind=_migration_kind(migration) if migration else None,
134
+ migration_policy=migration.policy if migration else None,
135
+ migrated_at=migration.migrated_at if migration else None,
136
+ overwritten_at=migration.overwritten_at if migration else None,
137
+ migration_origin=migration.origin if migration else None,
138
+ migration_note=migration.note if migration else None,
139
+ from_namespace=migration.from_namespace if migration else None,
140
+ from_hash=migration.from_hash if migration else None,
141
+ to_namespace=migration.to_namespace if migration else None,
142
+ to_hash=migration.to_hash if migration else None,
143
+ original_result_status=original_status,
144
+ )
145
+
146
+
147
+ def _find_experiment_dirs(root: Path) -> list[Path]:
148
+ """Find all directories containing .furu/state.json files."""
149
+ experiments = []
150
+
151
+ # Walk the directory tree looking for .furu directories
152
+ for furu_dir in root.rglob(StateManager.INTERNAL_DIR):
153
+ if furu_dir.is_dir():
154
+ state_file = furu_dir / StateManager.STATE_FILE
155
+ if state_file.is_file():
156
+ experiments.append(furu_dir.parent)
157
+
158
+ return experiments
159
+
160
+
161
+ def _parse_datetime(value: str | None) -> _dt.datetime | None:
162
+ """Parse ISO datetime string to datetime object."""
163
+ if not value:
164
+ return None
165
+ dt = _dt.datetime.fromisoformat(value)
166
+ if dt.tzinfo is None:
167
+ dt = dt.replace(tzinfo=_dt.timezone.utc)
168
+ return dt
169
+
170
+
171
+ def _read_metadata_with_defaults(
172
+ directory: Path, migration: MigrationRecord | None
173
+ ) -> JsonDict | None:
174
+ metadata = MetadataManager.read_metadata_raw(directory)
175
+ if not metadata or migration is None:
176
+ return metadata
177
+ if migration.kind != "alias" or migration.overwritten_at is not None:
178
+ return metadata
179
+ if not migration.default_values:
180
+ return metadata
181
+
182
+ furu_obj = metadata.get("furu_obj")
183
+ if not isinstance(furu_obj, dict):
184
+ return metadata
185
+
186
+ defaults = migration.default_values
187
+ updates: dict[str, str | int | float | bool] = {}
188
+ for field, value in defaults.items():
189
+ if field not in furu_obj:
190
+ updates[field] = value
191
+
192
+ if not updates:
193
+ return metadata
194
+
195
+ updated_obj = dict(furu_obj)
196
+ updated_obj.update(updates)
197
+ updated_metadata = dict(metadata)
198
+ updated_metadata["furu_obj"] = updated_obj
199
+ return updated_metadata
200
+
201
+
202
+ def _get_nested_value(data: dict, path: str) -> str | int | float | bool | None:
203
+ """
204
+ Get a nested value from a dict using dot notation.
205
+
206
+ Example: _get_nested_value({"a": {"b": 1}}, "a.b") -> 1
207
+ """
208
+ keys = path.split(".")
209
+ current = data
210
+ for key in keys:
211
+ if not isinstance(current, dict):
212
+ return None
213
+ if key not in current:
214
+ return None
215
+ current = current[key]
216
+ # Only return primitive values that can be compared as strings
217
+ if isinstance(current, (str, int, float, bool)):
218
+ return current
219
+ return None
220
+
221
+
222
+ def scan_experiments(
223
+ *,
224
+ result_status: str | None = None,
225
+ attempt_status: str | None = None,
226
+ namespace_prefix: str | None = None,
227
+ backend: str | None = None,
228
+ hostname: str | None = None,
229
+ user: str | None = None,
230
+ started_after: str | None = None,
231
+ started_before: str | None = None,
232
+ updated_after: str | None = None,
233
+ updated_before: str | None = None,
234
+ config_filter: str | None = None,
235
+ migration_kind: str | None = None,
236
+ migration_policy: str | None = None,
237
+ view: str = "resolved",
238
+ ) -> list[ExperimentSummary]:
239
+ """
240
+ Scan the filesystem for Furu experiments.
241
+
242
+ Args:
243
+ result_status: Filter by result status (absent, incomplete, success, failed)
244
+ attempt_status: Filter by attempt status (queued, running, success, failed, etc.)
245
+ namespace_prefix: Filter by namespace prefix
246
+ backend: Filter by backend (local, submitit)
247
+ hostname: Filter by hostname
248
+ user: Filter by user who ran the experiment
249
+ started_after: Filter experiments started after this ISO datetime
250
+ started_before: Filter experiments started before this ISO datetime
251
+ updated_after: Filter experiments updated after this ISO datetime
252
+ updated_before: Filter experiments updated before this ISO datetime
253
+ config_filter: Filter by config field in format "field.path=value"
254
+ view: "resolved" uses alias metadata; "original" uses original metadata/state.
255
+
256
+ Returns:
257
+ List of experiment summaries, sorted by updated_at (newest first)
258
+ """
259
+ experiments: list[ExperimentSummary] = []
260
+ seen_original: set[tuple[str, str, str]] = set()
261
+
262
+ # Parse datetime filters
263
+ started_after_dt = _parse_datetime(started_after)
264
+ started_before_dt = _parse_datetime(started_before)
265
+ updated_after_dt = _parse_datetime(updated_after)
266
+ updated_before_dt = _parse_datetime(updated_before)
267
+
268
+ # Parse config filter (format: "field.path=value")
269
+ config_field: str | None = None
270
+ config_value: str | None = None
271
+ if config_filter and "=" in config_filter:
272
+ config_field, config_value = config_filter.split("=", 1)
273
+
274
+ for root in _iter_roots():
275
+ for experiment_dir in _find_experiment_dirs(root):
276
+ state = StateManager.read_state(experiment_dir)
277
+ namespace, furu_hash = _parse_namespace_from_path(experiment_dir, root)
278
+ migration = MigrationManager.read_migration(experiment_dir)
279
+ original_status: str | None = None
280
+ original_state: _FuruState | None = None
281
+ metadata_dir = experiment_dir
282
+ alias_active = False
283
+
284
+ if migration is not None and migration.kind == "alias":
285
+ original_dir = MigrationManager.resolve_dir(migration, target="from")
286
+ original_state = StateManager.read_state(original_dir)
287
+ original_status = original_state.result.status
288
+ alias_active = (
289
+ migration.overwritten_at is None
290
+ and state.result.status == "migrated"
291
+ and original_status == "success"
292
+ )
293
+ original_key = (
294
+ migration.from_namespace,
295
+ migration.from_hash,
296
+ migration.from_root,
297
+ )
298
+ if view == "original":
299
+ if original_key in seen_original:
300
+ continue
301
+ seen_original.add(original_key)
302
+ state = original_state
303
+ namespace = migration.from_namespace
304
+ furu_hash = migration.from_hash
305
+ metadata_dir = original_dir
306
+ elif alias_active:
307
+ metadata_dir = original_dir
308
+ elif view == "original":
309
+ original_key = (
310
+ namespace,
311
+ furu_hash,
312
+ MigrationManager.root_kind_for_dir(experiment_dir),
313
+ )
314
+ if original_key in seen_original:
315
+ continue
316
+ seen_original.add(original_key)
317
+
318
+ summary = _state_to_summary(
319
+ state,
320
+ namespace,
321
+ furu_hash,
322
+ migration=migration,
323
+ original_status=original_status,
324
+ original_namespace=migration.from_namespace if migration else None,
325
+ original_hash=migration.from_hash if migration else None,
326
+ )
327
+
328
+ if (
329
+ migration is not None
330
+ and migration.kind == "alias"
331
+ and view == "resolved"
332
+ and alias_active
333
+ and original_state is not None
334
+ ):
335
+ summary = _override_summary_attempts(summary, original_state)
336
+
337
+ filter_updated_at = summary.updated_at
338
+ if (
339
+ migration is not None
340
+ and migration.kind == "alias"
341
+ and view == "resolved"
342
+ and alias_active
343
+ and original_state is not None
344
+ ):
345
+ filter_updated_at = original_state.updated_at
346
+
347
+ # Apply filters
348
+ if result_status and summary.result_status != result_status:
349
+ continue
350
+ if attempt_status and summary.attempt_status != attempt_status:
351
+ continue
352
+ if namespace_prefix and not summary.namespace.startswith(namespace_prefix):
353
+ continue
354
+ if backend and summary.backend != backend:
355
+ continue
356
+ if hostname and summary.hostname != hostname:
357
+ continue
358
+ if user and summary.user != user:
359
+ continue
360
+ if migration_kind and summary.migration_kind != migration_kind:
361
+ continue
362
+ if migration_policy and summary.migration_policy != migration_policy:
363
+ continue
364
+
365
+ # Date filters
366
+ if started_after_dt or started_before_dt:
367
+ started_dt = _parse_datetime(summary.started_at)
368
+ if started_dt:
369
+ if started_after_dt and started_dt < started_after_dt:
370
+ continue
371
+ if started_before_dt and started_dt > started_before_dt:
372
+ continue
373
+ elif started_after_dt or started_before_dt:
374
+ # No started_at but we're filtering by it - exclude
375
+ continue
376
+
377
+ if updated_after_dt or updated_before_dt:
378
+ updated_dt = _parse_datetime(filter_updated_at)
379
+ if updated_dt:
380
+ if updated_after_dt and updated_dt < updated_after_dt:
381
+ continue
382
+ if updated_before_dt and updated_dt > updated_before_dt:
383
+ continue
384
+ elif updated_after_dt or updated_before_dt:
385
+ # No updated_at but we're filtering by it - exclude
386
+ continue
387
+
388
+ # Config field filter - requires reading metadata
389
+ if config_field and config_value is not None:
390
+ defaults_migration = migration if view == "resolved" else None
391
+ metadata = _read_metadata_with_defaults(
392
+ metadata_dir,
393
+ defaults_migration,
394
+ )
395
+ if metadata:
396
+ furu_obj = metadata.get("furu_obj")
397
+ if isinstance(furu_obj, dict):
398
+ actual_value = _get_nested_value(furu_obj, config_field)
399
+ if str(actual_value) != config_value:
400
+ continue
401
+ else:
402
+ continue
403
+ else:
404
+ continue
405
+
406
+ experiments.append(summary)
407
+
408
+ # Sort by updated_at (newest first), with None values at the end
409
+ experiments.sort(
410
+ key=lambda e: (e.updated_at is None, e.updated_at or ""),
411
+ reverse=True,
412
+ )
413
+
414
+ return experiments
415
+
416
+
417
+ def get_experiment_detail(
418
+ namespace: str,
419
+ furu_hash: str,
420
+ *,
421
+ view: str = "resolved",
422
+ ) -> ExperimentDetail | None:
423
+ """
424
+ Get detailed information about a specific experiment.
425
+
426
+ Args:
427
+ namespace: Dot-separated namespace (e.g., "my_project.pipelines.TrainModel")
428
+ furu_hash: Hash identifying the specific experiment
429
+ view: "resolved" uses alias metadata; "original" uses original metadata/state.
430
+
431
+ Returns:
432
+ Experiment detail or None if not found
433
+ """
434
+ # Convert namespace to path
435
+ namespace_path = Path(*namespace.split("."))
436
+ alias_reference = _alias_reference(_collect_aliases())
437
+
438
+ for root in _iter_roots():
439
+ experiment_dir = root / namespace_path / furu_hash
440
+ state_path = StateManager.get_state_path(experiment_dir)
441
+
442
+ if not state_path.is_file():
443
+ continue
444
+
445
+ state = StateManager.read_state(experiment_dir)
446
+ migration = MigrationManager.read_migration(experiment_dir)
447
+ metadata = _read_metadata_with_defaults(
448
+ experiment_dir,
449
+ migration if view == "resolved" else None,
450
+ )
451
+ original_status: str | None = None
452
+ original_namespace: str | None = None
453
+ original_hash: str | None = None
454
+
455
+ if migration is not None and migration.kind == "alias":
456
+ original_dir = MigrationManager.resolve_dir(migration, target="from")
457
+ original_state = StateManager.read_state(original_dir)
458
+ original_status = original_state.result.status
459
+ original_namespace = migration.from_namespace
460
+ original_hash = migration.from_hash
461
+ if view == "original":
462
+ state = original_state
463
+ metadata = MetadataManager.read_metadata_raw(original_dir)
464
+ experiment_dir = original_dir
465
+ namespace = original_namespace
466
+ furu_hash = original_hash
467
+ else:
468
+ metadata = _read_metadata_with_defaults(original_dir, migration)
469
+ elif migration is not None and migration.kind in {
470
+ "moved",
471
+ "copied",
472
+ "migrated",
473
+ }:
474
+ if view == "original":
475
+ original_dir = MigrationManager.resolve_dir(migration, target="from")
476
+ state = StateManager.read_state(original_dir)
477
+ metadata = MetadataManager.read_metadata_raw(original_dir)
478
+ experiment_dir = original_dir
479
+ namespace = migration.from_namespace
480
+ furu_hash = migration.from_hash
481
+ original_namespace = migration.from_namespace
482
+ original_hash = migration.from_hash
483
+
484
+ attempt = state.attempt
485
+ if view == "original" and migration is not None and migration.kind == "alias":
486
+ alias_source_namespace = migration.from_namespace
487
+ alias_source_hash = migration.from_hash
488
+ else:
489
+ alias_source_namespace = namespace
490
+ alias_source_hash = furu_hash
491
+
492
+ alias_keys = alias_reference.get(alias_source_namespace, {}).get(
493
+ alias_source_hash,
494
+ [],
495
+ )
496
+ alias_namespaces = (
497
+ [alias_source_namespace] * len(alias_keys) if alias_keys else None
498
+ )
499
+ alias_hashes = alias_keys if alias_keys else None
500
+ return ExperimentDetail(
501
+ namespace=namespace,
502
+ furu_hash=furu_hash,
503
+ class_name=_get_class_name(namespace),
504
+ result_status=state.result.status,
505
+ attempt_status=attempt.status if attempt else None,
506
+ attempt_number=attempt.number if attempt else None,
507
+ updated_at=state.updated_at,
508
+ started_at=attempt.started_at if attempt else None,
509
+ backend=attempt.backend if attempt else None,
510
+ hostname=attempt.owner.hostname if attempt else None,
511
+ user=attempt.owner.user if attempt else None,
512
+ directory=str(experiment_dir),
513
+ state=state.model_dump(mode="json"),
514
+ metadata=metadata,
515
+ attempt=StateAttempt.from_internal(attempt) if attempt else None,
516
+ migration_kind=_migration_kind(migration) if migration else None,
517
+ migration_policy=migration.policy if migration else None,
518
+ migrated_at=migration.migrated_at if migration else None,
519
+ overwritten_at=migration.overwritten_at if migration else None,
520
+ migration_origin=migration.origin if migration else None,
521
+ migration_note=migration.note if migration else None,
522
+ from_namespace=migration.from_namespace if migration else None,
523
+ from_hash=migration.from_hash if migration else None,
524
+ to_namespace=migration.to_namespace if migration else None,
525
+ to_hash=migration.to_hash if migration else None,
526
+ original_result_status=original_status,
527
+ original_namespace=original_namespace,
528
+ original_hash=original_hash,
529
+ alias_namespaces=alias_namespaces,
530
+ alias_hashes=alias_hashes,
531
+ )
532
+
533
+ return None
534
+
535
+
536
+ def get_stats() -> DashboardStats:
537
+ """
538
+ Get aggregate statistics for the dashboard.
539
+
540
+ Returns:
541
+ Dashboard statistics including counts by status
542
+ """
543
+ result_counts: dict[str, int] = defaultdict(int)
544
+ attempt_counts: dict[str, int] = defaultdict(int)
545
+ total = 0
546
+ running = 0
547
+ queued = 0
548
+ failed = 0
549
+ success = 0
550
+
551
+ for root in _iter_roots():
552
+ for experiment_dir in _find_experiment_dirs(root):
553
+ state = StateManager.read_state(experiment_dir)
554
+ total += 1
555
+
556
+ result_counts[state.result.status] += 1
557
+
558
+ if state.result.status == "success":
559
+ success += 1
560
+ elif state.result.status == "failed":
561
+ failed += 1
562
+
563
+ attempt = state.attempt
564
+ if attempt:
565
+ attempt_counts[attempt.status] += 1
566
+ if attempt.status == "running":
567
+ running += 1
568
+ elif attempt.status == "queued":
569
+ queued += 1
570
+
571
+ return DashboardStats(
572
+ total=total,
573
+ by_result_status=[
574
+ StatusCount(status=status, count=count)
575
+ for status, count in sorted(result_counts.items())
576
+ ],
577
+ by_attempt_status=[
578
+ StatusCount(status=status, count=count)
579
+ for status, count in sorted(attempt_counts.items())
580
+ ],
581
+ running_count=running,
582
+ queued_count=queued,
583
+ failed_count=failed,
584
+ success_count=success,
585
+ )
586
+
587
+
588
+ def _extract_dependencies_from_furu_obj(
589
+ furu_obj: dict[str, object],
590
+ ) -> list[tuple[str, str]]:
591
+ """
592
+ Extract dependency class names from a serialized furu object.
593
+
594
+ Looks for nested objects with __class__ markers, which indicate Furu dependencies.
595
+
596
+ Args:
597
+ furu_obj: The serialized furu object (from metadata.furu_obj)
598
+
599
+ Returns:
600
+ List of (field_name, dependency_class_name) tuples
601
+ """
602
+ dependencies: list[tuple[str, str]] = []
603
+
604
+ for key, value in furu_obj.items():
605
+ if key == "__class__":
606
+ continue
607
+ if isinstance(value, dict):
608
+ nested_obj = cast(dict[str, object], value)
609
+ dep_class_value = nested_obj.get("__class__")
610
+ if dep_class_value is not None:
611
+ # This is a nested Furu object (dependency)
612
+ dependencies.append((key, str(dep_class_value)))
613
+
614
+ return dependencies
615
+
616
+
617
+ def _get_class_hierarchy(full_class_name: str) -> str | None:
618
+ """
619
+ Try to determine the parent class from the full class name.
620
+
621
+ This is a heuristic - we look at class naming patterns.
622
+ In the future, this could be enhanced to read actual class hierarchies.
623
+ """
624
+ # For now, we don't have access to actual class hierarchies at runtime
625
+ # This would require importing the classes or storing hierarchy info in metadata
626
+ return None
627
+
628
+
629
+ def get_experiment_dag() -> ExperimentDAG:
630
+ """
631
+ Build a DAG of all experiments based on their dependencies.
632
+
633
+ The DAG is organized by class types:
634
+ - Each node represents a class (e.g., TrainModel)
635
+ - Experiments of the same class are grouped into the same node
636
+ - Edges represent dependencies between classes (field references)
637
+
638
+ Returns:
639
+ ExperimentDAG with nodes and edges for visualization
640
+ """
641
+ # Collect all experiments with their metadata
642
+ experiments_by_class: dict[str, list[tuple[str, str, str, str | None]]] = (
643
+ defaultdict(list)
644
+ )
645
+ # Maps full class name -> (short name, experiments)
646
+ class_info: dict[str, str] = {} # full_class_name -> short_class_name
647
+ # Collect all edges (deduped by class pair)
648
+ edge_set: set[tuple[str, str, str]] = set() # (source_class, target_class, field)
649
+
650
+ for root in _iter_roots():
651
+ for experiment_dir in _find_experiment_dirs(root):
652
+ state = StateManager.read_state(experiment_dir)
653
+ namespace, furu_hash = _parse_namespace_from_path(experiment_dir, root)
654
+ metadata = MetadataManager.read_metadata_raw(experiment_dir)
655
+
656
+ if not metadata:
657
+ continue
658
+
659
+ migration = MigrationManager.read_migration(experiment_dir)
660
+ if migration is not None and migration.kind == "alias":
661
+ continue
662
+
663
+ furu_obj = metadata.get("furu_obj")
664
+ if not isinstance(furu_obj, dict):
665
+ continue
666
+
667
+ full_class_name = furu_obj.get("__class__")
668
+ if not isinstance(full_class_name, str):
669
+ continue
670
+
671
+ # Extract short class name
672
+ short_class_name = full_class_name.split(".")[-1]
673
+ class_info[full_class_name] = short_class_name
674
+
675
+ # Get attempt status
676
+ attempt_status = state.attempt.status if state.attempt else None
677
+
678
+ # Store experiment info
679
+ experiments_by_class[full_class_name].append(
680
+ (namespace, furu_hash, state.result.status, attempt_status)
681
+ )
682
+
683
+ # Extract dependencies and create edges
684
+ dependencies = _extract_dependencies_from_furu_obj(furu_obj)
685
+ for field_name, dep_class in dependencies:
686
+ # Edge goes from dependency (source/upstream) to this class (target/downstream)
687
+ edge_set.add((dep_class, full_class_name, field_name))
688
+ # Also make sure the dependency class is in our class_info
689
+ if dep_class not in class_info:
690
+ class_info[dep_class] = dep_class.split(".")[-1]
691
+
692
+ # Build nodes
693
+ nodes: list[DAGNode] = []
694
+ for full_class_name, short_class_name in class_info.items():
695
+ experiments = experiments_by_class.get(full_class_name, [])
696
+
697
+ # Count statuses
698
+ success_count = sum(1 for _, _, rs, _ in experiments if rs == "success")
699
+ failed_count = sum(1 for _, _, rs, _ in experiments if rs == "failed")
700
+ running_count = sum(
701
+ 1 for _, _, _, attempt_status in experiments if attempt_status == "running"
702
+ )
703
+
704
+ node = DAGNode(
705
+ id=full_class_name,
706
+ class_name=short_class_name,
707
+ full_class_name=full_class_name,
708
+ experiments=[
709
+ DAGExperiment(
710
+ namespace=ns,
711
+ furu_hash=h,
712
+ result_status=rs,
713
+ attempt_status=attempt_status,
714
+ )
715
+ for ns, h, rs, attempt_status in experiments
716
+ ],
717
+ total_count=len(experiments),
718
+ success_count=success_count,
719
+ failed_count=failed_count,
720
+ running_count=running_count,
721
+ parent_class=_get_class_hierarchy(full_class_name),
722
+ )
723
+ nodes.append(node)
724
+
725
+ # Build edges
726
+ edges: list[DAGEdge] = [
727
+ DAGEdge(source=source, target=target, field_name=field)
728
+ for source, target, field in edge_set
729
+ ]
730
+
731
+ # Sort nodes by class name for consistent ordering
732
+ nodes.sort(key=lambda n: n.class_name)
733
+ edges.sort(key=lambda e: (e.source, e.target))
734
+
735
+ return ExperimentDAG(
736
+ nodes=nodes,
737
+ edges=edges,
738
+ total_nodes=len(nodes),
739
+ total_edges=len(edges),
740
+ total_experiments=sum(node.total_count for node in nodes),
741
+ )
742
+
743
+
744
+ def _find_experiment_by_furu_obj(
745
+ furu_obj: dict[str, object],
746
+ ) -> tuple[str, str, str] | None:
747
+ """
748
+ Find an experiment that matches the given furu_obj serialization.
749
+
750
+ Args:
751
+ furu_obj: The serialized furu object to find
752
+
753
+ Returns:
754
+ Tuple of (namespace, furu_hash, result_status) if found, None otherwise
755
+ """
756
+ full_class_name = furu_obj.get("__class__")
757
+ if not isinstance(full_class_name, str):
758
+ return None
759
+
760
+ # Convert class name to potential namespace path
761
+ # e.g., "my_project.pipelines.TrainModel" -> "my_project/pipelines/TrainModel"
762
+ namespace_path = Path(*full_class_name.split("."))
763
+
764
+ for root in _iter_roots():
765
+ class_dir = root / namespace_path
766
+ if not class_dir.exists():
767
+ continue
768
+
769
+ # Search through experiments of this class
770
+ for experiment_dir in _find_experiment_dirs(class_dir):
771
+ metadata = MetadataManager.read_metadata_raw(experiment_dir)
772
+ if not metadata:
773
+ continue
774
+
775
+ migration = MigrationManager.read_migration(experiment_dir)
776
+ if migration is not None and migration.kind == "alias":
777
+ continue
778
+
779
+ stored_furu_obj = metadata.get("furu_obj")
780
+ if stored_furu_obj == furu_obj:
781
+ namespace, furu_hash = _parse_namespace_from_path(experiment_dir, root)
782
+ state = StateManager.read_state(experiment_dir)
783
+ return namespace, furu_hash, state.result.status
784
+
785
+ return None
786
+
787
+
788
+ def get_experiment_relationships(
789
+ namespace: str,
790
+ furu_hash: str,
791
+ *,
792
+ view: str = "resolved",
793
+ ) -> ExperimentRelationships | None:
794
+ """
795
+ Get parent and child relationships for an experiment.
796
+
797
+ Args:
798
+ namespace: Dot-separated namespace (e.g., "my_project.pipelines.TrainModel")
799
+ furu_hash: Hash identifying the specific experiment
800
+
801
+ Returns:
802
+ ExperimentRelationships or None if experiment not found
803
+ """
804
+ # First get the experiment's metadata
805
+ namespace_path = Path(*namespace.split("."))
806
+
807
+ target_metadata: JsonDict | None = None
808
+
809
+ for root in _iter_roots():
810
+ experiment_dir = root / namespace_path / furu_hash
811
+ state_path = StateManager.get_state_path(experiment_dir)
812
+
813
+ if state_path.is_file():
814
+ migration = MigrationManager.read_migration(experiment_dir)
815
+ if (
816
+ view == "original"
817
+ and migration is not None
818
+ and migration.kind == "alias"
819
+ ):
820
+ experiment_dir = MigrationManager.resolve_dir(migration, target="from")
821
+ target_metadata = MetadataManager.read_metadata_raw(experiment_dir)
822
+ else:
823
+ target_metadata = _read_metadata_with_defaults(
824
+ experiment_dir,
825
+ migration if view == "resolved" else None,
826
+ )
827
+ break
828
+
829
+ if not target_metadata:
830
+ return None
831
+
832
+ target_furu_obj_raw = target_metadata.get("furu_obj")
833
+ if not isinstance(target_furu_obj_raw, dict):
834
+ return None
835
+ target_furu_obj = cast(dict[str, object], target_furu_obj_raw)
836
+
837
+ # Extract parents from this experiment's furu_obj
838
+ parents: list[ParentExperiment] = []
839
+ dependencies = _extract_dependencies_from_furu_obj(target_furu_obj)
840
+
841
+ for field_name, dep_class in dependencies:
842
+ parent_obj = target_furu_obj.get(field_name)
843
+ if not isinstance(parent_obj, dict):
844
+ continue
845
+
846
+ parent_obj_dict = cast(dict[str, object], parent_obj)
847
+ short_class_name = dep_class.split(".")[-1]
848
+
849
+ # Try to find the actual experiment
850
+ found = _find_experiment_by_furu_obj(parent_obj_dict)
851
+
852
+ # Extract config (everything except __class__)
853
+ config = {k: v for k, v in parent_obj_dict.items() if k != "__class__"}
854
+
855
+ if found:
856
+ parent_ns, parent_hash, parent_status = found
857
+ parents.append(
858
+ ParentExperiment(
859
+ field_name=field_name,
860
+ class_name=short_class_name,
861
+ full_class_name=dep_class,
862
+ namespace=parent_ns,
863
+ furu_hash=parent_hash,
864
+ result_status=parent_status,
865
+ config=config,
866
+ )
867
+ )
868
+ else:
869
+ parents.append(
870
+ ParentExperiment(
871
+ field_name=field_name,
872
+ class_name=short_class_name,
873
+ full_class_name=dep_class,
874
+ namespace=None,
875
+ furu_hash=None,
876
+ result_status=None,
877
+ config=config,
878
+ )
879
+ )
880
+
881
+ # Find children by scanning all experiments
882
+ children: list[ChildExperiment] = []
883
+
884
+ for root in _iter_roots():
885
+ for experiment_dir in _find_experiment_dirs(root):
886
+ migration = MigrationManager.read_migration(experiment_dir)
887
+ if migration is not None and migration.kind == "alias":
888
+ continue
889
+
890
+ child_namespace, child_hash = _parse_namespace_from_path(
891
+ experiment_dir, root
892
+ )
893
+
894
+ # Skip self
895
+ if child_namespace == namespace and child_hash == furu_hash:
896
+ continue
897
+
898
+ child_metadata = MetadataManager.read_metadata_raw(experiment_dir)
899
+ if not child_metadata:
900
+ continue
901
+
902
+ child_furu_obj = child_metadata.get("furu_obj")
903
+ if not isinstance(child_furu_obj, dict):
904
+ continue
905
+
906
+ child_obj_dict = cast(dict[str, object], child_furu_obj)
907
+
908
+ # Check if this experiment depends on our target
909
+ for field_name, value in child_obj_dict.items():
910
+ if field_name == "__class__":
911
+ continue
912
+
913
+ if isinstance(value, dict) and value == target_furu_obj:
914
+ # This experiment depends on our target
915
+ child_class = child_obj_dict.get("__class__")
916
+ if not isinstance(child_class, str):
917
+ continue
918
+
919
+ state = StateManager.read_state(experiment_dir)
920
+ children.append(
921
+ ChildExperiment(
922
+ field_name=field_name,
923
+ class_name=child_class.split(".")[-1],
924
+ full_class_name=child_class,
925
+ namespace=child_namespace,
926
+ furu_hash=child_hash,
927
+ result_status=state.result.status,
928
+ )
929
+ )
930
+
931
+ return ExperimentRelationships(parents=parents, children=children)