b-load 0.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
src/evaluation.py ADDED
@@ -0,0 +1,642 @@
1
+ """
2
+ evaluation.py — Ground-truth computation and algorithm evaluation metrics.
3
+
4
+ This module provides two groups of functionality:
5
+
6
+ 1. **Ground-truth helpers**: convert DAGs to CPDAGs, enumerate true optimal
7
+ adjustment sets, and compute true average treatment effects (ATEs) under
8
+ linear Gaussian and binary SEMs.
9
+
10
+ 2. **Evaluation metrics**: precision, recall, F1 and intervention distance for
11
+ comparing estimated adjustment sets or ATEs to ground truth.
12
+
13
+ These functions are shared by both the LOAD (base) and b-LOAD pipelines.
14
+
15
+ Notes
16
+ -----
17
+ Several functions call R via ``rpy2`` (specifically ``pcalg``'s
18
+ ``dag2cpdag`` and ``optAdjSet``, and the binary ATE helper). The R
19
+ evaluation script ``evaluate.R`` is sourced automatically at module import.
20
+ """
21
+
22
+ from pathlib import Path
23
+ from itertools import permutations
24
+
25
+ import numpy as np
26
+ import networkx as nx
27
+ from rpy2.robjects import default_converter, globalenv, numpy2ri, r
28
+ from rpy2 import robjects
29
+
30
+ # Source the R evaluation functions from the project root
31
+ file_path = Path(__file__).resolve().parent.parent / "evaluate.R"
32
+
33
+ # Source the file using the R 'source' function via robjects
34
+ if file_path.exists():
35
+ robjects.r.source(str(file_path))
36
+ print(f"Successfully sourced: {file_path}")
37
+ else:
38
+ print(f"Error: File not found at {file_path}")
39
+ raise FileNotFoundError(f"R evaluation script not found at {file_path}")
40
+
41
+
42
+ def dag2cpdag(dag: np.ndarray) -> np.ndarray:
43
+ """
44
+ Convert a DAG adjacency matrix to its CPDAG (Markov equivalence class)
45
+ using the ``pcalg`` R package.
46
+
47
+ Args:
48
+ dag (np.ndarray): Binary DAG adjacency matrix (non-zero entry = edge).
49
+
50
+ Returns:
51
+ np.ndarray: CPDAG adjacency matrix as an ``int8`` numpy array.
52
+ ``cpdag[i, j] == 1`` indicates an edge from i to j (directed or
53
+ undirected in the pcalg convention).
54
+ """
55
+ with (default_converter + numpy2ri.converter).context():
56
+ cpdag = r["dag2cpdag"](r["as"](dag != 0, "graphNEL"))
57
+ cpdag = np.array(r["as"](cpdag, "matrix")).astype(np.int8)
58
+ return cpdag
59
+
60
+
61
+ def get_test_nums(results: list) -> np.ndarray:
62
+ """Return the total number of CI tests performed for each result dict."""
63
+ test_sum = [np.sum(h["tests"]) for h in results]
64
+ return np.array(test_sum)
65
+
66
+
67
+ def get_times(results: list) -> np.ndarray:
68
+ """Return the wall-clock time (seconds) recorded for each result dict."""
69
+ return np.array([h["time"] for h in results])
70
+
71
+
72
+ # ADJ-SETS
73
+ def get_optimal_adj_set(amat: np.ndarray, treatment: int, outcome: int):
74
+ """
75
+ Compute the optimal adjustment set (o-set) for the effect of ``treatment``
76
+ on ``outcome`` in the CPDAG / DAG represented by ``amat``, using
77
+ ``pcalg``'s ``optAdjSet`` R function.
78
+
79
+ Returns ``None`` if ``treatment`` is not an ancestor of ``outcome`` in
80
+ ``amat``, or if ``pcalg`` raises an exception (e.g. the effect is not
81
+ identifiable from the given graph).
82
+
83
+ Args:
84
+ amat (np.ndarray): CPDAG or DAG adjacency matrix. A non-zero entry
85
+ at ``[i, j]`` indicates an edge from i toward j (pcalg convention).
86
+ treatment (int): 0-based index of the treatment variable.
87
+ outcome (int): 0-based index of the outcome variable.
88
+
89
+ Returns:
90
+ Optional[set[int]]: The optimal adjustment set as a set of 0-based node
91
+ indices, or ``None`` if the effect is not identifiable.
92
+ """
93
+ if not is_ancestor(treatment, outcome, amat):
94
+ return None
95
+ try:
96
+ with (default_converter + numpy2ri.converter).context():
97
+ S = r["optAdjSet"](
98
+ (r["as"](amat != 0, "graphNEL")), treatment + 1, outcome + 1
99
+ )
100
+ return set(np.array(S) - 1)
101
+ except Exception:
102
+ return None
103
+
104
+
105
+ def get_true_osets(experiments: dict) -> dict:
106
+ """
107
+ Compute the true optimal adjustment sets for a dictionary of synthetic
108
+ experiments.
109
+
110
+ For each experiment, the true DAG is first converted to its CPDAG (Markov
111
+ equivalence class), and the optimal adjustment set is then retrieved from
112
+ that CPDAG via ``pcalg``'s ``optAdjSet``.
113
+
114
+ Args:
115
+ experiments (dict): Keyed by experiment ID. Each value must contain:
116
+
117
+ * ``"true_dag"`` (nx.DiGraph): Ground-truth DAG.
118
+ * ``"targets"`` (sequence): Pair of 0-based node indices
119
+ ``[t1, t2]``; the ancestor direction is inferred from the DAG.
120
+
121
+ Returns:
122
+ dict: Keyed by experiment ID. Each value is a dict with:
123
+
124
+ * ``"treatment"`` (int): 0-based treatment index.
125
+ * ``"outcome"`` (int): 0-based outcome index.
126
+ * ``"oset"`` (Optional[set[int]]): True optimal adjustment set, or
127
+ ``None`` if not identifiable from the CPDAG.
128
+ """
129
+ true_osets = {}
130
+
131
+ for exp_id, exp in experiments.items():
132
+ # 1. Convert the true DAG to a CPDAG
133
+ # Using a helper (dag2cpdag) to represent the Markov Equivalence Class
134
+ true_dag_matrix = nx.to_numpy_array(exp["true_dag"])
135
+ cpdag = dag2cpdag(true_dag_matrix)
136
+
137
+ # 2. Identify treatment and outcome from the true DAG
138
+ # Based on explicit ancestral relations
139
+ targets = exp["targets"]
140
+ if nx.has_path(exp["true_dag"], targets[0], targets[1]):
141
+ treatment, outcome = targets[0], targets[1]
142
+ else:
143
+ treatment, outcome = targets[1], targets[0]
144
+
145
+ # 3. Retrieve the true optimal adjustment set relative to the CPDAG
146
+ true_oset = get_optimal_adj_set(cpdag, treatment, outcome)
147
+
148
+ # 4. Store the ground truth result
149
+ true_osets[exp_id] = {
150
+ "treatment": treatment,
151
+ "outcome": outcome,
152
+ "oset": true_oset,
153
+ }
154
+
155
+ # print(f"Processed Experiment {exp_id}: Treatment={treatment}, Outcome={outcome}")
156
+
157
+ return true_osets
158
+
159
+ def get_precision(true: set, pred: set) -> float:
160
+ """
161
+ Compute precision of a predicted adjustment set against the ground truth.
162
+
163
+ Args:
164
+ true (set): True optimal adjustment set.
165
+ pred (set): Predicted adjustment set.
166
+
167
+ Returns:
168
+ float: Precision = TP / (TP + FP). Returns 1.0 if ``true == pred``,
169
+ 0.0 if ``pred`` is empty and ``true`` is not.
170
+ """
171
+ if true == pred:
172
+ return 1.0
173
+ tp = len(true.intersection(pred))
174
+ fp = len(pred - true)
175
+ return tp / (tp + fp) if (tp + fp) > 0 else 0
176
+
177
+
178
+ def get_recall(true: set, pred: set) -> float:
179
+ """
180
+ Compute recall of a predicted adjustment set against the ground truth.
181
+
182
+ Args:
183
+ true (set): True optimal adjustment set.
184
+ pred (set): Predicted adjustment set.
185
+
186
+ Returns:
187
+ float: Recall = TP / (TP + FN). Returns 1.0 if ``true == pred``.
188
+ """
189
+ if true == pred:
190
+ return 1.0
191
+ tp = len(true.intersection(pred))
192
+ fn = len(true - pred)
193
+ return tp / (tp + fn) if (tp + fn) > 0 else 0
194
+
195
+
196
+ def get_f1(precision: float, recall: float) -> float:
197
+ """
198
+ Compute the F1 score from precision and recall.
199
+
200
+ Args:
201
+ precision (float): Precision score in [0, 1].
202
+ recall (float): Recall score in [0, 1].
203
+
204
+ Returns:
205
+ float: Harmonic mean of precision and recall; 0.0 if both are zero.
206
+ """
207
+ if (precision + recall) > 0:
208
+ return 2 * (precision * recall) / (precision + recall)
209
+ else:
210
+ return 0
211
+
212
+
213
+ def get_oset(project: str, h: dict, treatment: int, outcome: int):
214
+ if project in ["pc", "fges", "marvel", "snap"]:
215
+ est_oset = get_optimal_adj_set(np.array(h["amat"]), treatment, outcome)
216
+ if est_oset is not None:
217
+ est_osets = [est_oset]
218
+ else:
219
+ est_osets = None
220
+ elif project in ["mb_by_mb", "ldecc", "mb_by_mb_plus", "ldecc_plus"]:
221
+ adj_sets = eval(h["adj_sets"])
222
+ if (treatment, outcome) in adj_sets:
223
+ est_osets = adj_sets[(treatment, outcome)]
224
+ else:
225
+ est_osets = None
226
+ elif project in ["ldp", "ldp_plus"]:
227
+ results = eval(h["results"])
228
+ if (treatment, outcome) in results and results[(treatment, outcome)][
229
+ "vas_exists"
230
+ ]:
231
+ parts = results[(treatment, outcome)]
232
+ est_osets = [
233
+ set(parts["Z1"]),
234
+ set(parts["Z1"] + parts["Z4"]),
235
+ set(parts["Z1"] + parts["Z4"] + parts["Z5"]),
236
+ ]
237
+ else:
238
+ est_osets = None
239
+ elif project in ["load", "load_oracle"]:
240
+ adj_sets = eval(h["adj_sets"])
241
+ if h["identifiable"] and (treatment, outcome) in adj_sets:
242
+ est_osets = adj_sets[(treatment, outcome)]
243
+ else:
244
+ est_osets = None
245
+ else:
246
+ raise ValueError("Invalid project")
247
+ return est_osets
248
+
249
+
250
+ def evaluate_oset(algorithm, results, true_osets: dict):
251
+ """
252
+ Sequentially evaluates discovered adjustment sets against ground truth.
253
+ Returns trimmed arrays (removing outliers) for precision, recall, and F1.
254
+ """
255
+ prec_scores = []
256
+ rec_scores = []
257
+ f1_scores = []
258
+
259
+ for h in results:
260
+ # 1. Handle algorithm failures
261
+ if "failed" in h and h["failed"]:
262
+ prec_scores.append(0.0)
263
+ rec_scores.append(0.0)
264
+ f1_scores.append(0.0)
265
+ continue
266
+
267
+ exp_id = h["id"]
268
+ treatment = true_osets[exp_id]["treatment"]
269
+ outcome = true_osets[exp_id]["outcome"]
270
+
271
+ # 2. Get estimated optimal adjustment sets
272
+ # In a sequential run, we call the function directly
273
+ est_osets = get_oset(algorithm, h, treatment, outcome)
274
+ true_oset = true_osets[exp_id]["oset"]
275
+
276
+ # 3. Calculate Scores (Precision, Recall, F1)
277
+ # Based on the paper's metrics for adjustment set quality
278
+ if true_oset is not None and est_osets is not None:
279
+ precs = [get_precision(true_oset, s) for s in est_osets]
280
+ recs = [get_recall(true_oset, s) for s in est_osets]
281
+ f1s = [get_f1(p, r) for p, r in zip(precs, recs)]
282
+
283
+ # Record the best match among returned sets
284
+ prec_scores.append(max(precs))
285
+ rec_scores.append(max(recs))
286
+ f1_scores.append(max(f1s))
287
+
288
+ elif true_oset == est_osets: # Both are None (case for non-existent Oset)
289
+ prec_scores.append(1.0)
290
+ rec_scores.append(1.0)
291
+ f1_scores.append(1.0)
292
+ else:
293
+ # Case where one exists and the other doesn't
294
+ prec_scores.append(0.0)
295
+ rec_scores.append(0.0)
296
+ f1_scores.append(0.0)
297
+
298
+ # 4. Return results
299
+ return (
300
+ (prec_scores),
301
+ (rec_scores),
302
+ (f1_scores),
303
+ )
304
+
305
+
306
+ # Modified functions to evaluate on real data
307
+
308
+
309
+ def get_true_osets_real_data(experiments: dict) -> dict:
310
+ """
311
+ Compute the true optimal adjustment sets for a dictionary of real-data
312
+ experiments.
313
+
314
+ Unlike :func:`get_true_osets`, this function queries the o-set directly
315
+ from the **true DAG matrix** (not from the CPDAG), which is appropriate
316
+ for real datasets where the ground-truth DAG is known (e.g. Sachs protein
317
+ network, DREAM benchmark) but no CPDAG conversion is available or needed.
318
+
319
+ Args:
320
+ experiments (dict): Keyed by experiment ID. Each value must contain:
321
+
322
+ * ``"true_dag"`` (nx.DiGraph): Ground-truth DAG with node indices
323
+ matching the columns of the data matrix.
324
+ * ``"targets"`` (sequence): Pair of 0-based node indices
325
+ ``[t1, t2]``; the ancestor direction is inferred from the DAG.
326
+
327
+ Returns:
328
+ dict: Keyed by experiment ID. Each value is a dict with:
329
+
330
+ * ``"treatment"`` (int): 0-based treatment index.
331
+ * ``"outcome"`` (int): 0-based outcome index.
332
+ * ``"oset"`` (Optional[set[int]]): True optimal adjustment set from
333
+ the true DAG, or ``None`` if not identifiable.
334
+ """
335
+ true_osets = {}
336
+ for exp_id, exp in experiments.items():
337
+ true_dag_matrix = nx.to_numpy_array(exp["true_dag"])
338
+
339
+ targets = exp["targets"]
340
+ if nx.has_path(exp["true_dag"], targets[0], targets[1]):
341
+ treatment, outcome = targets[0], targets[1]
342
+ else:
343
+ treatment, outcome = targets[1], targets[0]
344
+
345
+ true_oset = get_optimal_adj_set(true_dag_matrix, treatment, outcome)
346
+
347
+ true_osets[exp_id] = {
348
+ "treatment": treatment,
349
+ "outcome": outcome,
350
+ "oset": true_oset,
351
+ }
352
+
353
+ return true_osets
354
+
355
+
356
+ # INTERVENTION DISTANCE
357
+ def is_ancestor(t1: int, t2: int, amat: np.ndarray) -> bool:
358
+ """
359
+ Check whether ``t1`` is an ancestor of ``t2`` in the directed graph
360
+ represented by ``amat``.
361
+
362
+ Uses matrix exponentiation on the boolean reachability matrix.
363
+
364
+ Args:
365
+ t1 (int): Candidate ancestor (0-based index).
366
+ t2 (int): Candidate descendant (0-based index).
367
+ amat (np.ndarray): Adjacency matrix where ``amat[i,j] != 0`` encodes
368
+ a directed edge i → j.
369
+
370
+ Returns:
371
+ bool: ``True`` if there exists a directed path t1 → … → t2.
372
+ """
373
+ reach = amat.copy().astype(bool)
374
+ np.fill_diagonal(reach, True)
375
+ reach = np.linalg.matrix_power(reach, reach.shape[0] - 1)
376
+ return reach[t1, t2]
377
+
378
+
379
+ def true_linear_gaussian_effect(
380
+ treatment: int, outcome: int, true_dag: nx.DiGraph, **kwargs
381
+ ) -> float:
382
+ """
383
+ Compute the true total causal effect of ``treatment`` on ``outcome`` in a
384
+ linear Gaussian SEM by summing path products.
385
+
386
+ The edge weights in ``true_dag`` are the linear SEM coefficients (set by
387
+ :func:`~src.data_generation.generate_data`). The total effect equals the
388
+ sum over all directed paths of the products of edge coefficients along each
389
+ path.
390
+
391
+ Args:
392
+ treatment (int): 0-based treatment index.
393
+ outcome (int): 0-based outcome index.
394
+ true_dag (nx.DiGraph): Weighted ground-truth DAG.
395
+ **kwargs: Silently ignored (for interface compatibility with
396
+ ``true_binary_effect``).
397
+
398
+ Returns:
399
+ float: True total causal effect; 0.0 if ``treatment`` is not an
400
+ ancestor of ``outcome``.
401
+ """
402
+ if treatment not in nx.ancestors(true_dag, outcome):
403
+ return 0.0
404
+ amat = nx.to_numpy_array(true_dag)
405
+ return sum(
406
+ np.prod([amat[path[i], path[i + 1]] for i in range(len(path) - 1)])
407
+ for path in nx.all_simple_paths(true_dag, treatment, outcome)
408
+ )
409
+
410
+
411
+ def true_binary_effect(
412
+ treatment: int, outcome: int, true_dag: nx.DiGraph, cpt: object, **kwargs
413
+ ) -> float:
414
+ """
415
+ Compute the true total causal effect of ``treatment`` on ``outcome`` in a
416
+ discrete binary SEM, using the R helper ``true_binary_effect``.
417
+
418
+ Args:
419
+ treatment (int): 0-based treatment index.
420
+ outcome (int): 0-based outcome index.
421
+ true_dag (nx.DiGraph): Ground-truth DAG.
422
+ cpt: R conditional-probability table object produced by
423
+ :func:`~src.data_generation.generate_data`.
424
+ **kwargs: Silently ignored.
425
+
426
+ Returns:
427
+ float: True binary causal effect E[Y | do(X=1)] − E[Y | do(X=0)];
428
+ 0.0 if ``treatment`` is not an ancestor of ``outcome``.
429
+ """
430
+ if treatment not in nx.ancestors(true_dag, outcome):
431
+ return 0.0
432
+ with (default_converter + numpy2ri.converter).context():
433
+ return globalenv["true_binary_effect"](
434
+ int(treatment) + 1, int(outcome) + 1, nx.to_numpy_array(true_dag), cpt
435
+ )[0]
436
+
437
+
438
+ def true_causal_effects(experiments: dict, family: str = "gaussian") -> dict:
439
+ """
440
+ Compute the true causal effects for all target pairs across all experiments.
441
+
442
+ Args:
443
+ experiments (dict): Keyed by experiment ID. Each value must contain
444
+ ``"targets"`` (pair of 0-based node indices), ``"true_dag"``
445
+ (``nx.DiGraph``), and ``"cpt"`` (R CPT object, only used for
446
+ ``family="binary"``).
447
+ family (str): SEM family — ``"gaussian"`` (uses
448
+ :func:`true_linear_gaussian_effect`) or ``"binary"`` (uses
449
+ :func:`true_binary_effect`).
450
+
451
+ Returns:
452
+ dict: Nested dict ``effects[exp_id][(t1, t2)]`` = true total causal
453
+ effect of ``t1`` on ``t2``.
454
+ """
455
+ if family == "gaussian":
456
+ get_effect = true_linear_gaussian_effect
457
+ elif family == "binary":
458
+ get_effect = true_binary_effect
459
+ else:
460
+ raise ValueError("Invalid family")
461
+ effects = {}
462
+ for exp_id, exp in experiments.items():
463
+ effects[exp_id] = {}
464
+ for t1, t2 in permutations(exp["targets"], 2):
465
+ effects[exp_id][(t1, t2)] = get_effect(
466
+ treatment=t1, outcome=t2, true_dag=exp["true_dag"], cpt=exp["cpt"]
467
+ )
468
+ return effects
469
+
470
+
471
+ def estimate_binary(samples, val, outcome, treatment, adj_set):
472
+ # Filter rows based on the treatment value
473
+ samples_val = samples[samples[:, treatment] == val]
474
+ # Create the design matrix X
475
+ X = np.column_stack(
476
+ [np.ones(len(samples_val))] + [samples_val[:, idx] for idx in adj_set]
477
+ )
478
+ # Outcome variable (filtered by val)
479
+ y = samples_val[:, outcome]
480
+ # Solve the least squares problem X * beta = y
481
+ beta, _, _, _ = np.linalg.lstsq(X, y, rcond=None)
482
+ # Predict the outcome for all rows in the original samples using the estimated beta
483
+ X_full = np.column_stack(
484
+ [np.ones(len(samples))] + [samples[:, idx] for idx in adj_set]
485
+ )
486
+ y_pred = X_full @ beta
487
+ return y_pred.mean()
488
+
489
+
490
+ def estimate_ate(
491
+ treatment: int,
492
+ outcome: int,
493
+ adj_sets: list[list[int]],
494
+ samples: np.ndarray,
495
+ family: str = "gaussian",
496
+ ) -> list[float]:
497
+ effects = []
498
+ for adj_set in adj_sets:
499
+ adj_set = list(adj_set)
500
+ if family == "gaussian":
501
+ x = samples[:, [treatment] + adj_set]
502
+ y = samples[:, outcome]
503
+ A = np.vstack((x.T, np.ones(len(x))))
504
+ effect = np.linalg.lstsq(A.T, y, rcond=None)[0][0]
505
+ effects.append(effect)
506
+ elif family == "binary":
507
+ do_1 = estimate_binary(samples, 1, outcome, treatment, adj_set)
508
+ do_0 = estimate_binary(samples, 0, outcome, treatment, adj_set)
509
+ effects.append(do_1 - do_0)
510
+ return effects
511
+
512
+
513
+ def get_adj_sets(project: str, h: dict, t1: int, t2: int):
514
+ if "failed" in h and h["failed"]:
515
+ return [[]]
516
+ if project in ["pc", "fges", "marvel", "snap"]:
517
+ amat = np.array(h["amat"], dtype=np.int8)
518
+ if is_ancestor(t1, t2, amat):
519
+ oset = get_optimal_adj_set(amat, t1, t2)
520
+ if oset != None: # identifiable
521
+ return [oset]
522
+ else: # unidentifiable, fall back to local IDA
523
+ amat = -amat.copy()
524
+ amat[np.logical_and(amat == 0, amat.T == -1)] = 1
525
+ # return get_locally_valid_parent_sets(amat, t1, t2)
526
+ return [[]] # Placeholder
527
+ else:
528
+ return None
529
+ elif project in ["mb_by_mb", "ldecc", "mb_by_mb_plus", "ldecc_plus"]:
530
+ adj_sets: dict = eval(h["adj_sets"])
531
+ return adj_sets.get((t1, t2), None)
532
+ elif project in ["ldp", "ldp_plus"]:
533
+ results: dict = eval(h["results"])
534
+ if (t1, t2) in results:
535
+ parts = results[(t1, t2)]
536
+ return [
537
+ parts["Z1"],
538
+ parts["Z1"] + parts["Z4"],
539
+ parts["Z1"] + parts["Z4"] + parts["Z5"],
540
+ ]
541
+ else:
542
+ return None
543
+ elif project == "local_optimal" or project.startswith("load"):
544
+ adj_sets: dict = eval(h["adj_sets"])
545
+ return adj_sets.get((t1, t2), None)
546
+ else:
547
+ raise ValueError("Invalid project")
548
+
549
+
550
+ def estimate_ates(
551
+ results,
552
+ algorithm: str,
553
+ samples: dict,
554
+ family: str = "gaussian",
555
+ ) -> dict:
556
+ ates = {}
557
+ for h in results:
558
+ exp_id = h["id"]
559
+ ates[exp_id] = {}
560
+ for t1, t2 in permutations(h["targets"], 2):
561
+ adj_sets = get_adj_sets(algorithm, h, t1, t2)
562
+ if adj_sets is None:
563
+ ates[exp_id][(t1, t2)] = [0.0]
564
+ else:
565
+ ates[exp_id][(t1, t2)] = estimate_ate(
566
+ treatment=t1,
567
+ outcome=t2,
568
+ adj_sets=adj_sets,
569
+ samples=samples[exp_id],
570
+ family=family,
571
+ )
572
+ return ates
573
+
574
+
575
+ def intervention_distance(
576
+ est_ates: dict, true_ates: dict, aggr: str = "abs"
577
+ ) -> np.ndarray:
578
+ """
579
+ Compute the intervention distance between estimated and true ATEs.
580
+
581
+ For each experiment and each treatment–outcome pair, the distance is the
582
+ mean absolute error (or mean squared error) across all candidate adjustment
583
+ sets returned by the algorithm. The per-experiment distance is the mean
584
+ over all target pairs.
585
+
586
+ Args:
587
+ est_ates (dict): Estimated ATEs as produced by :func:`estimate_ates`.
588
+ ``est_ates[exp_id][(t1, t2)]`` is a list of ATE estimates (one per
589
+ candidate adjustment set).
590
+ true_ates (dict): True ATEs as produced by :func:`true_causal_effects`.
591
+ aggr (str): Aggregation — ``"abs"`` (mean absolute error, default) or
592
+ ``"mse"`` (mean squared error).
593
+
594
+ Returns:
595
+ np.ndarray: 1-D array of per-experiment distances.
596
+ """
597
+ distances = []
598
+ for exp in est_ates:
599
+ exp_dist = []
600
+ for pair in est_ates[exp]:
601
+ true_ate = true_ates[exp][pair]
602
+ if aggr == "mse":
603
+ dist = [(true_ate - est_ate) ** 2 for est_ate in est_ates[exp][pair]]
604
+ elif aggr == "abs":
605
+ dist = [np.abs(true_ate - est_ate) for est_ate in est_ates[exp][pair]]
606
+ else:
607
+ raise ValueError("Invalid aggregation")
608
+ exp_dist.append(np.mean(dist))
609
+ distances.append(np.mean(exp_dist))
610
+ return np.array(distances)
611
+
612
+
613
+ def evaluate_intervention(algorithm: str, results: list, true_effects: dict, family: str = "gaussian") -> np.ndarray:
614
+ """
615
+ Evaluate the interventional performance of an algorithm by comparing its
616
+ estimated ATEs to the ground-truth ATEs.
617
+
618
+ Pipeline:
619
+
620
+ 1. Build a ``samples_dict`` mapping each experiment ID to its data matrix.
621
+ 2. Estimate ATEs from each result's discovered adjustment sets using
622
+ :func:`estimate_ates`.
623
+ 3. Compute the intervention distance (mean absolute error) via
624
+ :func:`intervention_distance`.
625
+
626
+ Args:
627
+ algorithm (str): Algorithm name used to parse adjustment sets from
628
+ each result dict (passed to :func:`get_adj_sets`).
629
+ results (list): List of result dicts, each containing at minimum
630
+ ``"id"``, ``"data"``, and the adjustment-set key appropriate for
631
+ ``algorithm``.
632
+ true_effects (dict): Ground-truth ATEs, keyed by experiment ID then
633
+ (treatment, outcome) pair. Produced by :func:`true_causal_effects`.
634
+ family (str): SEM family — ``"gaussian"`` (default) or ``"binary"``.
635
+
636
+ Returns:
637
+ np.ndarray: 1-D array of intervention distances (one per experiment).
638
+ """
639
+ samples_dict = {h["id"]: h["data"] for h in results}
640
+ est_ates = estimate_ates(results, algorithm, samples_dict, family=family)
641
+ distances = intervention_distance(est_ates, true_effects)
642
+ return distances