b-load 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- b_load-0.1.2.dist-info/METADATA +266 -0
- b_load-0.1.2.dist-info/RECORD +33 -0
- b_load-0.1.2.dist-info/WHEEL +5 -0
- b_load-0.1.2.dist-info/licenses/LICENSE +21 -0
- b_load-0.1.2.dist-info/top_level.txt +2 -0
- bload/__init__.py +10 -0
- bload/analysis.py +1 -0
- bload/background_knowledge.py +1 -0
- bload/counting_ci_tests.py +1 -0
- bload/data_generation.py +1 -0
- bload/evaluation.py +1 -0
- bload/load.py +1 -0
- bload/mb_by_mb.py +1 -0
- bload/pc.py +1 -0
- bload/real_data_applications/__init__.py +1 -0
- bload/real_data_applications/dream.py +1 -0
- bload/real_data_applications/sachs.py +1 -0
- bload/running.py +1 -0
- bload/utils.py +1 -0
- src/__init__.py +5 -0
- src/analysis.py +805 -0
- src/background_knowledge.py +283 -0
- src/counting_ci_tests.py +101 -0
- src/data_generation.py +185 -0
- src/evaluation.py +642 -0
- src/load.py +553 -0
- src/mb_by_mb.py +930 -0
- src/pc.py +341 -0
- src/real_data_applications/__init__.py +1 -0
- src/real_data_applications/dream.py +306 -0
- src/real_data_applications/sachs.py +202 -0
- src/running.py +1194 -0
- src/utils.py +106 -0
src/evaluation.py
ADDED
|
@@ -0,0 +1,642 @@
|
|
|
1
|
+
"""
|
|
2
|
+
evaluation.py — Ground-truth computation and algorithm evaluation metrics.
|
|
3
|
+
|
|
4
|
+
This module provides two groups of functionality:
|
|
5
|
+
|
|
6
|
+
1. **Ground-truth helpers**: convert DAGs to CPDAGs, enumerate true optimal
|
|
7
|
+
adjustment sets, and compute true average treatment effects (ATEs) under
|
|
8
|
+
linear Gaussian and binary SEMs.
|
|
9
|
+
|
|
10
|
+
2. **Evaluation metrics**: precision, recall, F1 and intervention distance for
|
|
11
|
+
comparing estimated adjustment sets or ATEs to ground truth.
|
|
12
|
+
|
|
13
|
+
These functions are shared by both the LOAD (base) and b-LOAD pipelines.
|
|
14
|
+
|
|
15
|
+
Notes
|
|
16
|
+
-----
|
|
17
|
+
Several functions call R via ``rpy2`` (specifically ``pcalg``'s
|
|
18
|
+
``dag2cpdag`` and ``optAdjSet``, and the binary ATE helper). The R
|
|
19
|
+
evaluation script ``evaluate.R`` is sourced automatically at module import.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from itertools import permutations
|
|
24
|
+
|
|
25
|
+
import numpy as np
|
|
26
|
+
import networkx as nx
|
|
27
|
+
from rpy2.robjects import default_converter, globalenv, numpy2ri, r
|
|
28
|
+
from rpy2 import robjects
|
|
29
|
+
|
|
30
|
+
# Source the R evaluation functions from the project root
|
|
31
|
+
file_path = Path(__file__).resolve().parent.parent / "evaluate.R"
|
|
32
|
+
|
|
33
|
+
# Source the file using the R 'source' function via robjects
|
|
34
|
+
if file_path.exists():
|
|
35
|
+
robjects.r.source(str(file_path))
|
|
36
|
+
print(f"Successfully sourced: {file_path}")
|
|
37
|
+
else:
|
|
38
|
+
print(f"Error: File not found at {file_path}")
|
|
39
|
+
raise FileNotFoundError(f"R evaluation script not found at {file_path}")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def dag2cpdag(dag: np.ndarray) -> np.ndarray:
|
|
43
|
+
"""
|
|
44
|
+
Convert a DAG adjacency matrix to its CPDAG (Markov equivalence class)
|
|
45
|
+
using the ``pcalg`` R package.
|
|
46
|
+
|
|
47
|
+
Args:
|
|
48
|
+
dag (np.ndarray): Binary DAG adjacency matrix (non-zero entry = edge).
|
|
49
|
+
|
|
50
|
+
Returns:
|
|
51
|
+
np.ndarray: CPDAG adjacency matrix as an ``int8`` numpy array.
|
|
52
|
+
``cpdag[i, j] == 1`` indicates an edge from i to j (directed or
|
|
53
|
+
undirected in the pcalg convention).
|
|
54
|
+
"""
|
|
55
|
+
with (default_converter + numpy2ri.converter).context():
|
|
56
|
+
cpdag = r["dag2cpdag"](r["as"](dag != 0, "graphNEL"))
|
|
57
|
+
cpdag = np.array(r["as"](cpdag, "matrix")).astype(np.int8)
|
|
58
|
+
return cpdag
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def get_test_nums(results: list) -> np.ndarray:
|
|
62
|
+
"""Return the total number of CI tests performed for each result dict."""
|
|
63
|
+
test_sum = [np.sum(h["tests"]) for h in results]
|
|
64
|
+
return np.array(test_sum)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def get_times(results: list) -> np.ndarray:
|
|
68
|
+
"""Return the wall-clock time (seconds) recorded for each result dict."""
|
|
69
|
+
return np.array([h["time"] for h in results])
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
# ADJ-SETS
|
|
73
|
+
def get_optimal_adj_set(amat: np.ndarray, treatment: int, outcome: int):
|
|
74
|
+
"""
|
|
75
|
+
Compute the optimal adjustment set (o-set) for the effect of ``treatment``
|
|
76
|
+
on ``outcome`` in the CPDAG / DAG represented by ``amat``, using
|
|
77
|
+
``pcalg``'s ``optAdjSet`` R function.
|
|
78
|
+
|
|
79
|
+
Returns ``None`` if ``treatment`` is not an ancestor of ``outcome`` in
|
|
80
|
+
``amat``, or if ``pcalg`` raises an exception (e.g. the effect is not
|
|
81
|
+
identifiable from the given graph).
|
|
82
|
+
|
|
83
|
+
Args:
|
|
84
|
+
amat (np.ndarray): CPDAG or DAG adjacency matrix. A non-zero entry
|
|
85
|
+
at ``[i, j]`` indicates an edge from i toward j (pcalg convention).
|
|
86
|
+
treatment (int): 0-based index of the treatment variable.
|
|
87
|
+
outcome (int): 0-based index of the outcome variable.
|
|
88
|
+
|
|
89
|
+
Returns:
|
|
90
|
+
Optional[set[int]]: The optimal adjustment set as a set of 0-based node
|
|
91
|
+
indices, or ``None`` if the effect is not identifiable.
|
|
92
|
+
"""
|
|
93
|
+
if not is_ancestor(treatment, outcome, amat):
|
|
94
|
+
return None
|
|
95
|
+
try:
|
|
96
|
+
with (default_converter + numpy2ri.converter).context():
|
|
97
|
+
S = r["optAdjSet"](
|
|
98
|
+
(r["as"](amat != 0, "graphNEL")), treatment + 1, outcome + 1
|
|
99
|
+
)
|
|
100
|
+
return set(np.array(S) - 1)
|
|
101
|
+
except Exception:
|
|
102
|
+
return None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def get_true_osets(experiments: dict) -> dict:
|
|
106
|
+
"""
|
|
107
|
+
Compute the true optimal adjustment sets for a dictionary of synthetic
|
|
108
|
+
experiments.
|
|
109
|
+
|
|
110
|
+
For each experiment, the true DAG is first converted to its CPDAG (Markov
|
|
111
|
+
equivalence class), and the optimal adjustment set is then retrieved from
|
|
112
|
+
that CPDAG via ``pcalg``'s ``optAdjSet``.
|
|
113
|
+
|
|
114
|
+
Args:
|
|
115
|
+
experiments (dict): Keyed by experiment ID. Each value must contain:
|
|
116
|
+
|
|
117
|
+
* ``"true_dag"`` (nx.DiGraph): Ground-truth DAG.
|
|
118
|
+
* ``"targets"`` (sequence): Pair of 0-based node indices
|
|
119
|
+
``[t1, t2]``; the ancestor direction is inferred from the DAG.
|
|
120
|
+
|
|
121
|
+
Returns:
|
|
122
|
+
dict: Keyed by experiment ID. Each value is a dict with:
|
|
123
|
+
|
|
124
|
+
* ``"treatment"`` (int): 0-based treatment index.
|
|
125
|
+
* ``"outcome"`` (int): 0-based outcome index.
|
|
126
|
+
* ``"oset"`` (Optional[set[int]]): True optimal adjustment set, or
|
|
127
|
+
``None`` if not identifiable from the CPDAG.
|
|
128
|
+
"""
|
|
129
|
+
true_osets = {}
|
|
130
|
+
|
|
131
|
+
for exp_id, exp in experiments.items():
|
|
132
|
+
# 1. Convert the true DAG to a CPDAG
|
|
133
|
+
# Using a helper (dag2cpdag) to represent the Markov Equivalence Class
|
|
134
|
+
true_dag_matrix = nx.to_numpy_array(exp["true_dag"])
|
|
135
|
+
cpdag = dag2cpdag(true_dag_matrix)
|
|
136
|
+
|
|
137
|
+
# 2. Identify treatment and outcome from the true DAG
|
|
138
|
+
# Based on explicit ancestral relations
|
|
139
|
+
targets = exp["targets"]
|
|
140
|
+
if nx.has_path(exp["true_dag"], targets[0], targets[1]):
|
|
141
|
+
treatment, outcome = targets[0], targets[1]
|
|
142
|
+
else:
|
|
143
|
+
treatment, outcome = targets[1], targets[0]
|
|
144
|
+
|
|
145
|
+
# 3. Retrieve the true optimal adjustment set relative to the CPDAG
|
|
146
|
+
true_oset = get_optimal_adj_set(cpdag, treatment, outcome)
|
|
147
|
+
|
|
148
|
+
# 4. Store the ground truth result
|
|
149
|
+
true_osets[exp_id] = {
|
|
150
|
+
"treatment": treatment,
|
|
151
|
+
"outcome": outcome,
|
|
152
|
+
"oset": true_oset,
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
# print(f"Processed Experiment {exp_id}: Treatment={treatment}, Outcome={outcome}")
|
|
156
|
+
|
|
157
|
+
return true_osets
|
|
158
|
+
|
|
159
|
+
def get_precision(true: set, pred: set) -> float:
|
|
160
|
+
"""
|
|
161
|
+
Compute precision of a predicted adjustment set against the ground truth.
|
|
162
|
+
|
|
163
|
+
Args:
|
|
164
|
+
true (set): True optimal adjustment set.
|
|
165
|
+
pred (set): Predicted adjustment set.
|
|
166
|
+
|
|
167
|
+
Returns:
|
|
168
|
+
float: Precision = TP / (TP + FP). Returns 1.0 if ``true == pred``,
|
|
169
|
+
0.0 if ``pred`` is empty and ``true`` is not.
|
|
170
|
+
"""
|
|
171
|
+
if true == pred:
|
|
172
|
+
return 1.0
|
|
173
|
+
tp = len(true.intersection(pred))
|
|
174
|
+
fp = len(pred - true)
|
|
175
|
+
return tp / (tp + fp) if (tp + fp) > 0 else 0
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def get_recall(true: set, pred: set) -> float:
|
|
179
|
+
"""
|
|
180
|
+
Compute recall of a predicted adjustment set against the ground truth.
|
|
181
|
+
|
|
182
|
+
Args:
|
|
183
|
+
true (set): True optimal adjustment set.
|
|
184
|
+
pred (set): Predicted adjustment set.
|
|
185
|
+
|
|
186
|
+
Returns:
|
|
187
|
+
float: Recall = TP / (TP + FN). Returns 1.0 if ``true == pred``.
|
|
188
|
+
"""
|
|
189
|
+
if true == pred:
|
|
190
|
+
return 1.0
|
|
191
|
+
tp = len(true.intersection(pred))
|
|
192
|
+
fn = len(true - pred)
|
|
193
|
+
return tp / (tp + fn) if (tp + fn) > 0 else 0
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def get_f1(precision: float, recall: float) -> float:
|
|
197
|
+
"""
|
|
198
|
+
Compute the F1 score from precision and recall.
|
|
199
|
+
|
|
200
|
+
Args:
|
|
201
|
+
precision (float): Precision score in [0, 1].
|
|
202
|
+
recall (float): Recall score in [0, 1].
|
|
203
|
+
|
|
204
|
+
Returns:
|
|
205
|
+
float: Harmonic mean of precision and recall; 0.0 if both are zero.
|
|
206
|
+
"""
|
|
207
|
+
if (precision + recall) > 0:
|
|
208
|
+
return 2 * (precision * recall) / (precision + recall)
|
|
209
|
+
else:
|
|
210
|
+
return 0
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def get_oset(project: str, h: dict, treatment: int, outcome: int):
|
|
214
|
+
if project in ["pc", "fges", "marvel", "snap"]:
|
|
215
|
+
est_oset = get_optimal_adj_set(np.array(h["amat"]), treatment, outcome)
|
|
216
|
+
if est_oset is not None:
|
|
217
|
+
est_osets = [est_oset]
|
|
218
|
+
else:
|
|
219
|
+
est_osets = None
|
|
220
|
+
elif project in ["mb_by_mb", "ldecc", "mb_by_mb_plus", "ldecc_plus"]:
|
|
221
|
+
adj_sets = eval(h["adj_sets"])
|
|
222
|
+
if (treatment, outcome) in adj_sets:
|
|
223
|
+
est_osets = adj_sets[(treatment, outcome)]
|
|
224
|
+
else:
|
|
225
|
+
est_osets = None
|
|
226
|
+
elif project in ["ldp", "ldp_plus"]:
|
|
227
|
+
results = eval(h["results"])
|
|
228
|
+
if (treatment, outcome) in results and results[(treatment, outcome)][
|
|
229
|
+
"vas_exists"
|
|
230
|
+
]:
|
|
231
|
+
parts = results[(treatment, outcome)]
|
|
232
|
+
est_osets = [
|
|
233
|
+
set(parts["Z1"]),
|
|
234
|
+
set(parts["Z1"] + parts["Z4"]),
|
|
235
|
+
set(parts["Z1"] + parts["Z4"] + parts["Z5"]),
|
|
236
|
+
]
|
|
237
|
+
else:
|
|
238
|
+
est_osets = None
|
|
239
|
+
elif project in ["load", "load_oracle"]:
|
|
240
|
+
adj_sets = eval(h["adj_sets"])
|
|
241
|
+
if h["identifiable"] and (treatment, outcome) in adj_sets:
|
|
242
|
+
est_osets = adj_sets[(treatment, outcome)]
|
|
243
|
+
else:
|
|
244
|
+
est_osets = None
|
|
245
|
+
else:
|
|
246
|
+
raise ValueError("Invalid project")
|
|
247
|
+
return est_osets
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def evaluate_oset(algorithm, results, true_osets: dict):
|
|
251
|
+
"""
|
|
252
|
+
Sequentially evaluates discovered adjustment sets against ground truth.
|
|
253
|
+
Returns trimmed arrays (removing outliers) for precision, recall, and F1.
|
|
254
|
+
"""
|
|
255
|
+
prec_scores = []
|
|
256
|
+
rec_scores = []
|
|
257
|
+
f1_scores = []
|
|
258
|
+
|
|
259
|
+
for h in results:
|
|
260
|
+
# 1. Handle algorithm failures
|
|
261
|
+
if "failed" in h and h["failed"]:
|
|
262
|
+
prec_scores.append(0.0)
|
|
263
|
+
rec_scores.append(0.0)
|
|
264
|
+
f1_scores.append(0.0)
|
|
265
|
+
continue
|
|
266
|
+
|
|
267
|
+
exp_id = h["id"]
|
|
268
|
+
treatment = true_osets[exp_id]["treatment"]
|
|
269
|
+
outcome = true_osets[exp_id]["outcome"]
|
|
270
|
+
|
|
271
|
+
# 2. Get estimated optimal adjustment sets
|
|
272
|
+
# In a sequential run, we call the function directly
|
|
273
|
+
est_osets = get_oset(algorithm, h, treatment, outcome)
|
|
274
|
+
true_oset = true_osets[exp_id]["oset"]
|
|
275
|
+
|
|
276
|
+
# 3. Calculate Scores (Precision, Recall, F1)
|
|
277
|
+
# Based on the paper's metrics for adjustment set quality
|
|
278
|
+
if true_oset is not None and est_osets is not None:
|
|
279
|
+
precs = [get_precision(true_oset, s) for s in est_osets]
|
|
280
|
+
recs = [get_recall(true_oset, s) for s in est_osets]
|
|
281
|
+
f1s = [get_f1(p, r) for p, r in zip(precs, recs)]
|
|
282
|
+
|
|
283
|
+
# Record the best match among returned sets
|
|
284
|
+
prec_scores.append(max(precs))
|
|
285
|
+
rec_scores.append(max(recs))
|
|
286
|
+
f1_scores.append(max(f1s))
|
|
287
|
+
|
|
288
|
+
elif true_oset == est_osets: # Both are None (case for non-existent Oset)
|
|
289
|
+
prec_scores.append(1.0)
|
|
290
|
+
rec_scores.append(1.0)
|
|
291
|
+
f1_scores.append(1.0)
|
|
292
|
+
else:
|
|
293
|
+
# Case where one exists and the other doesn't
|
|
294
|
+
prec_scores.append(0.0)
|
|
295
|
+
rec_scores.append(0.0)
|
|
296
|
+
f1_scores.append(0.0)
|
|
297
|
+
|
|
298
|
+
# 4. Return results
|
|
299
|
+
return (
|
|
300
|
+
(prec_scores),
|
|
301
|
+
(rec_scores),
|
|
302
|
+
(f1_scores),
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
# Modified functions to evaluate on real data
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def get_true_osets_real_data(experiments: dict) -> dict:
|
|
310
|
+
"""
|
|
311
|
+
Compute the true optimal adjustment sets for a dictionary of real-data
|
|
312
|
+
experiments.
|
|
313
|
+
|
|
314
|
+
Unlike :func:`get_true_osets`, this function queries the o-set directly
|
|
315
|
+
from the **true DAG matrix** (not from the CPDAG), which is appropriate
|
|
316
|
+
for real datasets where the ground-truth DAG is known (e.g. Sachs protein
|
|
317
|
+
network, DREAM benchmark) but no CPDAG conversion is available or needed.
|
|
318
|
+
|
|
319
|
+
Args:
|
|
320
|
+
experiments (dict): Keyed by experiment ID. Each value must contain:
|
|
321
|
+
|
|
322
|
+
* ``"true_dag"`` (nx.DiGraph): Ground-truth DAG with node indices
|
|
323
|
+
matching the columns of the data matrix.
|
|
324
|
+
* ``"targets"`` (sequence): Pair of 0-based node indices
|
|
325
|
+
``[t1, t2]``; the ancestor direction is inferred from the DAG.
|
|
326
|
+
|
|
327
|
+
Returns:
|
|
328
|
+
dict: Keyed by experiment ID. Each value is a dict with:
|
|
329
|
+
|
|
330
|
+
* ``"treatment"`` (int): 0-based treatment index.
|
|
331
|
+
* ``"outcome"`` (int): 0-based outcome index.
|
|
332
|
+
* ``"oset"`` (Optional[set[int]]): True optimal adjustment set from
|
|
333
|
+
the true DAG, or ``None`` if not identifiable.
|
|
334
|
+
"""
|
|
335
|
+
true_osets = {}
|
|
336
|
+
for exp_id, exp in experiments.items():
|
|
337
|
+
true_dag_matrix = nx.to_numpy_array(exp["true_dag"])
|
|
338
|
+
|
|
339
|
+
targets = exp["targets"]
|
|
340
|
+
if nx.has_path(exp["true_dag"], targets[0], targets[1]):
|
|
341
|
+
treatment, outcome = targets[0], targets[1]
|
|
342
|
+
else:
|
|
343
|
+
treatment, outcome = targets[1], targets[0]
|
|
344
|
+
|
|
345
|
+
true_oset = get_optimal_adj_set(true_dag_matrix, treatment, outcome)
|
|
346
|
+
|
|
347
|
+
true_osets[exp_id] = {
|
|
348
|
+
"treatment": treatment,
|
|
349
|
+
"outcome": outcome,
|
|
350
|
+
"oset": true_oset,
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
return true_osets
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
# INTERVENTION DISTANCE
|
|
357
|
+
def is_ancestor(t1: int, t2: int, amat: np.ndarray) -> bool:
|
|
358
|
+
"""
|
|
359
|
+
Check whether ``t1`` is an ancestor of ``t2`` in the directed graph
|
|
360
|
+
represented by ``amat``.
|
|
361
|
+
|
|
362
|
+
Uses matrix exponentiation on the boolean reachability matrix.
|
|
363
|
+
|
|
364
|
+
Args:
|
|
365
|
+
t1 (int): Candidate ancestor (0-based index).
|
|
366
|
+
t2 (int): Candidate descendant (0-based index).
|
|
367
|
+
amat (np.ndarray): Adjacency matrix where ``amat[i,j] != 0`` encodes
|
|
368
|
+
a directed edge i → j.
|
|
369
|
+
|
|
370
|
+
Returns:
|
|
371
|
+
bool: ``True`` if there exists a directed path t1 → … → t2.
|
|
372
|
+
"""
|
|
373
|
+
reach = amat.copy().astype(bool)
|
|
374
|
+
np.fill_diagonal(reach, True)
|
|
375
|
+
reach = np.linalg.matrix_power(reach, reach.shape[0] - 1)
|
|
376
|
+
return reach[t1, t2]
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def true_linear_gaussian_effect(
|
|
380
|
+
treatment: int, outcome: int, true_dag: nx.DiGraph, **kwargs
|
|
381
|
+
) -> float:
|
|
382
|
+
"""
|
|
383
|
+
Compute the true total causal effect of ``treatment`` on ``outcome`` in a
|
|
384
|
+
linear Gaussian SEM by summing path products.
|
|
385
|
+
|
|
386
|
+
The edge weights in ``true_dag`` are the linear SEM coefficients (set by
|
|
387
|
+
:func:`~src.data_generation.generate_data`). The total effect equals the
|
|
388
|
+
sum over all directed paths of the products of edge coefficients along each
|
|
389
|
+
path.
|
|
390
|
+
|
|
391
|
+
Args:
|
|
392
|
+
treatment (int): 0-based treatment index.
|
|
393
|
+
outcome (int): 0-based outcome index.
|
|
394
|
+
true_dag (nx.DiGraph): Weighted ground-truth DAG.
|
|
395
|
+
**kwargs: Silently ignored (for interface compatibility with
|
|
396
|
+
``true_binary_effect``).
|
|
397
|
+
|
|
398
|
+
Returns:
|
|
399
|
+
float: True total causal effect; 0.0 if ``treatment`` is not an
|
|
400
|
+
ancestor of ``outcome``.
|
|
401
|
+
"""
|
|
402
|
+
if treatment not in nx.ancestors(true_dag, outcome):
|
|
403
|
+
return 0.0
|
|
404
|
+
amat = nx.to_numpy_array(true_dag)
|
|
405
|
+
return sum(
|
|
406
|
+
np.prod([amat[path[i], path[i + 1]] for i in range(len(path) - 1)])
|
|
407
|
+
for path in nx.all_simple_paths(true_dag, treatment, outcome)
|
|
408
|
+
)
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def true_binary_effect(
|
|
412
|
+
treatment: int, outcome: int, true_dag: nx.DiGraph, cpt: object, **kwargs
|
|
413
|
+
) -> float:
|
|
414
|
+
"""
|
|
415
|
+
Compute the true total causal effect of ``treatment`` on ``outcome`` in a
|
|
416
|
+
discrete binary SEM, using the R helper ``true_binary_effect``.
|
|
417
|
+
|
|
418
|
+
Args:
|
|
419
|
+
treatment (int): 0-based treatment index.
|
|
420
|
+
outcome (int): 0-based outcome index.
|
|
421
|
+
true_dag (nx.DiGraph): Ground-truth DAG.
|
|
422
|
+
cpt: R conditional-probability table object produced by
|
|
423
|
+
:func:`~src.data_generation.generate_data`.
|
|
424
|
+
**kwargs: Silently ignored.
|
|
425
|
+
|
|
426
|
+
Returns:
|
|
427
|
+
float: True binary causal effect E[Y | do(X=1)] − E[Y | do(X=0)];
|
|
428
|
+
0.0 if ``treatment`` is not an ancestor of ``outcome``.
|
|
429
|
+
"""
|
|
430
|
+
if treatment not in nx.ancestors(true_dag, outcome):
|
|
431
|
+
return 0.0
|
|
432
|
+
with (default_converter + numpy2ri.converter).context():
|
|
433
|
+
return globalenv["true_binary_effect"](
|
|
434
|
+
int(treatment) + 1, int(outcome) + 1, nx.to_numpy_array(true_dag), cpt
|
|
435
|
+
)[0]
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def true_causal_effects(experiments: dict, family: str = "gaussian") -> dict:
|
|
439
|
+
"""
|
|
440
|
+
Compute the true causal effects for all target pairs across all experiments.
|
|
441
|
+
|
|
442
|
+
Args:
|
|
443
|
+
experiments (dict): Keyed by experiment ID. Each value must contain
|
|
444
|
+
``"targets"`` (pair of 0-based node indices), ``"true_dag"``
|
|
445
|
+
(``nx.DiGraph``), and ``"cpt"`` (R CPT object, only used for
|
|
446
|
+
``family="binary"``).
|
|
447
|
+
family (str): SEM family — ``"gaussian"`` (uses
|
|
448
|
+
:func:`true_linear_gaussian_effect`) or ``"binary"`` (uses
|
|
449
|
+
:func:`true_binary_effect`).
|
|
450
|
+
|
|
451
|
+
Returns:
|
|
452
|
+
dict: Nested dict ``effects[exp_id][(t1, t2)]`` = true total causal
|
|
453
|
+
effect of ``t1`` on ``t2``.
|
|
454
|
+
"""
|
|
455
|
+
if family == "gaussian":
|
|
456
|
+
get_effect = true_linear_gaussian_effect
|
|
457
|
+
elif family == "binary":
|
|
458
|
+
get_effect = true_binary_effect
|
|
459
|
+
else:
|
|
460
|
+
raise ValueError("Invalid family")
|
|
461
|
+
effects = {}
|
|
462
|
+
for exp_id, exp in experiments.items():
|
|
463
|
+
effects[exp_id] = {}
|
|
464
|
+
for t1, t2 in permutations(exp["targets"], 2):
|
|
465
|
+
effects[exp_id][(t1, t2)] = get_effect(
|
|
466
|
+
treatment=t1, outcome=t2, true_dag=exp["true_dag"], cpt=exp["cpt"]
|
|
467
|
+
)
|
|
468
|
+
return effects
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def estimate_binary(samples, val, outcome, treatment, adj_set):
|
|
472
|
+
# Filter rows based on the treatment value
|
|
473
|
+
samples_val = samples[samples[:, treatment] == val]
|
|
474
|
+
# Create the design matrix X
|
|
475
|
+
X = np.column_stack(
|
|
476
|
+
[np.ones(len(samples_val))] + [samples_val[:, idx] for idx in adj_set]
|
|
477
|
+
)
|
|
478
|
+
# Outcome variable (filtered by val)
|
|
479
|
+
y = samples_val[:, outcome]
|
|
480
|
+
# Solve the least squares problem X * beta = y
|
|
481
|
+
beta, _, _, _ = np.linalg.lstsq(X, y, rcond=None)
|
|
482
|
+
# Predict the outcome for all rows in the original samples using the estimated beta
|
|
483
|
+
X_full = np.column_stack(
|
|
484
|
+
[np.ones(len(samples))] + [samples[:, idx] for idx in adj_set]
|
|
485
|
+
)
|
|
486
|
+
y_pred = X_full @ beta
|
|
487
|
+
return y_pred.mean()
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def estimate_ate(
|
|
491
|
+
treatment: int,
|
|
492
|
+
outcome: int,
|
|
493
|
+
adj_sets: list[list[int]],
|
|
494
|
+
samples: np.ndarray,
|
|
495
|
+
family: str = "gaussian",
|
|
496
|
+
) -> list[float]:
|
|
497
|
+
effects = []
|
|
498
|
+
for adj_set in adj_sets:
|
|
499
|
+
adj_set = list(adj_set)
|
|
500
|
+
if family == "gaussian":
|
|
501
|
+
x = samples[:, [treatment] + adj_set]
|
|
502
|
+
y = samples[:, outcome]
|
|
503
|
+
A = np.vstack((x.T, np.ones(len(x))))
|
|
504
|
+
effect = np.linalg.lstsq(A.T, y, rcond=None)[0][0]
|
|
505
|
+
effects.append(effect)
|
|
506
|
+
elif family == "binary":
|
|
507
|
+
do_1 = estimate_binary(samples, 1, outcome, treatment, adj_set)
|
|
508
|
+
do_0 = estimate_binary(samples, 0, outcome, treatment, adj_set)
|
|
509
|
+
effects.append(do_1 - do_0)
|
|
510
|
+
return effects
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def get_adj_sets(project: str, h: dict, t1: int, t2: int):
|
|
514
|
+
if "failed" in h and h["failed"]:
|
|
515
|
+
return [[]]
|
|
516
|
+
if project in ["pc", "fges", "marvel", "snap"]:
|
|
517
|
+
amat = np.array(h["amat"], dtype=np.int8)
|
|
518
|
+
if is_ancestor(t1, t2, amat):
|
|
519
|
+
oset = get_optimal_adj_set(amat, t1, t2)
|
|
520
|
+
if oset != None: # identifiable
|
|
521
|
+
return [oset]
|
|
522
|
+
else: # unidentifiable, fall back to local IDA
|
|
523
|
+
amat = -amat.copy()
|
|
524
|
+
amat[np.logical_and(amat == 0, amat.T == -1)] = 1
|
|
525
|
+
# return get_locally_valid_parent_sets(amat, t1, t2)
|
|
526
|
+
return [[]] # Placeholder
|
|
527
|
+
else:
|
|
528
|
+
return None
|
|
529
|
+
elif project in ["mb_by_mb", "ldecc", "mb_by_mb_plus", "ldecc_plus"]:
|
|
530
|
+
adj_sets: dict = eval(h["adj_sets"])
|
|
531
|
+
return adj_sets.get((t1, t2), None)
|
|
532
|
+
elif project in ["ldp", "ldp_plus"]:
|
|
533
|
+
results: dict = eval(h["results"])
|
|
534
|
+
if (t1, t2) in results:
|
|
535
|
+
parts = results[(t1, t2)]
|
|
536
|
+
return [
|
|
537
|
+
parts["Z1"],
|
|
538
|
+
parts["Z1"] + parts["Z4"],
|
|
539
|
+
parts["Z1"] + parts["Z4"] + parts["Z5"],
|
|
540
|
+
]
|
|
541
|
+
else:
|
|
542
|
+
return None
|
|
543
|
+
elif project == "local_optimal" or project.startswith("load"):
|
|
544
|
+
adj_sets: dict = eval(h["adj_sets"])
|
|
545
|
+
return adj_sets.get((t1, t2), None)
|
|
546
|
+
else:
|
|
547
|
+
raise ValueError("Invalid project")
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def estimate_ates(
|
|
551
|
+
results,
|
|
552
|
+
algorithm: str,
|
|
553
|
+
samples: dict,
|
|
554
|
+
family: str = "gaussian",
|
|
555
|
+
) -> dict:
|
|
556
|
+
ates = {}
|
|
557
|
+
for h in results:
|
|
558
|
+
exp_id = h["id"]
|
|
559
|
+
ates[exp_id] = {}
|
|
560
|
+
for t1, t2 in permutations(h["targets"], 2):
|
|
561
|
+
adj_sets = get_adj_sets(algorithm, h, t1, t2)
|
|
562
|
+
if adj_sets is None:
|
|
563
|
+
ates[exp_id][(t1, t2)] = [0.0]
|
|
564
|
+
else:
|
|
565
|
+
ates[exp_id][(t1, t2)] = estimate_ate(
|
|
566
|
+
treatment=t1,
|
|
567
|
+
outcome=t2,
|
|
568
|
+
adj_sets=adj_sets,
|
|
569
|
+
samples=samples[exp_id],
|
|
570
|
+
family=family,
|
|
571
|
+
)
|
|
572
|
+
return ates
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def intervention_distance(
|
|
576
|
+
est_ates: dict, true_ates: dict, aggr: str = "abs"
|
|
577
|
+
) -> np.ndarray:
|
|
578
|
+
"""
|
|
579
|
+
Compute the intervention distance between estimated and true ATEs.
|
|
580
|
+
|
|
581
|
+
For each experiment and each treatment–outcome pair, the distance is the
|
|
582
|
+
mean absolute error (or mean squared error) across all candidate adjustment
|
|
583
|
+
sets returned by the algorithm. The per-experiment distance is the mean
|
|
584
|
+
over all target pairs.
|
|
585
|
+
|
|
586
|
+
Args:
|
|
587
|
+
est_ates (dict): Estimated ATEs as produced by :func:`estimate_ates`.
|
|
588
|
+
``est_ates[exp_id][(t1, t2)]`` is a list of ATE estimates (one per
|
|
589
|
+
candidate adjustment set).
|
|
590
|
+
true_ates (dict): True ATEs as produced by :func:`true_causal_effects`.
|
|
591
|
+
aggr (str): Aggregation — ``"abs"`` (mean absolute error, default) or
|
|
592
|
+
``"mse"`` (mean squared error).
|
|
593
|
+
|
|
594
|
+
Returns:
|
|
595
|
+
np.ndarray: 1-D array of per-experiment distances.
|
|
596
|
+
"""
|
|
597
|
+
distances = []
|
|
598
|
+
for exp in est_ates:
|
|
599
|
+
exp_dist = []
|
|
600
|
+
for pair in est_ates[exp]:
|
|
601
|
+
true_ate = true_ates[exp][pair]
|
|
602
|
+
if aggr == "mse":
|
|
603
|
+
dist = [(true_ate - est_ate) ** 2 for est_ate in est_ates[exp][pair]]
|
|
604
|
+
elif aggr == "abs":
|
|
605
|
+
dist = [np.abs(true_ate - est_ate) for est_ate in est_ates[exp][pair]]
|
|
606
|
+
else:
|
|
607
|
+
raise ValueError("Invalid aggregation")
|
|
608
|
+
exp_dist.append(np.mean(dist))
|
|
609
|
+
distances.append(np.mean(exp_dist))
|
|
610
|
+
return np.array(distances)
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
def evaluate_intervention(algorithm: str, results: list, true_effects: dict, family: str = "gaussian") -> np.ndarray:
|
|
614
|
+
"""
|
|
615
|
+
Evaluate the interventional performance of an algorithm by comparing its
|
|
616
|
+
estimated ATEs to the ground-truth ATEs.
|
|
617
|
+
|
|
618
|
+
Pipeline:
|
|
619
|
+
|
|
620
|
+
1. Build a ``samples_dict`` mapping each experiment ID to its data matrix.
|
|
621
|
+
2. Estimate ATEs from each result's discovered adjustment sets using
|
|
622
|
+
:func:`estimate_ates`.
|
|
623
|
+
3. Compute the intervention distance (mean absolute error) via
|
|
624
|
+
:func:`intervention_distance`.
|
|
625
|
+
|
|
626
|
+
Args:
|
|
627
|
+
algorithm (str): Algorithm name used to parse adjustment sets from
|
|
628
|
+
each result dict (passed to :func:`get_adj_sets`).
|
|
629
|
+
results (list): List of result dicts, each containing at minimum
|
|
630
|
+
``"id"``, ``"data"``, and the adjustment-set key appropriate for
|
|
631
|
+
``algorithm``.
|
|
632
|
+
true_effects (dict): Ground-truth ATEs, keyed by experiment ID then
|
|
633
|
+
(treatment, outcome) pair. Produced by :func:`true_causal_effects`.
|
|
634
|
+
family (str): SEM family — ``"gaussian"`` (default) or ``"binary"``.
|
|
635
|
+
|
|
636
|
+
Returns:
|
|
637
|
+
np.ndarray: 1-D array of intervention distances (one per experiment).
|
|
638
|
+
"""
|
|
639
|
+
samples_dict = {h["id"]: h["data"] for h in results}
|
|
640
|
+
est_ates = estimate_ates(results, algorithm, samples_dict, family=family)
|
|
641
|
+
distances = intervention_distance(est_ates, true_effects)
|
|
642
|
+
return distances
|