dfeols 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dfeols/__init__.py +34 -0
- dfeols/bench.py +543 -0
- dfeols/cli.py +314 -0
- dfeols/cluster.py +318 -0
- dfeols/demean.py +181 -0
- dfeols/estimate.py +536 -0
- dfeols/executors.py +141 -0
- dfeols/formula.py +190 -0
- dfeols/linalg.py +41 -0
- dfeols/lock.py +136 -0
- dfeols/prep.py +348 -0
- dfeols/prep_dist.py +849 -0
- dfeols/progress.py +263 -0
- dfeols/results.py +108 -0
- dfeols/site.py +209 -0
- dfeols/synth/__init__.py +209 -0
- dfeols/synth/__main__.py +5 -0
- dfeols/synth/influencer.py +250 -0
- dfeols/synth/panel.py +210 -0
- dfeols/validate.py +59 -0
- dfeols/vcov.py +40 -0
- dfeols/worker.py +426 -0
- dfeols-0.7.0.dist-info/METADATA +315 -0
- dfeols-0.7.0.dist-info/RECORD +28 -0
- dfeols-0.7.0.dist-info/WHEEL +5 -0
- dfeols-0.7.0.dist-info/entry_points.txt +2 -0
- dfeols-0.7.0.dist-info/licenses/LICENSE +674 -0
- dfeols-0.7.0.dist-info/top_level.txt +1 -0
dfeols/__init__.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
2
|
+
# Copyright (C) 2026 Ted Matherly
|
|
3
|
+
"""dfeols: sharded, elastic high-dimensional fixed-effects OLS for HPC clusters."""
|
|
4
|
+
|
|
5
|
+
import logging as _logging
|
|
6
|
+
|
|
7
|
+
from .cluster import executor
|
|
8
|
+
from .estimate import build_plan, fit
|
|
9
|
+
from .executors import DaskExecutor, SerialExecutor
|
|
10
|
+
from .lock import LockedError
|
|
11
|
+
from .prep import Dataset, prep
|
|
12
|
+
from .results import Result
|
|
13
|
+
from .site import profile
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"prep",
|
|
17
|
+
"fit",
|
|
18
|
+
"Dataset",
|
|
19
|
+
"Result",
|
|
20
|
+
"SerialExecutor",
|
|
21
|
+
"DaskExecutor",
|
|
22
|
+
"build_plan",
|
|
23
|
+
"executor",
|
|
24
|
+
"profile",
|
|
25
|
+
"LockedError",
|
|
26
|
+
]
|
|
27
|
+
__version__ = "0.7.0"
|
|
28
|
+
|
|
29
|
+
_log = _logging.getLogger("dfeols")
|
|
30
|
+
if not _log.handlers:
|
|
31
|
+
_h = _logging.StreamHandler()
|
|
32
|
+
_h.setFormatter(_logging.Formatter("%(asctime)s %(message)s", "%H:%M:%S"))
|
|
33
|
+
_log.addHandler(_h)
|
|
34
|
+
_log.setLevel(_logging.INFO)
|
dfeols/bench.py
ADDED
|
@@ -0,0 +1,543 @@
|
|
|
1
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
2
|
+
# Copyright (C) 2026 Ted Matherly
|
|
3
|
+
"""Benchmark dfeols on synthetic data with known parameters.
|
|
4
|
+
|
|
5
|
+
For each size it generates data (reusing what exists), preps one shard set per
|
|
6
|
+
partition key the specs need, fits each spec, and records:
|
|
7
|
+
|
|
8
|
+
timing generate / prep / round 1 / solve / round 2 / total seconds
|
|
9
|
+
throughput rows per wall-second and per worker-second (queue-independent)
|
|
10
|
+
workers average and max workers connected during the fit
|
|
11
|
+
memory peak worker RSS
|
|
12
|
+
validity max |z| against the truth, joint Wald p, and whether that is what
|
|
13
|
+
the spec should find (unbiased / biased / se_wrong)
|
|
14
|
+
ETA predicted vs actual finish at 10/25/50/75% progress
|
|
15
|
+
baseline pyfixest time and max coefficient difference (small sizes only)
|
|
16
|
+
|
|
17
|
+
Examples
|
|
18
|
+
python -m dfeols.bench --rows 1e6 1e7 # local, generic panel
|
|
19
|
+
python -m dfeols.bench --rows 1e7 1e8 --profile mycluster # elastic cluster workers
|
|
20
|
+
python -m dfeols.bench --dgp influencer --specs level shape --rows 1e8 --profile mycluster
|
|
21
|
+
python -m dfeols.bench --rows 1e8 --set T=520 --set n_groups=2000 --profile mycluster
|
|
22
|
+
python -m dfeols.bench --preset units10m_t100 --profile mycluster # 1e9 rows, 10M units, T=100
|
|
23
|
+
python -m dfeols.bench --preset units1m_t1000 --rows 1e8 --profile mycluster # 1e8 first
|
|
24
|
+
python -m dfeols.bench --preset units100m_t100 --profile mycluster # 1e10 stress test
|
|
25
|
+
python -m dfeols.bench --preset skewed_groups --profile mycluster # oversized shards
|
|
26
|
+
python -m dfeols.bench --preset units100m_t100 --rows 2e6 --profile local \
|
|
27
|
+
--prep-opt hash_levels=0 --prep-opt range_groups=0 --prep-opt max_files=0 # 1e10 code paths, small
|
|
28
|
+
dfeols bench --profile mycluster --rows 1e8 1e9
|
|
29
|
+
|
|
30
|
+
Writes results.csv, results.json and summary.md to --out (default <root>/results).
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import argparse
|
|
36
|
+
import json
|
|
37
|
+
import logging
|
|
38
|
+
import os
|
|
39
|
+
import shutil
|
|
40
|
+
import socket
|
|
41
|
+
import time
|
|
42
|
+
from pathlib import Path
|
|
43
|
+
|
|
44
|
+
import numpy as np
|
|
45
|
+
import polars as pl
|
|
46
|
+
|
|
47
|
+
import dfeols
|
|
48
|
+
|
|
49
|
+
from .formula import pyfixest_tol_kwargs, to_pyfixest
|
|
50
|
+
from .lock import DirLock
|
|
51
|
+
from .synth import generate, load_truth, plan_for_rows
|
|
52
|
+
from .validate import recovery
|
|
53
|
+
|
|
54
|
+
# Large-scale presets: billion-row unit x time panels with crossing time FEs.
|
|
55
|
+
# Balanced, so units = rows / T. Specs: unit + time partitioned by unit (time
|
|
56
|
+
# FE crosses every shard: sparse path) and unit + group^time partitioned by
|
|
57
|
+
# group (both FEs local: exact two-way solve).
|
|
58
|
+
PRESETS = {
|
|
59
|
+
"units10m_t100": {
|
|
60
|
+
"rows": [1e9],
|
|
61
|
+
"dgp": "panel",
|
|
62
|
+
"specs": ["twfe", "grouptime"],
|
|
63
|
+
"set": {"balanced": True, "T": 100, "n_groups": 500, "units_per_chunk": 20_000},
|
|
64
|
+
},
|
|
65
|
+
"units1m_t1000": {
|
|
66
|
+
"rows": [1e9],
|
|
67
|
+
"dgp": "panel",
|
|
68
|
+
"specs": ["twfe", "grouptime"],
|
|
69
|
+
"set": {"balanced": True, "T": 1000, "n_groups": 500, "units_per_chunk": 2_000},
|
|
70
|
+
},
|
|
71
|
+
# Stress tests.
|
|
72
|
+
# 1e10 rows, 100M units: hash ids for the unit FE, range-based shards by unit,
|
|
73
|
+
# a two-level shuffle when partitioning by group, 5,000 shards per set.
|
|
74
|
+
# Needs ~1 TB scratch (raw ~370 GB + two shard sets) and a 64 GB coordinator.
|
|
75
|
+
"units100m_t100": {
|
|
76
|
+
"rows": [1e10],
|
|
77
|
+
"dgp": "panel",
|
|
78
|
+
"specs": ["twfe", "grouptime"],
|
|
79
|
+
"gen_on_workers": True,
|
|
80
|
+
"set": {"balanced": True, "T": 100, "n_groups": 5000, "units_per_chunk": 20_000},
|
|
81
|
+
},
|
|
82
|
+
# Skewed group sizes (Zipf 0.7 over 5,000 groups): the largest group holds
|
|
83
|
+
# ~2.5% of rows (25M at 1e9) and becomes one oversized shard when
|
|
84
|
+
# partitioning by group; tests uneven shards, worker memory and the ETA floor.
|
|
85
|
+
"skewed_groups": {
|
|
86
|
+
"rows": [1e9],
|
|
87
|
+
"dgp": "panel",
|
|
88
|
+
"specs": ["twfe", "grouptime"],
|
|
89
|
+
"set": {"balanced": True, "T": 100, "n_groups": 5000, "group_zipf": 0.7, "units_per_chunk": 20_000},
|
|
90
|
+
},
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _available_cores() -> int:
|
|
95
|
+
"""Cores this process may use (the job's allocation, not the whole node)."""
|
|
96
|
+
try:
|
|
97
|
+
return max(len(os.sched_getaffinity(0)), 1)
|
|
98
|
+
except AttributeError: # macOS
|
|
99
|
+
return os.cpu_count() or 1
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def eta_and_workers(workdir: Path) -> dict:
|
|
103
|
+
p = workdir / "history.jsonl"
|
|
104
|
+
if not p.exists():
|
|
105
|
+
return {}
|
|
106
|
+
h = [json.loads(l) for l in p.read_text().splitlines() if l.strip()]
|
|
107
|
+
if not h:
|
|
108
|
+
return {}
|
|
109
|
+
actual = h[-1]["t"]
|
|
110
|
+
w = [x["workers"] for x in h]
|
|
111
|
+
out = {"workers_avg": round(float(np.mean(w)), 1), "workers_max": int(max(w))}
|
|
112
|
+
for q in (0.10, 0.25, 0.50, 0.75):
|
|
113
|
+
pts = [x for x in h if x["frac_done"] >= q and x["eta_total"] is not None]
|
|
114
|
+
if pts and actual > 0:
|
|
115
|
+
x = pts[0]
|
|
116
|
+
out[f"eta_err_{int(q * 100)}"] = round((x["t"] + x["eta_total"] - actual) / actual, 3)
|
|
117
|
+
return out
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def pyfixest_baseline(raw: Path, spec: dict, res) -> dict:
|
|
121
|
+
import pyfixest as pf
|
|
122
|
+
|
|
123
|
+
pdf = pl.read_parquet(str(raw / "*.parquet")).to_pandas()
|
|
124
|
+
t = time.perf_counter()
|
|
125
|
+
m = pf.feols(
|
|
126
|
+
to_pyfixest(spec["formula"]), data=pdf, vcov={"CRV1": spec["cluster"]}, **pyfixest_tol_kwargs(1e-10)
|
|
127
|
+
)
|
|
128
|
+
secs = time.perf_counter() - t
|
|
129
|
+
tab = m.tidy().loc[res.coefnames]
|
|
130
|
+
return {
|
|
131
|
+
"pyfixest_s": round(secs, 1),
|
|
132
|
+
"pyfixest_max_coef_diff": float(np.max(np.abs(tab["Estimate"].to_numpy() - res.coef()))),
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
# ---------------------------------------------------------------- layout
|
|
137
|
+
#
|
|
138
|
+
# <root>/raw_<tag>/ generated data (reused across runs)
|
|
139
|
+
# <root>/shards_<tag>_by_<key>_<n>/ prepared shards (reused across runs)
|
|
140
|
+
# <root>/runs/<run_id>/<tag>/<spec>/ one fit: result.json, bench.json,
|
|
141
|
+
# status.json, history.jsonl (never overwritten)
|
|
142
|
+
# <root>/results/results.csv every fit ever run here, rebuilt from runs/
|
|
143
|
+
# <root>/results/<run_id>.md summary of one benchmark invocation
|
|
144
|
+
# <root>/results/latest.md copy of the most recent summary
|
|
145
|
+
#
|
|
146
|
+
# `dfeols results [root]` prints the table; nothing is overwritten, so earlier
|
|
147
|
+
# results stay where they were written.
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _param_suffix(dgp: str, kw: dict | None) -> str:
|
|
151
|
+
"""Data folders are named by DGP, size and any non-default parameters, so
|
|
152
|
+
different configurations never collide (e.g. T=100 vs T=1000 at 1e9)."""
|
|
153
|
+
if not kw:
|
|
154
|
+
return ""
|
|
155
|
+
keep = {k: v for k, v in kw.items() if k not in ("units_per_chunk", "accounts_per_chunk")}
|
|
156
|
+
if not keep:
|
|
157
|
+
return ""
|
|
158
|
+
short = {"balanced": "bal", "n_groups": "G", "n_units": "U"}
|
|
159
|
+
parts = []
|
|
160
|
+
for k in sorted(keep):
|
|
161
|
+
v = keep[k]
|
|
162
|
+
v = ("1" if v else "0") if isinstance(v, bool) else str(v).replace(" ", "")
|
|
163
|
+
parts.append(f"{short.get(k, k)}{v}")
|
|
164
|
+
return "_" + "_".join(parts)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def make_run_id() -> str:
|
|
168
|
+
job = next(
|
|
169
|
+
(os.environ[v] for v in ("SLURM_JOB_ID", "PBS_JOBID", "JOB_ID", "LSB_JOBID") if os.environ.get(v)),
|
|
170
|
+
None,
|
|
171
|
+
)
|
|
172
|
+
rid = time.strftime("%Y%m%d-%H%M%S")
|
|
173
|
+
return f"{rid}-{job.split('.')[0]}" if job else rid
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _verdict(expect: str, r: dict) -> str:
|
|
177
|
+
if expect == "unbiased":
|
|
178
|
+
return "ok" if (r["ok"] and r["p"] > 0.001) else "FAIL"
|
|
179
|
+
if expect == "biased":
|
|
180
|
+
return "ok" if r["p"] < 1e-4 else "FAIL"
|
|
181
|
+
return "n/a"
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _row_from_result(res, truth, spec_name, *, n_rows=None, n_shards=None, wd=None) -> dict:
|
|
185
|
+
s = truth["specs"][spec_name]
|
|
186
|
+
tm = res.timing
|
|
187
|
+
r = recovery(res, truth)
|
|
188
|
+
return {
|
|
189
|
+
"dgp": truth.get("kind", "?"),
|
|
190
|
+
"rows": n_rows or res.N + res.N_dropped,
|
|
191
|
+
"N": res.N,
|
|
192
|
+
"spec": spec_name,
|
|
193
|
+
"expect": s["expect"],
|
|
194
|
+
"partition": s["partition"],
|
|
195
|
+
"shards": n_shards or tm.get("n_shards"),
|
|
196
|
+
"clusters": res.G,
|
|
197
|
+
"round1_s": round(tm["round1_seconds"], 1),
|
|
198
|
+
"round2_s": round(tm["round2_seconds"], 1),
|
|
199
|
+
"fit_s": round(res.seconds, 1),
|
|
200
|
+
"rows_per_s": int(res.N / max(res.seconds, 1e-9)),
|
|
201
|
+
"rows_per_worker_s": int(res.N / max(tm["worker_seconds"], 1e-9)),
|
|
202
|
+
"peak_worker_rss_gb": round(tm["max_worker_rss_gb"], 2),
|
|
203
|
+
"max_abs_z": round(r["max_abs_z"], 2),
|
|
204
|
+
"wald_p": round(r["p"], 4),
|
|
205
|
+
"validity": _verdict(s["expect"], r),
|
|
206
|
+
**(eta_and_workers(Path(wd)) if wd else {}),
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def collect(root) -> list[dict]:
|
|
211
|
+
"""Every benchmark fit under <root>/runs, newest first. Rows come from each
|
|
212
|
+
fit's bench.json; fits from before bench.json existed (runs/<tag>/<spec>/)
|
|
213
|
+
are rebuilt from result.json and the dataset's truth.json."""
|
|
214
|
+
from .results import Result
|
|
215
|
+
|
|
216
|
+
root = Path(root)
|
|
217
|
+
rows = []
|
|
218
|
+
for bj in root.glob("runs/*/*/*/bench.json"):
|
|
219
|
+
try:
|
|
220
|
+
rows.append(json.loads(bj.read_text()))
|
|
221
|
+
except Exception:
|
|
222
|
+
pass
|
|
223
|
+
for rj in root.glob("runs/*/*/result.json"): # legacy layout
|
|
224
|
+
tag, spec = rj.parent.parent.name, rj.parent.name
|
|
225
|
+
tp = root / f"raw_{tag}" / "truth.json"
|
|
226
|
+
if not tp.exists():
|
|
227
|
+
continue
|
|
228
|
+
try:
|
|
229
|
+
truth = json.loads(tp.read_text())
|
|
230
|
+
res = Result.load(rj)
|
|
231
|
+
row = _row_from_result(res, truth, spec, wd=rj.parent)
|
|
232
|
+
except Exception:
|
|
233
|
+
continue
|
|
234
|
+
ts = time.strftime("%Y%m%d-%H%M%S", time.localtime(rj.stat().st_mtime))
|
|
235
|
+
rows.append(
|
|
236
|
+
{
|
|
237
|
+
"run_id": f"{ts}-legacy",
|
|
238
|
+
"tag": tag,
|
|
239
|
+
"finished": time.strftime("%Y-%m-%d %H:%M", time.localtime(rj.stat().st_mtime)),
|
|
240
|
+
"workdir": str(rj.parent),
|
|
241
|
+
**row,
|
|
242
|
+
}
|
|
243
|
+
)
|
|
244
|
+
rows.sort(key=lambda r: (r.get("finished", ""), r.get("run_id", "")), reverse=True)
|
|
245
|
+
return rows
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def write_ledger(root) -> list[dict]:
|
|
249
|
+
root = Path(root)
|
|
250
|
+
out = root / "results"
|
|
251
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
252
|
+
rows = collect(root)
|
|
253
|
+
if rows:
|
|
254
|
+
cols = list(dict.fromkeys(k for r in rows for k in r))
|
|
255
|
+
pl.DataFrame([{c: r.get(c) for c in cols} for r in rows], infer_schema_length=None).write_csv(
|
|
256
|
+
out / "results.csv"
|
|
257
|
+
)
|
|
258
|
+
(out / "results.json").write_text(json.dumps(rows, indent=1))
|
|
259
|
+
return rows
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def stale_dataset(ds, target_rows) -> str:
|
|
263
|
+
"""Why an existing benchmark shard folder must be rebuilt ("" to reuse it)."""
|
|
264
|
+
m = ds.meta
|
|
265
|
+
if sum(s["rows"] for s in ds.shards) != ds.n_rows:
|
|
266
|
+
return "shard row counts don't add up"
|
|
267
|
+
if m.get("shard_assignment") == "range" and not m.get("prep_version"):
|
|
268
|
+
# range shards before 0.6.2 used UInt32 row counts that overflowed
|
|
269
|
+
return "built by a version with the range-shard overflow bug (before 0.6.2)"
|
|
270
|
+
biggest = max(s["rows"] for s in ds.shards)
|
|
271
|
+
if biggest > 10 * target_rows and len(ds.shards) < ds.n_rows / (10 * target_rows):
|
|
272
|
+
return f"shards are far from the target size (largest {biggest:,} rows)"
|
|
273
|
+
return ""
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def run(
|
|
277
|
+
rows_list,
|
|
278
|
+
*,
|
|
279
|
+
root,
|
|
280
|
+
out=None,
|
|
281
|
+
dgp="panel",
|
|
282
|
+
dgp_kw=None,
|
|
283
|
+
specs=None,
|
|
284
|
+
profile=None,
|
|
285
|
+
target_rows=2e6,
|
|
286
|
+
gen_jobs=1,
|
|
287
|
+
pyfixest_max_rows=0,
|
|
288
|
+
keep_data=True,
|
|
289
|
+
gen_on_workers=False,
|
|
290
|
+
prep_opts=None,
|
|
291
|
+
log=print,
|
|
292
|
+
):
|
|
293
|
+
root = Path(root)
|
|
294
|
+
out = Path(out) if out else root / "results"
|
|
295
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
296
|
+
run_id = make_run_id()
|
|
297
|
+
summary = out / f"{run_id}.md"
|
|
298
|
+
log(
|
|
299
|
+
f"benchmark run {run_id}\n this run's summary: {summary}\n all results: "
|
|
300
|
+
f"{out / 'results.csv'} (or: dfeols results {root})"
|
|
301
|
+
)
|
|
302
|
+
ex = dfeols.executor(profile) if profile else dfeols.SerialExecutor()
|
|
303
|
+
host = socket.gethostname()
|
|
304
|
+
results = []
|
|
305
|
+
try:
|
|
306
|
+
for rows in rows_list:
|
|
307
|
+
tag = f"{dgp}_{rows:.0e}".replace("+", "") + _param_suffix(dgp, dgp_kw)
|
|
308
|
+
raw = root / f"raw_{tag}"
|
|
309
|
+
d = plan_for_rows(rows, dgp, **(dgp_kw or {}))
|
|
310
|
+
# One job at a time per data set: a second job would regenerate or
|
|
311
|
+
# re-prep (deleting) the folders the first one is still writing.
|
|
312
|
+
# Held for generation and prep only; fits only read the shards.
|
|
313
|
+
with DirLock(root / "locks" / tag, what=f"benchmark data set {tag}"):
|
|
314
|
+
t = time.perf_counter()
|
|
315
|
+
# generate on the coordinator by default: it runs while the first
|
|
316
|
+
# worker jobs wait in the queue (on workers it would wait for them)
|
|
317
|
+
generate(
|
|
318
|
+
raw, d, jobs=gen_jobs, executor=ex if (profile and gen_on_workers) else None, verbose=True
|
|
319
|
+
)
|
|
320
|
+
t_gen = time.perf_counter() - t
|
|
321
|
+
truth = load_truth(raw)
|
|
322
|
+
todo = specs or list(truth["specs"])
|
|
323
|
+
missing = set(todo) - set(truth["specs"])
|
|
324
|
+
if missing:
|
|
325
|
+
raise KeyError(f"{dgp} DGP has no specs {sorted(missing)}; have {sorted(truth['specs'])}")
|
|
326
|
+
|
|
327
|
+
datasets, t_prep = {}, {}
|
|
328
|
+
for part in sorted({truth["specs"][s]["partition"] for s in todo}):
|
|
329
|
+
sd = root / f"shards_{tag}_by_{part}_{int(target_rows)}"
|
|
330
|
+
t = time.perf_counter()
|
|
331
|
+
reuse = (sd / "dfeols_meta.json").exists()
|
|
332
|
+
if reuse:
|
|
333
|
+
why = stale_dataset(dfeols.Dataset.open(sd), target_rows)
|
|
334
|
+
if why:
|
|
335
|
+
log(f"re-preparing {sd.name}: {why}")
|
|
336
|
+
reuse = False
|
|
337
|
+
if reuse:
|
|
338
|
+
datasets[part] = dfeols.Dataset.open(sd)
|
|
339
|
+
t_prep[part] = float("nan")
|
|
340
|
+
else:
|
|
341
|
+
datasets[part] = dfeols.prep(
|
|
342
|
+
str(raw / "*.parquet"),
|
|
343
|
+
shard_dir=sd,
|
|
344
|
+
partition=part,
|
|
345
|
+
target_rows=int(target_rows),
|
|
346
|
+
overwrite=True,
|
|
347
|
+
executor=ex if profile else None,
|
|
348
|
+
**truth["prep"],
|
|
349
|
+
**((prep_opts or {}) if profile else {}),
|
|
350
|
+
)
|
|
351
|
+
t_prep[part] = time.perf_counter() - t
|
|
352
|
+
n_rows = next(iter(datasets.values())).n_rows
|
|
353
|
+
log(
|
|
354
|
+
f"\n== {dgp}: {n_rows:,} rows, {d.n_units:,} units (generate {t_gen:.0f}s, prep "
|
|
355
|
+
+ ", ".join(f"by {k} {v:.0f}s" for k, v in t_prep.items())
|
|
356
|
+
+ ")"
|
|
357
|
+
)
|
|
358
|
+
|
|
359
|
+
for name in todo:
|
|
360
|
+
s = truth["specs"][name]
|
|
361
|
+
ds = datasets[s["partition"]]
|
|
362
|
+
wd = root / "runs" / run_id / tag / name
|
|
363
|
+
res = dfeols.fit(
|
|
364
|
+
ds,
|
|
365
|
+
s["formula"],
|
|
366
|
+
cluster=s["cluster"],
|
|
367
|
+
executor=ex,
|
|
368
|
+
workdir=wd,
|
|
369
|
+
label=f"{tag}-{name}",
|
|
370
|
+
log_every=15,
|
|
371
|
+
)
|
|
372
|
+
row = {
|
|
373
|
+
"run_id": run_id,
|
|
374
|
+
"tag": tag,
|
|
375
|
+
"finished": time.strftime("%Y-%m-%d %H:%M"),
|
|
376
|
+
"profile": profile or "serial",
|
|
377
|
+
"coordinator": host,
|
|
378
|
+
**_row_from_result(res, truth, name, n_rows=ds.n_rows, n_shards=len(ds.shards), wd=wd),
|
|
379
|
+
"generate_s": round(t_gen, 1),
|
|
380
|
+
"prep_s": round(t_prep[s["partition"]], 1),
|
|
381
|
+
"workdir": str(wd),
|
|
382
|
+
}
|
|
383
|
+
if pyfixest_max_rows and ds.n_rows <= pyfixest_max_rows:
|
|
384
|
+
row.update(pyfixest_baseline(raw, s, res))
|
|
385
|
+
(wd / "bench.json").write_text(json.dumps(row, indent=1))
|
|
386
|
+
results.append(row)
|
|
387
|
+
log(json.dumps(row))
|
|
388
|
+
# after every fit, so partial runs are visible
|
|
389
|
+
write_ledger(root)
|
|
390
|
+
write_summary(results, summary, profile or "serial", host, target_rows, run_id)
|
|
391
|
+
shutil.copyfile(summary, out / "latest.md")
|
|
392
|
+
if not keep_data:
|
|
393
|
+
for sd in {Path(ds.root) for ds in datasets.values()}:
|
|
394
|
+
shutil.rmtree(sd, ignore_errors=True)
|
|
395
|
+
finally:
|
|
396
|
+
if hasattr(ex, "client"):
|
|
397
|
+
ex.client.close()
|
|
398
|
+
ex.cluster.close()
|
|
399
|
+
if results:
|
|
400
|
+
write_ledger(root)
|
|
401
|
+
write_summary(results, summary, profile or "serial", host, target_rows, run_id)
|
|
402
|
+
shutil.copyfile(summary, out / "latest.md")
|
|
403
|
+
return results
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def write_summary(rows, path: Path, where: str, host: str, target_rows, run_id: str = ""):
|
|
407
|
+
cols = [
|
|
408
|
+
"dgp",
|
|
409
|
+
"rows",
|
|
410
|
+
"spec",
|
|
411
|
+
"expect",
|
|
412
|
+
"validity",
|
|
413
|
+
"shards",
|
|
414
|
+
"clusters",
|
|
415
|
+
"workers_avg",
|
|
416
|
+
"fit_s",
|
|
417
|
+
"round1_s",
|
|
418
|
+
"round2_s",
|
|
419
|
+
"rows_per_worker_s",
|
|
420
|
+
"rows_per_s",
|
|
421
|
+
"peak_worker_rss_gb",
|
|
422
|
+
"max_abs_z",
|
|
423
|
+
"wald_p",
|
|
424
|
+
"eta_err_25",
|
|
425
|
+
"eta_err_50",
|
|
426
|
+
"pyfixest_s",
|
|
427
|
+
"pyfixest_max_coef_diff",
|
|
428
|
+
]
|
|
429
|
+
cols = [c for c in cols if any(c in r for r in rows)]
|
|
430
|
+
lines = [
|
|
431
|
+
f"# dfeols benchmark {run_id} ({where}, coordinator {host}, {time.strftime('%Y-%m-%d %H:%M')})",
|
|
432
|
+
"",
|
|
433
|
+
f"Shard target {int(target_rows):,} rows. Fit directories: runs/{run_id}/.",
|
|
434
|
+
"",
|
|
435
|
+
"| " + " | ".join(cols) + " |",
|
|
436
|
+
"|" + "---|" * len(cols),
|
|
437
|
+
]
|
|
438
|
+
for r in rows:
|
|
439
|
+
vals = []
|
|
440
|
+
for c in cols:
|
|
441
|
+
v = r.get(c, "")
|
|
442
|
+
vals.append(
|
|
443
|
+
f"{v:,}"
|
|
444
|
+
if isinstance(v, int) and not isinstance(v, bool)
|
|
445
|
+
else (f"{v:.1e}" if c.endswith("diff") and v != "" else str(v))
|
|
446
|
+
)
|
|
447
|
+
lines.append("| " + " | ".join(vals) + " |")
|
|
448
|
+
lines += [
|
|
449
|
+
"",
|
|
450
|
+
"validity: ok if an 'unbiased' spec recovers the truth (every |z| under the "
|
|
451
|
+
"Bonferroni bound and Wald p > 0.001) or a 'biased' spec is rejected (p < 1e-4).",
|
|
452
|
+
"rows_per_worker_s is throughput per worker, independent of how many workers the "
|
|
453
|
+
"queue provided; workers_avg shows how many there were.",
|
|
454
|
+
"eta_err_q = (predicted - actual) / actual finish time when q% of rows were done.",
|
|
455
|
+
]
|
|
456
|
+
path.write_text("\n".join(lines) + "\n")
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def main(argv=None):
|
|
460
|
+
ap = argparse.ArgumentParser(
|
|
461
|
+
prog="python -m dfeols.bench",
|
|
462
|
+
description=__doc__,
|
|
463
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
464
|
+
)
|
|
465
|
+
ap.add_argument(
|
|
466
|
+
"--preset",
|
|
467
|
+
choices=sorted(PRESETS),
|
|
468
|
+
default=None,
|
|
469
|
+
help="large-scale configuration; --rows/--set/--specs still override",
|
|
470
|
+
)
|
|
471
|
+
ap.add_argument("--rows", type=float, nargs="+", default=None, help="default 1e6 1e7")
|
|
472
|
+
ap.add_argument("--dgp", default="panel")
|
|
473
|
+
ap.add_argument("--set", action="append", default=[], metavar="KEY=VALUE", help="DGP parameter")
|
|
474
|
+
ap.add_argument("--specs", nargs="+", default=None, help="spec names (default: all of the DGP's)")
|
|
475
|
+
ap.add_argument(
|
|
476
|
+
"--profile",
|
|
477
|
+
default=os.environ.get("DFEOLS_PROFILE"),
|
|
478
|
+
help="site profile for workers (default: the profile of the job this runs in, "
|
|
479
|
+
"set by `dfeols submit`/`dfeols bench`; serial when run directly)",
|
|
480
|
+
)
|
|
481
|
+
ap.add_argument("--root", default=None, help="data/shards/runs (default: <profile scratch>/bench)")
|
|
482
|
+
ap.add_argument("--out", default=None)
|
|
483
|
+
ap.add_argument("--target-rows", type=float, default=2e6, help="rows per shard")
|
|
484
|
+
ap.add_argument(
|
|
485
|
+
"--gen-jobs",
|
|
486
|
+
type=int,
|
|
487
|
+
default=_available_cores(),
|
|
488
|
+
help="local processes for data generation (default: cores allocated to this job)",
|
|
489
|
+
)
|
|
490
|
+
ap.add_argument(
|
|
491
|
+
"--gen-on-workers", action="store_true", help="generate on cluster workers instead of the coordinator"
|
|
492
|
+
)
|
|
493
|
+
ap.add_argument("--pyfixest-max-rows", type=float, default=0)
|
|
494
|
+
ap.add_argument("--drop-data", action="store_true", help="delete shards after each size")
|
|
495
|
+
ap.add_argument(
|
|
496
|
+
"--prep-opt",
|
|
497
|
+
action="append",
|
|
498
|
+
default=[],
|
|
499
|
+
metavar="KEY=VALUE",
|
|
500
|
+
help="distributed prep option, e.g. --prep-opt max_files=0 (forces the two-level "
|
|
501
|
+
"shuffle); see dfeols.prep_dist.prep_distributed",
|
|
502
|
+
)
|
|
503
|
+
a = ap.parse_args(argv)
|
|
504
|
+
logging.getLogger("dfeols").setLevel(logging.INFO)
|
|
505
|
+
pre = PRESETS.get(a.preset, {})
|
|
506
|
+
if pre:
|
|
507
|
+
a.dgp = pre["dgp"] if a.dgp == "panel" else a.dgp
|
|
508
|
+
a.rows = a.rows or pre["rows"]
|
|
509
|
+
a.specs = a.specs or pre["specs"]
|
|
510
|
+
a.gen_on_workers = a.gen_on_workers or pre.get("gen_on_workers", False)
|
|
511
|
+
a.rows = a.rows or [1e6, 1e7]
|
|
512
|
+
kw = dict(pre.get("set", {}))
|
|
513
|
+
for s in a.set:
|
|
514
|
+
k, v = s.split("=", 1)
|
|
515
|
+
try:
|
|
516
|
+
kw[k] = json.loads(v)
|
|
517
|
+
except json.JSONDecodeError:
|
|
518
|
+
kw[k] = v
|
|
519
|
+
if "theta" in kw:
|
|
520
|
+
kw["theta"] = tuple(kw["theta"])
|
|
521
|
+
root = a.root or str(dfeols.profile(a.profile).scratch_dir("bench"))
|
|
522
|
+
run(
|
|
523
|
+
a.rows,
|
|
524
|
+
root=root,
|
|
525
|
+
out=a.out,
|
|
526
|
+
dgp=a.dgp,
|
|
527
|
+
dgp_kw=kw,
|
|
528
|
+
specs=a.specs,
|
|
529
|
+
profile=a.profile,
|
|
530
|
+
target_rows=a.target_rows,
|
|
531
|
+
gen_jobs=a.gen_jobs,
|
|
532
|
+
pyfixest_max_rows=a.pyfixest_max_rows,
|
|
533
|
+
keep_data=not a.drop_data,
|
|
534
|
+
gen_on_workers=a.gen_on_workers,
|
|
535
|
+
prep_opts={k: json.loads(v) for k, v in (o.split("=", 1) for o in a.prep_opt)},
|
|
536
|
+
)
|
|
537
|
+
latest = Path(a.out or Path(root) / "results") / "latest.md"
|
|
538
|
+
if latest.exists():
|
|
539
|
+
print(latest.read_text())
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
if __name__ == "__main__":
|
|
543
|
+
main()
|