dfeols 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dfeols/__init__.py ADDED
@@ -0,0 +1,34 @@
1
+ # SPDX-License-Identifier: GPL-3.0-or-later
2
+ # Copyright (C) 2026 Ted Matherly
3
+ """dfeols: sharded, elastic high-dimensional fixed-effects OLS for HPC clusters."""
4
+
5
+ import logging as _logging
6
+
7
+ from .cluster import executor
8
+ from .estimate import build_plan, fit
9
+ from .executors import DaskExecutor, SerialExecutor
10
+ from .lock import LockedError
11
+ from .prep import Dataset, prep
12
+ from .results import Result
13
+ from .site import profile
14
+
15
+ __all__ = [
16
+ "prep",
17
+ "fit",
18
+ "Dataset",
19
+ "Result",
20
+ "SerialExecutor",
21
+ "DaskExecutor",
22
+ "build_plan",
23
+ "executor",
24
+ "profile",
25
+ "LockedError",
26
+ ]
27
+ __version__ = "0.7.0"
28
+
29
+ _log = _logging.getLogger("dfeols")
30
+ if not _log.handlers:
31
+ _h = _logging.StreamHandler()
32
+ _h.setFormatter(_logging.Formatter("%(asctime)s %(message)s", "%H:%M:%S"))
33
+ _log.addHandler(_h)
34
+ _log.setLevel(_logging.INFO)
dfeols/bench.py ADDED
@@ -0,0 +1,543 @@
1
+ # SPDX-License-Identifier: GPL-3.0-or-later
2
+ # Copyright (C) 2026 Ted Matherly
3
+ """Benchmark dfeols on synthetic data with known parameters.
4
+
5
+ For each size it generates data (reusing what exists), preps one shard set per
6
+ partition key the specs need, fits each spec, and records:
7
+
8
+ timing generate / prep / round 1 / solve / round 2 / total seconds
9
+ throughput rows per wall-second and per worker-second (queue-independent)
10
+ workers average and max workers connected during the fit
11
+ memory peak worker RSS
12
+ validity max |z| against the truth, joint Wald p, and whether that is what
13
+ the spec should find (unbiased / biased / se_wrong)
14
+ ETA predicted vs actual finish at 10/25/50/75% progress
15
+ baseline pyfixest time and max coefficient difference (small sizes only)
16
+
17
+ Examples
18
+ python -m dfeols.bench --rows 1e6 1e7 # local, generic panel
19
+ python -m dfeols.bench --rows 1e7 1e8 --profile mycluster # elastic cluster workers
20
+ python -m dfeols.bench --dgp influencer --specs level shape --rows 1e8 --profile mycluster
21
+ python -m dfeols.bench --rows 1e8 --set T=520 --set n_groups=2000 --profile mycluster
22
+ python -m dfeols.bench --preset units10m_t100 --profile mycluster # 1e9 rows, 10M units, T=100
23
+ python -m dfeols.bench --preset units1m_t1000 --rows 1e8 --profile mycluster # 1e8 first
24
+ python -m dfeols.bench --preset units100m_t100 --profile mycluster # 1e10 stress test
25
+ python -m dfeols.bench --preset skewed_groups --profile mycluster # oversized shards
26
+ python -m dfeols.bench --preset units100m_t100 --rows 2e6 --profile local \
27
+ --prep-opt hash_levels=0 --prep-opt range_groups=0 --prep-opt max_files=0 # 1e10 code paths, small
28
+ dfeols bench --profile mycluster --rows 1e8 1e9
29
+
30
+ Writes results.csv, results.json and summary.md to --out (default <root>/results).
31
+ """
32
+
33
+ from __future__ import annotations
34
+
35
+ import argparse
36
+ import json
37
+ import logging
38
+ import os
39
+ import shutil
40
+ import socket
41
+ import time
42
+ from pathlib import Path
43
+
44
+ import numpy as np
45
+ import polars as pl
46
+
47
+ import dfeols
48
+
49
+ from .formula import pyfixest_tol_kwargs, to_pyfixest
50
+ from .lock import DirLock
51
+ from .synth import generate, load_truth, plan_for_rows
52
+ from .validate import recovery
53
+
54
+ # Large-scale presets: billion-row unit x time panels with crossing time FEs.
55
+ # Balanced, so units = rows / T. Specs: unit + time partitioned by unit (time
56
+ # FE crosses every shard: sparse path) and unit + group^time partitioned by
57
+ # group (both FEs local: exact two-way solve).
58
+ PRESETS = {
59
+ "units10m_t100": {
60
+ "rows": [1e9],
61
+ "dgp": "panel",
62
+ "specs": ["twfe", "grouptime"],
63
+ "set": {"balanced": True, "T": 100, "n_groups": 500, "units_per_chunk": 20_000},
64
+ },
65
+ "units1m_t1000": {
66
+ "rows": [1e9],
67
+ "dgp": "panel",
68
+ "specs": ["twfe", "grouptime"],
69
+ "set": {"balanced": True, "T": 1000, "n_groups": 500, "units_per_chunk": 2_000},
70
+ },
71
+ # Stress tests.
72
+ # 1e10 rows, 100M units: hash ids for the unit FE, range-based shards by unit,
73
+ # a two-level shuffle when partitioning by group, 5,000 shards per set.
74
+ # Needs ~1 TB scratch (raw ~370 GB + two shard sets) and a 64 GB coordinator.
75
+ "units100m_t100": {
76
+ "rows": [1e10],
77
+ "dgp": "panel",
78
+ "specs": ["twfe", "grouptime"],
79
+ "gen_on_workers": True,
80
+ "set": {"balanced": True, "T": 100, "n_groups": 5000, "units_per_chunk": 20_000},
81
+ },
82
+ # Skewed group sizes (Zipf 0.7 over 5,000 groups): the largest group holds
83
+ # ~2.5% of rows (25M at 1e9) and becomes one oversized shard when
84
+ # partitioning by group; tests uneven shards, worker memory and the ETA floor.
85
+ "skewed_groups": {
86
+ "rows": [1e9],
87
+ "dgp": "panel",
88
+ "specs": ["twfe", "grouptime"],
89
+ "set": {"balanced": True, "T": 100, "n_groups": 5000, "group_zipf": 0.7, "units_per_chunk": 20_000},
90
+ },
91
+ }
92
+
93
+
94
+ def _available_cores() -> int:
95
+ """Cores this process may use (the job's allocation, not the whole node)."""
96
+ try:
97
+ return max(len(os.sched_getaffinity(0)), 1)
98
+ except AttributeError: # macOS
99
+ return os.cpu_count() or 1
100
+
101
+
102
+ def eta_and_workers(workdir: Path) -> dict:
103
+ p = workdir / "history.jsonl"
104
+ if not p.exists():
105
+ return {}
106
+ h = [json.loads(l) for l in p.read_text().splitlines() if l.strip()]
107
+ if not h:
108
+ return {}
109
+ actual = h[-1]["t"]
110
+ w = [x["workers"] for x in h]
111
+ out = {"workers_avg": round(float(np.mean(w)), 1), "workers_max": int(max(w))}
112
+ for q in (0.10, 0.25, 0.50, 0.75):
113
+ pts = [x for x in h if x["frac_done"] >= q and x["eta_total"] is not None]
114
+ if pts and actual > 0:
115
+ x = pts[0]
116
+ out[f"eta_err_{int(q * 100)}"] = round((x["t"] + x["eta_total"] - actual) / actual, 3)
117
+ return out
118
+
119
+
120
+ def pyfixest_baseline(raw: Path, spec: dict, res) -> dict:
121
+ import pyfixest as pf
122
+
123
+ pdf = pl.read_parquet(str(raw / "*.parquet")).to_pandas()
124
+ t = time.perf_counter()
125
+ m = pf.feols(
126
+ to_pyfixest(spec["formula"]), data=pdf, vcov={"CRV1": spec["cluster"]}, **pyfixest_tol_kwargs(1e-10)
127
+ )
128
+ secs = time.perf_counter() - t
129
+ tab = m.tidy().loc[res.coefnames]
130
+ return {
131
+ "pyfixest_s": round(secs, 1),
132
+ "pyfixest_max_coef_diff": float(np.max(np.abs(tab["Estimate"].to_numpy() - res.coef()))),
133
+ }
134
+
135
+
136
+ # ---------------------------------------------------------------- layout
137
+ #
138
+ # <root>/raw_<tag>/ generated data (reused across runs)
139
+ # <root>/shards_<tag>_by_<key>_<n>/ prepared shards (reused across runs)
140
+ # <root>/runs/<run_id>/<tag>/<spec>/ one fit: result.json, bench.json,
141
+ # status.json, history.jsonl (never overwritten)
142
+ # <root>/results/results.csv every fit ever run here, rebuilt from runs/
143
+ # <root>/results/<run_id>.md summary of one benchmark invocation
144
+ # <root>/results/latest.md copy of the most recent summary
145
+ #
146
+ # `dfeols results [root]` prints the table; nothing is overwritten, so earlier
147
+ # results stay where they were written.
148
+
149
+
150
+ def _param_suffix(dgp: str, kw: dict | None) -> str:
151
+ """Data folders are named by DGP, size and any non-default parameters, so
152
+ different configurations never collide (e.g. T=100 vs T=1000 at 1e9)."""
153
+ if not kw:
154
+ return ""
155
+ keep = {k: v for k, v in kw.items() if k not in ("units_per_chunk", "accounts_per_chunk")}
156
+ if not keep:
157
+ return ""
158
+ short = {"balanced": "bal", "n_groups": "G", "n_units": "U"}
159
+ parts = []
160
+ for k in sorted(keep):
161
+ v = keep[k]
162
+ v = ("1" if v else "0") if isinstance(v, bool) else str(v).replace(" ", "")
163
+ parts.append(f"{short.get(k, k)}{v}")
164
+ return "_" + "_".join(parts)
165
+
166
+
167
+ def make_run_id() -> str:
168
+ job = next(
169
+ (os.environ[v] for v in ("SLURM_JOB_ID", "PBS_JOBID", "JOB_ID", "LSB_JOBID") if os.environ.get(v)),
170
+ None,
171
+ )
172
+ rid = time.strftime("%Y%m%d-%H%M%S")
173
+ return f"{rid}-{job.split('.')[0]}" if job else rid
174
+
175
+
176
+ def _verdict(expect: str, r: dict) -> str:
177
+ if expect == "unbiased":
178
+ return "ok" if (r["ok"] and r["p"] > 0.001) else "FAIL"
179
+ if expect == "biased":
180
+ return "ok" if r["p"] < 1e-4 else "FAIL"
181
+ return "n/a"
182
+
183
+
184
+ def _row_from_result(res, truth, spec_name, *, n_rows=None, n_shards=None, wd=None) -> dict:
185
+ s = truth["specs"][spec_name]
186
+ tm = res.timing
187
+ r = recovery(res, truth)
188
+ return {
189
+ "dgp": truth.get("kind", "?"),
190
+ "rows": n_rows or res.N + res.N_dropped,
191
+ "N": res.N,
192
+ "spec": spec_name,
193
+ "expect": s["expect"],
194
+ "partition": s["partition"],
195
+ "shards": n_shards or tm.get("n_shards"),
196
+ "clusters": res.G,
197
+ "round1_s": round(tm["round1_seconds"], 1),
198
+ "round2_s": round(tm["round2_seconds"], 1),
199
+ "fit_s": round(res.seconds, 1),
200
+ "rows_per_s": int(res.N / max(res.seconds, 1e-9)),
201
+ "rows_per_worker_s": int(res.N / max(tm["worker_seconds"], 1e-9)),
202
+ "peak_worker_rss_gb": round(tm["max_worker_rss_gb"], 2),
203
+ "max_abs_z": round(r["max_abs_z"], 2),
204
+ "wald_p": round(r["p"], 4),
205
+ "validity": _verdict(s["expect"], r),
206
+ **(eta_and_workers(Path(wd)) if wd else {}),
207
+ }
208
+
209
+
210
+ def collect(root) -> list[dict]:
211
+ """Every benchmark fit under <root>/runs, newest first. Rows come from each
212
+ fit's bench.json; fits from before bench.json existed (runs/<tag>/<spec>/)
213
+ are rebuilt from result.json and the dataset's truth.json."""
214
+ from .results import Result
215
+
216
+ root = Path(root)
217
+ rows = []
218
+ for bj in root.glob("runs/*/*/*/bench.json"):
219
+ try:
220
+ rows.append(json.loads(bj.read_text()))
221
+ except Exception:
222
+ pass
223
+ for rj in root.glob("runs/*/*/result.json"): # legacy layout
224
+ tag, spec = rj.parent.parent.name, rj.parent.name
225
+ tp = root / f"raw_{tag}" / "truth.json"
226
+ if not tp.exists():
227
+ continue
228
+ try:
229
+ truth = json.loads(tp.read_text())
230
+ res = Result.load(rj)
231
+ row = _row_from_result(res, truth, spec, wd=rj.parent)
232
+ except Exception:
233
+ continue
234
+ ts = time.strftime("%Y%m%d-%H%M%S", time.localtime(rj.stat().st_mtime))
235
+ rows.append(
236
+ {
237
+ "run_id": f"{ts}-legacy",
238
+ "tag": tag,
239
+ "finished": time.strftime("%Y-%m-%d %H:%M", time.localtime(rj.stat().st_mtime)),
240
+ "workdir": str(rj.parent),
241
+ **row,
242
+ }
243
+ )
244
+ rows.sort(key=lambda r: (r.get("finished", ""), r.get("run_id", "")), reverse=True)
245
+ return rows
246
+
247
+
248
+ def write_ledger(root) -> list[dict]:
249
+ root = Path(root)
250
+ out = root / "results"
251
+ out.mkdir(parents=True, exist_ok=True)
252
+ rows = collect(root)
253
+ if rows:
254
+ cols = list(dict.fromkeys(k for r in rows for k in r))
255
+ pl.DataFrame([{c: r.get(c) for c in cols} for r in rows], infer_schema_length=None).write_csv(
256
+ out / "results.csv"
257
+ )
258
+ (out / "results.json").write_text(json.dumps(rows, indent=1))
259
+ return rows
260
+
261
+
262
+ def stale_dataset(ds, target_rows) -> str:
263
+ """Why an existing benchmark shard folder must be rebuilt ("" to reuse it)."""
264
+ m = ds.meta
265
+ if sum(s["rows"] for s in ds.shards) != ds.n_rows:
266
+ return "shard row counts don't add up"
267
+ if m.get("shard_assignment") == "range" and not m.get("prep_version"):
268
+ # range shards before 0.6.2 used UInt32 row counts that overflowed
269
+ return "built by a version with the range-shard overflow bug (before 0.6.2)"
270
+ biggest = max(s["rows"] for s in ds.shards)
271
+ if biggest > 10 * target_rows and len(ds.shards) < ds.n_rows / (10 * target_rows):
272
+ return f"shards are far from the target size (largest {biggest:,} rows)"
273
+ return ""
274
+
275
+
276
+ def run(
277
+ rows_list,
278
+ *,
279
+ root,
280
+ out=None,
281
+ dgp="panel",
282
+ dgp_kw=None,
283
+ specs=None,
284
+ profile=None,
285
+ target_rows=2e6,
286
+ gen_jobs=1,
287
+ pyfixest_max_rows=0,
288
+ keep_data=True,
289
+ gen_on_workers=False,
290
+ prep_opts=None,
291
+ log=print,
292
+ ):
293
+ root = Path(root)
294
+ out = Path(out) if out else root / "results"
295
+ out.mkdir(parents=True, exist_ok=True)
296
+ run_id = make_run_id()
297
+ summary = out / f"{run_id}.md"
298
+ log(
299
+ f"benchmark run {run_id}\n this run's summary: {summary}\n all results: "
300
+ f"{out / 'results.csv'} (or: dfeols results {root})"
301
+ )
302
+ ex = dfeols.executor(profile) if profile else dfeols.SerialExecutor()
303
+ host = socket.gethostname()
304
+ results = []
305
+ try:
306
+ for rows in rows_list:
307
+ tag = f"{dgp}_{rows:.0e}".replace("+", "") + _param_suffix(dgp, dgp_kw)
308
+ raw = root / f"raw_{tag}"
309
+ d = plan_for_rows(rows, dgp, **(dgp_kw or {}))
310
+ # One job at a time per data set: a second job would regenerate or
311
+ # re-prep (deleting) the folders the first one is still writing.
312
+ # Held for generation and prep only; fits only read the shards.
313
+ with DirLock(root / "locks" / tag, what=f"benchmark data set {tag}"):
314
+ t = time.perf_counter()
315
+ # generate on the coordinator by default: it runs while the first
316
+ # worker jobs wait in the queue (on workers it would wait for them)
317
+ generate(
318
+ raw, d, jobs=gen_jobs, executor=ex if (profile and gen_on_workers) else None, verbose=True
319
+ )
320
+ t_gen = time.perf_counter() - t
321
+ truth = load_truth(raw)
322
+ todo = specs or list(truth["specs"])
323
+ missing = set(todo) - set(truth["specs"])
324
+ if missing:
325
+ raise KeyError(f"{dgp} DGP has no specs {sorted(missing)}; have {sorted(truth['specs'])}")
326
+
327
+ datasets, t_prep = {}, {}
328
+ for part in sorted({truth["specs"][s]["partition"] for s in todo}):
329
+ sd = root / f"shards_{tag}_by_{part}_{int(target_rows)}"
330
+ t = time.perf_counter()
331
+ reuse = (sd / "dfeols_meta.json").exists()
332
+ if reuse:
333
+ why = stale_dataset(dfeols.Dataset.open(sd), target_rows)
334
+ if why:
335
+ log(f"re-preparing {sd.name}: {why}")
336
+ reuse = False
337
+ if reuse:
338
+ datasets[part] = dfeols.Dataset.open(sd)
339
+ t_prep[part] = float("nan")
340
+ else:
341
+ datasets[part] = dfeols.prep(
342
+ str(raw / "*.parquet"),
343
+ shard_dir=sd,
344
+ partition=part,
345
+ target_rows=int(target_rows),
346
+ overwrite=True,
347
+ executor=ex if profile else None,
348
+ **truth["prep"],
349
+ **((prep_opts or {}) if profile else {}),
350
+ )
351
+ t_prep[part] = time.perf_counter() - t
352
+ n_rows = next(iter(datasets.values())).n_rows
353
+ log(
354
+ f"\n== {dgp}: {n_rows:,} rows, {d.n_units:,} units (generate {t_gen:.0f}s, prep "
355
+ + ", ".join(f"by {k} {v:.0f}s" for k, v in t_prep.items())
356
+ + ")"
357
+ )
358
+
359
+ for name in todo:
360
+ s = truth["specs"][name]
361
+ ds = datasets[s["partition"]]
362
+ wd = root / "runs" / run_id / tag / name
363
+ res = dfeols.fit(
364
+ ds,
365
+ s["formula"],
366
+ cluster=s["cluster"],
367
+ executor=ex,
368
+ workdir=wd,
369
+ label=f"{tag}-{name}",
370
+ log_every=15,
371
+ )
372
+ row = {
373
+ "run_id": run_id,
374
+ "tag": tag,
375
+ "finished": time.strftime("%Y-%m-%d %H:%M"),
376
+ "profile": profile or "serial",
377
+ "coordinator": host,
378
+ **_row_from_result(res, truth, name, n_rows=ds.n_rows, n_shards=len(ds.shards), wd=wd),
379
+ "generate_s": round(t_gen, 1),
380
+ "prep_s": round(t_prep[s["partition"]], 1),
381
+ "workdir": str(wd),
382
+ }
383
+ if pyfixest_max_rows and ds.n_rows <= pyfixest_max_rows:
384
+ row.update(pyfixest_baseline(raw, s, res))
385
+ (wd / "bench.json").write_text(json.dumps(row, indent=1))
386
+ results.append(row)
387
+ log(json.dumps(row))
388
+ # after every fit, so partial runs are visible
389
+ write_ledger(root)
390
+ write_summary(results, summary, profile or "serial", host, target_rows, run_id)
391
+ shutil.copyfile(summary, out / "latest.md")
392
+ if not keep_data:
393
+ for sd in {Path(ds.root) for ds in datasets.values()}:
394
+ shutil.rmtree(sd, ignore_errors=True)
395
+ finally:
396
+ if hasattr(ex, "client"):
397
+ ex.client.close()
398
+ ex.cluster.close()
399
+ if results:
400
+ write_ledger(root)
401
+ write_summary(results, summary, profile or "serial", host, target_rows, run_id)
402
+ shutil.copyfile(summary, out / "latest.md")
403
+ return results
404
+
405
+
406
+ def write_summary(rows, path: Path, where: str, host: str, target_rows, run_id: str = ""):
407
+ cols = [
408
+ "dgp",
409
+ "rows",
410
+ "spec",
411
+ "expect",
412
+ "validity",
413
+ "shards",
414
+ "clusters",
415
+ "workers_avg",
416
+ "fit_s",
417
+ "round1_s",
418
+ "round2_s",
419
+ "rows_per_worker_s",
420
+ "rows_per_s",
421
+ "peak_worker_rss_gb",
422
+ "max_abs_z",
423
+ "wald_p",
424
+ "eta_err_25",
425
+ "eta_err_50",
426
+ "pyfixest_s",
427
+ "pyfixest_max_coef_diff",
428
+ ]
429
+ cols = [c for c in cols if any(c in r for r in rows)]
430
+ lines = [
431
+ f"# dfeols benchmark {run_id} ({where}, coordinator {host}, {time.strftime('%Y-%m-%d %H:%M')})",
432
+ "",
433
+ f"Shard target {int(target_rows):,} rows. Fit directories: runs/{run_id}/.",
434
+ "",
435
+ "| " + " | ".join(cols) + " |",
436
+ "|" + "---|" * len(cols),
437
+ ]
438
+ for r in rows:
439
+ vals = []
440
+ for c in cols:
441
+ v = r.get(c, "")
442
+ vals.append(
443
+ f"{v:,}"
444
+ if isinstance(v, int) and not isinstance(v, bool)
445
+ else (f"{v:.1e}" if c.endswith("diff") and v != "" else str(v))
446
+ )
447
+ lines.append("| " + " | ".join(vals) + " |")
448
+ lines += [
449
+ "",
450
+ "validity: ok if an 'unbiased' spec recovers the truth (every |z| under the "
451
+ "Bonferroni bound and Wald p > 0.001) or a 'biased' spec is rejected (p < 1e-4).",
452
+ "rows_per_worker_s is throughput per worker, independent of how many workers the "
453
+ "queue provided; workers_avg shows how many there were.",
454
+ "eta_err_q = (predicted - actual) / actual finish time when q% of rows were done.",
455
+ ]
456
+ path.write_text("\n".join(lines) + "\n")
457
+
458
+
459
+ def main(argv=None):
460
+ ap = argparse.ArgumentParser(
461
+ prog="python -m dfeols.bench",
462
+ description=__doc__,
463
+ formatter_class=argparse.RawDescriptionHelpFormatter,
464
+ )
465
+ ap.add_argument(
466
+ "--preset",
467
+ choices=sorted(PRESETS),
468
+ default=None,
469
+ help="large-scale configuration; --rows/--set/--specs still override",
470
+ )
471
+ ap.add_argument("--rows", type=float, nargs="+", default=None, help="default 1e6 1e7")
472
+ ap.add_argument("--dgp", default="panel")
473
+ ap.add_argument("--set", action="append", default=[], metavar="KEY=VALUE", help="DGP parameter")
474
+ ap.add_argument("--specs", nargs="+", default=None, help="spec names (default: all of the DGP's)")
475
+ ap.add_argument(
476
+ "--profile",
477
+ default=os.environ.get("DFEOLS_PROFILE"),
478
+ help="site profile for workers (default: the profile of the job this runs in, "
479
+ "set by `dfeols submit`/`dfeols bench`; serial when run directly)",
480
+ )
481
+ ap.add_argument("--root", default=None, help="data/shards/runs (default: <profile scratch>/bench)")
482
+ ap.add_argument("--out", default=None)
483
+ ap.add_argument("--target-rows", type=float, default=2e6, help="rows per shard")
484
+ ap.add_argument(
485
+ "--gen-jobs",
486
+ type=int,
487
+ default=_available_cores(),
488
+ help="local processes for data generation (default: cores allocated to this job)",
489
+ )
490
+ ap.add_argument(
491
+ "--gen-on-workers", action="store_true", help="generate on cluster workers instead of the coordinator"
492
+ )
493
+ ap.add_argument("--pyfixest-max-rows", type=float, default=0)
494
+ ap.add_argument("--drop-data", action="store_true", help="delete shards after each size")
495
+ ap.add_argument(
496
+ "--prep-opt",
497
+ action="append",
498
+ default=[],
499
+ metavar="KEY=VALUE",
500
+ help="distributed prep option, e.g. --prep-opt max_files=0 (forces the two-level "
501
+ "shuffle); see dfeols.prep_dist.prep_distributed",
502
+ )
503
+ a = ap.parse_args(argv)
504
+ logging.getLogger("dfeols").setLevel(logging.INFO)
505
+ pre = PRESETS.get(a.preset, {})
506
+ if pre:
507
+ a.dgp = pre["dgp"] if a.dgp == "panel" else a.dgp
508
+ a.rows = a.rows or pre["rows"]
509
+ a.specs = a.specs or pre["specs"]
510
+ a.gen_on_workers = a.gen_on_workers or pre.get("gen_on_workers", False)
511
+ a.rows = a.rows or [1e6, 1e7]
512
+ kw = dict(pre.get("set", {}))
513
+ for s in a.set:
514
+ k, v = s.split("=", 1)
515
+ try:
516
+ kw[k] = json.loads(v)
517
+ except json.JSONDecodeError:
518
+ kw[k] = v
519
+ if "theta" in kw:
520
+ kw["theta"] = tuple(kw["theta"])
521
+ root = a.root or str(dfeols.profile(a.profile).scratch_dir("bench"))
522
+ run(
523
+ a.rows,
524
+ root=root,
525
+ out=a.out,
526
+ dgp=a.dgp,
527
+ dgp_kw=kw,
528
+ specs=a.specs,
529
+ profile=a.profile,
530
+ target_rows=a.target_rows,
531
+ gen_jobs=a.gen_jobs,
532
+ pyfixest_max_rows=a.pyfixest_max_rows,
533
+ keep_data=not a.drop_data,
534
+ gen_on_workers=a.gen_on_workers,
535
+ prep_opts={k: json.loads(v) for k, v in (o.split("=", 1) for o in a.prep_opt)},
536
+ )
537
+ latest = Path(a.out or Path(root) / "results") / "latest.md"
538
+ if latest.exists():
539
+ print(latest.read_text())
540
+
541
+
542
+ if __name__ == "__main__":
543
+ main()