forexgen 2.1.0__py3-none-any.whl

forexgen/__init__.py

@@ -0,0 +1,27 @@
+"""forexgen 2.0 - offline synthetic forex / gold data generator.
+
+Quick start:
+    import forexgen
+    df = forexgen.get_data("EURUSD", "M15", "3m")            # different every call; seed in df.attrs
+    df = forexgen.get_data("GBPUSD", "H1", "1y", seed=7)      # reproducible
+    forexgen.describe("XAUUSD")                               # calibrated vs assumed parameters
+"""
+from .api import (generate_csv, get_data, get_multi, list_markets, parse_duration,  # noqa: F401
+                  parse_timeframe, register_market)
+from .profiles import (Profile, describe, get_profile, load_profile, save_profile,  # noqa: F401
+                       use_profile, reset_profiles)
+from .events import get_events, register_event, set_events, clear_user_events      # noqa: F401
+from .scenarios import SCENARIOS                                                    # noqa: F401
+from .ticks import get_ticks                                                        # noqa: F401
+from .report import scorecard, stats_m1                                             # noqa: F401
+from .calibration import calibrate, calibrate_many, CalibrationSet, read_histdata                                     # noqa: F401
+from .validation import get_null_paths, monte_carlo, percentile_in_null, backtest   # noqa: F401
+from .costs import slippage, stop_fill, swap_pips                                   # noqa: F401
+
+from .factors import set_correlation, reset_correlation                             # noqa: F401
+from .export import export_histdata, generate_histdata, to_histdata_frame            # noqa: F401
+from .diagnostics import selftest, robustness_report, trend_strategy, mean_reversion_strategy  # noqa: F401
+from . import fixtures                                                              # noqa: F401
+from .fixtures import indicator_stress_test                                         # noqa: F401
+
+__version__ = "2.1.0"

forexgen/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

forexgen/api.py

@@ -0,0 +1,94 @@
+"""Public API: get_data / generate_csv / get_multi (v1-compatible signatures)."""
+import os
+import re
+
+import pandas as pd
+
+from . import pipeline as pl
+from .marketcal import parse_duration, parse_timeframe          # noqa: F401  (re-exported)
+from .markets import get_market, list_markets, register_market, normalize_symbol  # noqa: F401
+from .pipeline import DEFAULT_START
+
+CSV_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+
+def get_data(market, timeframe="M15", duration="30d", start=DEFAULT_START, end=None,
+             seed=None, start_price=None, profile="typical", scenario=None, scenario_at=0.5,
+             scenario_strength=1.0, scenario_direction="auto", scenario_markets=None,
+             events="default", sparse=False, return_labels=False, edge=None, daily_anchor="utc", **model):
+    """Generate synthetic OHLCV bars for a market.
+
+    market     : e.g. 'EURUSD', 'GBPUSD', 'XAUUSD' (see list_markets())
+    timeframe  : M1 M5 M15 M30 H1 H4 D1 W1 ... (also '15m', '1h', 'daily')
+    duration   : '30d', '2w', '3m', '1y', '12h', '90min' (market time) - or use end=
+    start, end : UTC timestamps (naive UTC). Output index is naive UTC.
+    seed       : None (default) -> a different dataset every call; the seed used is in
+                 df.attrs['seed']. An int reproduces the data exactly.
+    profile    : 'typical' (default) | 'calm_aug2026' | 'stressed', or a forexgen.Profile
+    scenario   : None or one of forexgen.SCENARIOS, placed at scenario_at (timestamp or
+                 fraction 0..1 of the data) with scenario_strength
+    events     : 'default' (rule calendar) | 'none' | DataFrame/CSV of your own events
+    sparse     : True -> broker-like flat bars and missing minutes
+    return_labels : add columns regime, event_flag, true_vol
+    edge       : dict(kind='momentum'|'mean_reversion', strength=0.02, horizon='1h') plants a
+                 known edge (for testing whether a strategy/backtester can find it)
+    daily_anchor : 'utc' (default, as v1: D1/W1 cut at UTC midnight) or 'ny' (cut at the 17:00 New York close,
+                 5 bars per week, the usual FX convention)
+    **model    : v1 options still accepted: garch=(a,b), t_df, jump_rate_per_day, jump_sigma,
+                 weekend_gap, kappa, drift, usd_factor (+ vol_scale)
+
+    Returns a DataFrame indexed by UTC time with open, high, low, close, volume, spread_pips.
+    """
+    return pl.generate(market, timeframe, duration, start, end, seed, start_price, profile,
+                       scenario, scenario_at, scenario_strength, scenario_direction,
+                       scenario_markets, events, sparse, return_labels, edge, False,
+                       model=model, daily_anchor=daily_anchor)
+
+
+def default_csv_path(market, timeframe, duration, seed, folder="forex_data", end=None):
+    """<folder>/<SYMBOL>_<TF>_<duration>_seed<seed>.csv - the seed actually used is in the name."""
+    mk, tf = get_market(market), parse_timeframe(timeframe)
+    tag = re.sub(r"\W", "", str(duration))
+    if end is not None:
+        tag = "to-" + re.sub(r"\D", "", str(end))
+    return os.path.join(folder, f"{mk.symbol}_{tf}_{tag}_seed{seed}.csv")
+
+
+def generate_csv(market, timeframe="M15", duration="30d", path=None, folder="forex_data", **kwargs):
+    """Generate data and write a CSV. Returns the file path.
+
+    Default name: <folder>/<SYMBOL>_<TF>_<duration>_seed<seed>.csv - the seed actually used
+    is in the name, so any run can be reproduced with seed=<that number>."""
+    df = get_data(market, timeframe, duration, **kwargs)
+    if path is None:
+        path = default_csv_path(market, timeframe, duration, df.attrs["seed"], folder, kwargs.get("end"))
+    os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
+    df.to_csv(path, date_format=CSV_DATE_FORMAT)
+    return path
+
+
+_GEN_KW = {"scenario", "scenario_at", "scenario_strength", "scenario_direction", "scenario_markets",
+           "events", "sparse", "return_labels", "edge", "daily_anchor"}
+
+
+def get_multi(markets, timeframe="M15", duration="30d", start=DEFAULT_START, end=None, seed=None,
+              profile="typical", as_frame=False, **kwargs):
+    """Jointly simulate several markets: shared factors, correlations that rise in crises, and
+    crosses (EURGBP, EURJPY, GBPJPY) derived exactly from their legs.
+
+    Returns {symbol: DataFrame}, or one DataFrame with (market, field) columns if as_frame=True.
+    A market inside get_multi equals get_data(market, seed=seed) for the same arguments.
+    Accepts the same options as get_data (scenario=..., events=..., sparse=..., v1 model kwargs)."""
+    if isinstance(markets, str):
+        markets = [markets]
+    seed = pl.resolve_seed(seed)
+    gen = {k: v for k, v in kwargs.items() if k in _GEN_KW}
+    model = {k: v for k, v in kwargs.items() if k not in _GEN_KW}
+    cache, out = {}, {}
+    for m in markets:
+        mk = get_market(m)
+        out[mk.symbol] = pl.generate(mk.symbol, timeframe, duration, start, end, seed, None, profile,
+                                     cache=cache, model=model, **gen)
+    if as_frame:
+        return pd.concat(out, axis=1)
+    return out

forexgen/calibration.py

@@ -0,0 +1,357 @@
+"""calibrate(): fit a market Profile from a real M1 file (HistData / MetaTrader ASCII format).
+
+Fitted: volatility level, event-free seasonality (NY hour), microstructure (lag-1 bounce and
+variance-ratio profile, solved analytically), GARCH persistence (simulated method of moments on
+the |r| autocorrelations), bar-range scale, weekend-gap size, flat-bar / missing-minute rates and,
+where >= 8 events of an importance class exist, the event shock size.
+NOT fittable from price-only files: spreads and volume (flagged "assumed").
+"""
+import math
+
+import numpy as np
+import pandas as pd
+
+from . import events as ev
+from . import report
+from .marketcal import ny_clock
+from .markets import get_market
+from .profiles import Profile, bootstrap_eurusd, builtin_profile, get_profile
+
+
+# ---------------------------------------------------------------- reading
+def read_histdata(src, tz="America/New_York", fmt="histdata_mt"):
+    """Read a HistData/MT file (no header: date YYYY.MM.DD, time HH:MM, o, h, l, c, v) or a DataFrame
+    (index or 'time' column + open/high/low/close). Returns an M1 DataFrame, naive UTC index."""
+    if isinstance(src, pd.DataFrame):
+        df = src.copy()
+        if "time" in df.columns:
+            df = df.set_index("time")
+        df.index = pd.to_datetime(df.index)
+    else:
+        with open(src) as fh:
+            first = fh.readline()
+        if ";" in first:        # HistData ASCII M1: "YYYYMMDD HHMMSS;open;high;low;close;volume" (EST, no DST)
+            raw = pd.read_csv(src, sep=";", header=None, names=["dt", "open", "high", "low", "close", "volume"])
+            df = raw[["open", "high", "low", "close", "volume"]].copy()
+            df.index = pd.to_datetime(raw["dt"], format="%Y%m%d %H%M%S")
+            if tz == "America/New_York" and fmt == "histdata_mt":
+                tz = "EST"      # ASCII files are fixed UTC-5 unless the caller says otherwise
+        else:
+            raw = pd.read_csv(src, header=None, names=["d", "t", "open", "high", "low", "close", "volume"])
+            df = raw[["open", "high", "low", "close", "volume"]].copy()
+            df.index = pd.to_datetime(raw["d"] + " " + raw["t"], format="%Y.%m.%d %H:%M")
+    df = df[["open", "high", "low", "close"] + (["volume"] if "volume" in df.columns else [])]
+    df = df[~df.index.duplicated(keep="first")].sort_index()
+    if str(tz).upper() == "EST":                       # fixed UTC-5, no daylight saving
+        df.index = df.index + pd.Timedelta(hours=5)
+    elif tz and str(tz).upper() != "UTC":
+        try:
+            idx = df.index.tz_localize(tz, ambiguous="infer", nonexistent="NaT")
+        except Exception:
+            idx = df.index.tz_localize(tz, ambiguous="NaT", nonexistent="NaT")
+        keep = ~pd.isna(idx)
+        df, idx = df[keep], idx[keep]
+        df.index = idx.tz_convert("UTC").tz_localize(None)
+        df = df.sort_index()
+    df.index.name = "time"
+    return df
+
+
+# ---------------------------------------------------------------- microstructure fit
+def fit_micro(ac1, vr, pfs=(0.2, 0.35, 0.5), pss=(0.75, 0.80, 0.85, 0.90, 0.93, 0.95, 0.97, 0.98, 0.99, 0.995)):
+    """Solve for (fx, vf, vs, phi_f, phi_s): efficient-price share, fast-bounce variance and slow
+    mean-reverting variance, matching lag-1 autocorrelation and variance ratios (linear in the
+    three variances once the two AR coefficients are fixed; grid over the coefficients)."""
+    best = None
+    ks = [k for k in (5, 15, 60, 240) if k in vr and vr[k] == vr[k]]
+    if ac1 != ac1 or not ks:
+        return None
+    for pf in pfs:
+        for ps in pss:
+            A, y, w = [[1.0, 2 * (1 - pf), 2 * (1 - ps)]], [1.0], [10.0]
+            A.append([0.0, (1 - pf) ** 2, (1 - ps) ** 2])
+            y.append(-ac1)
+            w.append(25.0)
+            for k in ks:
+                A.append([1.0, 2 * (1 - pf ** k) / k, 2 * (1 - ps ** k) / k])
+                y.append(vr[k])
+                w.append(1.0)
+            A, y, w = np.array(A), np.array(y), np.array(w)
+            active = [0, 1, 2]
+            for _ in range(3):
+                sol, *_ = np.linalg.lstsq(A[:, active] * w[:, None], y * w, rcond=None)
+                full = np.zeros(3)
+                full[active] = sol
+                neg = [a for a, v in zip(active, sol) if v < 0]
+                if not neg:
+                    break
+                active = [a for a in active if a not in neg]
+            res = float(np.sum((w * (A @ full - y)) ** 2))
+            if best is None or res < best[0]:
+                best = (res, full, pf, ps)
+    _, (X, F, S), pf, ps = best
+    X = max(X, 0.05)
+    return dict(fx=float(X), vf=float(F), vs=float(S), phi_f=pf, phi_s=ps)
+
+
+# ---------------------------------------------------------------- helpers
+def _event_mask(index, start, end, minutes_before=5):
+    cal = ev.rule_events(start - pd.Timedelta(days=1), end + pd.Timedelta(days=1))
+    cal = cal[cal.importance >= 2]
+    mask = np.zeros(len(index), dtype=bool)
+    t = index.values.astype("datetime64[m]").astype(np.int64)
+    for t0, imp in zip(cal.time_utc.values.astype("datetime64[m]").astype(np.int64), cal.importance.values):
+        after = 150 if imp >= 4 else 60
+        lo, hi = np.searchsorted(t, t0 - minutes_before), np.searchsorted(t, t0 + after)
+        mask[lo:hi] = True
+    return mask
+
+
+def _run(profile, variant, days, seeds, sym, pip):
+    """Stats of simulated data for a profile in a given variant (profile object + variant name)."""
+    from . import pipeline as pl
+    res = []
+    for sd in seeds:
+        ts = pl.timeline("2024-03-04 00:00", f"{days}d", None, get_market(sym).daily_break)
+        cal = ev.calendar_between(ts[0] - pd.Timedelta(days=1), ts[-1] + pd.Timedelta(days=1), "default")
+        m1 = pl.build_m1(sym, ts, sd, variant, profile, None, None, cal, None, {}, None, {})
+        res.append(report.stats_m1(pl.finalize(m1, get_market(sym)), pip))
+    return res
+
+
+def _mean(stats, key, sub=None):
+    vals = [(s[key][sub] if sub is not None else s[key]) for s in stats]
+    return float(np.nanmean(vals))
+
+
+# ---------------------------------------------------------------- main
+def calibrate(src, market="EURUSD", tz="America/New_York", fmt="histdata_mt", variant="calm_aug2026",
+              sim_days=60, seeds=(11, 12, 13, 14), verbose=True):
+    """Fit and return a Profile from real M1 data of `market`. See module docstring.
+
+    variant: which volatility variant the file represents ('calm_aug2026' = a calm month; its
+             measured level becomes ann_vol_calm). Use 'typical' if the file is a normal year."""
+    mk = get_market(market)
+    sym = mk.symbol
+    log = (lambda *a: print(*a)) if verbose else (lambda *a: None)
+    df = read_histdata(src, tz, fmt)
+    pip = mk.pip
+    real = report.stats_m1(df, pip)
+    start_prof = bootstrap_eurusd() if sym == "EURUSD" else get_profile(sym)
+    p = start_prof.copy(symbol=sym)
+    prov = dict(p.provenance) if sym != "EURUSD" else {}
+    log(f"[calibrate] {sym}: {real['n_bars']} bars {df.index[0]} .. {df.index[-1]} (UTC)")
+
+    # ---- event-free seasonality (robust)
+    r = report.returns_m1(df)
+    mask_all = _event_mask(r.index, df.index[0], df.index[-1])
+    r_ex = r[~mask_all]
+    scale = 1.4826 * np.median(np.abs(r_ex - r_ex.median())) + 1e-12
+    rw = r_ex.clip(-8 * scale, 8 * scale)
+    hrs, _ = ny_clock(rw.index)
+    prof_h = rw.abs().groupby(np.floor(hrs).astype(int)).mean().reindex(range(24))
+    prof_h = prof_h.interpolate(limit_direction="both").values
+    sm = np.convolve(np.r_[prof_h[-1], prof_h, prof_h[0]], [0.25, 0.5, 0.25], mode="valid")
+    p.seasonality = (sm / math.sqrt(np.mean(sm ** 2))).tolist()
+    prov["seasonality"] = "calibrated (event windows excluded, robust, NY hour)"
+
+    # ---- microstructure
+    mic = fit_micro(real["ac1"], real["vr"])
+    if mic:
+        for k, v in mic.items():
+            setattr(p, k, v)
+        prov["microstructure"] = (f"calibrated (lag-1 ACF {real['ac1']:.3f}, variance ratios "
+                                  f"{', '.join(f'{k}m={v:.2f}' for k, v in real['vr'].items())})")
+    # ---- level, gaps, rates
+    p.ann_vol_calm = real["ann_vol_m1"] if variant == "calm_aug2026" else p.ann_vol_calm
+    if variant == "typical":
+        p.ann_vol_long = real["ann_vol_m1"]
+    prov["volatility level"] = f"calibrated ({variant} M1-implied {real['ann_vol_m1']:.2%}); the other level is assumed"
+    g = df.index.to_series().diff()
+    big = g[(g > pd.Timedelta("40h")) & (g < pd.Timedelta("60h"))].index   # weekend-sized gaps only (not month joins)
+    gaps_bp = [abs(df["open"].loc[i] / df["close"].shift(1).loc[i] - 1) * 1e4 for i in big]
+    if len(gaps_bp) >= 1:
+        p.weekend_gap_bp = float(np.mean(gaps_bp))   # measured in the file's own (calm) month
+        prov["weekend gap"] = f"calibrated (n={len(gaps_bp)} gaps)"
+    p.flat_share = real["flat_share"]
+    p.missing_share = real["missing_share"]
+    prov["flat bars / missing minutes"] = "calibrated"
+
+    # ---- event sizes (only with >= 8 events in a class)
+    cal = ev.rule_events(df.index[0], df.index[-1] + pd.Timedelta(days=1))
+    cal = cal[cal.importance >= 3]
+    tt = r.index.values.astype("datetime64[m]").astype(np.int64)
+    obs = {}
+    for t0, imp, name in zip(cal.time_utc.values.astype("datetime64[m]").astype(np.int64),
+                             cal.importance.values, cal.name.values):
+        lo, hi = np.searchsorted(tt, t0), np.searchsorted(tt, t0 + 5)
+        if hi > lo:
+            obs.setdefault(int(imp), []).append((name, float(r.iloc[lo:hi].abs().max() * 1e4)))
+    p.meta["observed_events_bp"] = {str(k): [[str(n), round(v, 1)] for n, v in vs] for k, vs in obs.items()}   # JSON-stable
+    done = []
+    for imp, vs in obs.items():
+        if len(vs) >= 8:
+            p.event_base_bp[str(imp)] = float(np.median([v for _, v in vs]) / 0.75)
+            done.append(imp)
+    anchors = sum(len(v) for v in obs.values())
+    prov["events"] = (f"calibrated for importance {done}" if done else
+                      f"assumed shape/size, sanity-anchored to {anchors} observed events "
+                      f"({'; '.join(f'{n} {v:.0f}bp' for vs in obs.values() for n, v in vs)})")
+    prov["spread"] = "assumed (price-only file has no bid/ask)"
+    prov["volume"] = "assumed (file volume is not usable)"
+    prov["regimes & scenarios"] = "assumed"
+
+    # ---- simulated-method-of-moments: GARCH persistence, then range scale and level
+    sim_variant = variant
+    target_acf = real["acf_abs"]
+    best = None
+    for alpha in (0.03, 0.05, 0.08, 0.12, 0.16):
+        for phi in (0.93, 0.95, 0.97, 0.985, 0.995):
+            q = p.copy(garch_alpha=alpha, garch_beta=phi - alpha)
+            st = _run(q, sim_variant, sim_days, seeds[:2], sym, pip)
+            loss = sum((_mean(st, "acf_abs", k) - target_acf[k]) ** 2 for k in (1, 10, 60) if k in target_acf)
+            if best is None or loss < best[0]:
+                best = (loss, alpha, phi)
+    p.garch_alpha, p.garch_beta = best[1], best[2] - best[1]
+    prov["garch"] = f"calibrated by simulated method of moments (alpha={best[1]:.2f}, persistence={best[2]:.3f})"
+    cur_ac, cur_vr = real["ac1"], dict(real["vr"])
+    for it in range(5):
+        st = _run(p, sim_variant, sim_days, seeds, sym, pip)
+        p.vol_base_adj *= (real["ann_vol_m1"] / _mean(st, "ann_vol_m1")) ** 1.0
+        # simulation feedback: scheduled-event spikes add permanent variance, so correct the
+        # analytic microstructure targets by the simulated-vs-real gap and refit
+        if mic:
+            cur_ac += 0.8 * (real["ac1"] - _mean(st, "ac1"))
+            for k in cur_vr:
+                cur_vr[k] += 0.8 * (real["vr"][k] - _mean(st, "vr", k))
+            mic2 = fit_micro(cur_ac, cur_vr)
+            for k, v in mic2.items():
+                setattr(p, k, v)
+        lo_, hi_ = 0.15, 1.2
+        for _ in range(7):
+            mid = 0.5 * (lo_ + hi_)
+            stt = _run(p.copy(range_scale=mid), sim_variant, 15, seeds[:2], sym, pip)
+            if _mean(stt, "range_mean_pips") > real["range_mean_pips"]:
+                hi_ = mid
+            else:
+                lo_ = mid
+        p.range_scale = 0.5 * (lo_ + hi_)
+    # ---- final polish on longer simulations (range scale and level)
+    for _ in range(3):
+        st = _run(p, sim_variant, 120, seeds, sym, pip)
+        p.range_scale *= (real["range_mean_pips"] / _mean(st, "range_mean_pips")) ** 1.2
+        p.vol_base_adj *= (real["ann_vol_m1"] / _mean(st, "ann_vol_m1")) ** 0.8
+    prov["bar range"] = f"calibrated (mean {real['range_mean_pips']:.2f} pips)"
+    p.provenance = prov
+    p.meta.update({"calibrated_from": str(src) if not isinstance(src, pd.DataFrame) else "DataFrame",
+                   "n_bars": real["n_bars"], "period_utc": [str(df.index[0]), str(df.index[-1])],
+                   "tz": tz, "variant": variant})
+    log("[calibrate] done: vol_base_adj=%.3f range_scale=%.3f garch=(%.3f, %.3f) micro=%s" % (
+        p.vol_base_adj, p.range_scale, p.garch_alpha, p.garch_beta,
+        {k: round(getattr(p, k), 3) for k in ("fx", "vf", "vs", "phi_f", "phi_s")}))
+    return p
+
+
+# ---------------------------------------------------------------- several files / markets / months
+import json as _json
+import os as _os
+import re as _re
+
+
+class CalibrationSet:
+    """Result of calibrate_many: per-market profiles, the realised correlation matrix of the data
+    (base markets, M15 returns) and metadata. apply() installs everything; save()/load() persist it."""
+
+    def __init__(self, profiles, corr=None, meta=None):
+        self.profiles, self.corr, self.meta = profiles, corr, meta or {}
+
+    def apply(self, correlation=True):
+        from .profiles import use_profile
+        from . import factors
+        for sym, p in self.profiles.items():
+            use_profile(p, sym)
+        if correlation and self.corr is not None and self.corr.shape[0] >= 2:
+            factors.set_correlation(self.corr)
+        return self
+
+    def save(self, folder):
+        _os.makedirs(folder, exist_ok=True)
+        for sym, p in self.profiles.items():
+            p.save(_os.path.join(folder, f"{sym}.json"))
+        if self.corr is not None:
+            self.corr.to_csv(_os.path.join(folder, "correlation.csv"))
+        with open(_os.path.join(folder, "meta.json"), "w") as f:
+            _json.dump(self.meta, f, indent=2, default=str)
+        return folder
+
+    @classmethod
+    def load(cls, folder):
+        profs = {fn[:-5]: Profile.load(_os.path.join(folder, fn)) for fn in sorted(_os.listdir(folder))
+                 if fn.endswith(".json") and fn != "meta.json"}
+        cp = _os.path.join(folder, "correlation.csv")
+        corr = pd.read_csv(cp, index_col=0) if _os.path.exists(cp) else None
+        mp = _os.path.join(folder, "meta.json")
+        meta = _json.load(open(mp)) if _os.path.exists(mp) else {}
+        return cls(profs, corr, meta)
+
+    def __repr__(self):
+        return f"CalibrationSet(markets={sorted(self.profiles)}, correlation={'yes' if self.corr is not None else 'no'})"
+
+
+def _group_files(files):
+    if isinstance(files, dict):
+        return {get_market(k).symbol: (v if isinstance(v, (list, tuple)) else [v]) for k, v in files.items()}
+    out = {}
+    for f in files:
+        m = _re.search(r"DAT_(?:MT|ASCII)_([A-Za-z]{6})_M1_", _os.path.basename(str(f)))
+        if not m:
+            raise ValueError(f"cannot read the market from file name '{f}'; pass a dict {{symbol: paths}} instead")
+        out.setdefault(m.group(1).upper(), []).append(f)
+    return out
+
+
+def calibrate_many(files, tz="America/New_York", variant="auto", fit_correlation=True,
+                   sim_days=30, seeds=(11, 12), verbose=True):
+    """Calibrate several markets, each from one or many monthly files, and measure their correlations.
+
+    files : {symbol: path | [paths] | DataFrame} or a list of HistData files named
+            DAT_MT_<SYMBOL>_M1_<YYYYMM>.csv (the symbol is read from the name).
+    variant : 'auto' (default) -> 'calm_aug2026' if the pooled data's volatility is below 75% of the
+            market's long-run level, else 'typical'; or force one of them.
+    The months of one market are pooled into ONE profile (no regime fitting: pooled level, pooled
+    seasonality, pooled microstructure). Derived crosses (EURGBP, EURJPY, GBPJPY) are skipped: they
+    follow their legs. The correlation matrix is measured from M15 returns of the base markets over
+    their common period, and installed by CalibrationSet.apply()."""
+    log = (lambda *a: print(*a)) if verbose else (lambda *a: None)
+    groups = _group_files(files)
+    profiles, data, meta = {}, {}, {}
+    from .profiles import get_profile
+    for sym, srcs in groups.items():
+        mk = get_market(sym)
+        if mk.legs:
+            log(f"[calibrate_many] {sym}: derived cross, skipped (follows its legs)")
+            continue
+        frames = [read_histdata(s, tz) if not isinstance(s, pd.DataFrame) else read_histdata(s, "UTC") for s in srcs]
+        df = pd.concat(frames).sort_index()
+        df = df[~df.index.duplicated(keep="first")]
+        data[sym] = df
+        r = report.returns_m1(df)
+        vol = float(r.std() * math.sqrt(260 * 1440))
+        v = variant
+        if variant == "auto":
+            v = "calm_aug2026" if vol < 0.75 * get_profile(sym).ann_vol_long else "typical"
+        log(f"[calibrate_many] {sym}: {len(srcs)} file(s), {len(df)} bars, M1-implied vol {vol:.2%} -> variant '{v}'")
+        p = calibrate(df, market=sym, tz="UTC", variant=v, sim_days=sim_days, seeds=seeds, verbose=False)
+        p.meta["n_files"] = len(srcs)
+        profiles[sym] = p
+        meta[sym] = dict(variant=v, bars=len(df), vol_m1=vol, period_utc=[str(df.index[0]), str(df.index[-1])])
+    corr = None
+    if fit_correlation:
+        from .factors import BASE
+        closes = {s: np.log(d["close"].resample("15min").last()) for s, d in data.items() if s in BASE}
+        if len(closes) >= 2:
+            R = pd.DataFrame({s: c.diff() for s, c in closes.items()}).dropna()
+            if len(R) >= 500:
+                corr = R.corr()
+                log(f"[calibrate_many] correlation from {len(R)} common M15 returns of {sorted(closes)}")
+    return CalibrationSet(profiles, corr, meta)

forexgen/cli.py

@@ -0,0 +1,108 @@
+"""Command line: python -m forexgen EURUSD M15 3m --seed 7 --out data"""
+import argparse
+import os
+import sys
+
+from . import api
+from .markets import list_markets
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="forexgen", description="Offline synthetic FX / gold data generator.")
+    ap.add_argument("market", nargs="?", help="e.g. EURUSD GBPUSD XAUUSD (see --list)")
+    ap.add_argument("timeframe", nargs="?", default="M15", help="M1 M5 M15 M30 H1 H4 D1 W1 (default M15)")
+    ap.add_argument("duration", nargs="?", default="30d", help="30d, 2w, 3m, 1y, 12h ... (default 30d)")
+    ap.add_argument("--start", default=api.DEFAULT_START, help="UTC start (default %(default)s)")
+    ap.add_argument("--end", default=None, help="UTC end (overrides duration)")
+    ap.add_argument("--seed", type=int, default=None, help="omit for different data every run; the seed used is printed")
+    ap.add_argument("--start-price", type=float, default=None)
+    ap.add_argument("--out", default="forex_data", help="output folder")
+    ap.add_argument("--file", default=None, help="exact output file path")
+    ap.add_argument("--list", action="store_true", help="list markets")
+    ap.add_argument("--profile", default="typical", help="typical | calm_aug2026 | stressed")
+    ap.add_argument("--scenario", default=None, help="flash_crash peg_break rate_shock trend_up trend_down range "
+                                                     "high_vol low_liquidity gap intervention crisis")
+    ap.add_argument("--scenario-at", default="0.5", help="timestamp or fraction 0..1 (default 0.5)")
+    ap.add_argument("--scenario-strength", type=float, default=1.0)
+    ap.add_argument("--sparse", action="store_true", help="broker-like flat bars and missing minutes")
+    ap.add_argument("--labels", action="store_true", help="add regime, event_flag, true_vol columns")
+    ap.add_argument("--ticks", action="store_true", help="write bid/ask ticks instead of bars")
+    ap.add_argument("--describe", action="store_true", help="show calibrated vs assumed parameters of the market")
+    ap.add_argument("--multi", default=None, help="comma list of markets simulated jointly, e.g. EURUSD,GBPUSD,XAUUSD")
+    ap.add_argument("--calibrate", default=None, metavar="CSV", help="fit a profile from a real HistData/MT M1 file")
+    ap.add_argument("--save-profile", default=None, help="where to save the calibrated profile (JSON)")
+    ap.add_argument("--tz", default="America/New_York", help="time zone of the calibration file")
+    ap.add_argument("--daily-anchor", default="utc", choices=["utc", "ny"], help="cut D1/W1 bars at UTC midnight or the 17:00 New York close")
+    ap.add_argument("--histdata", action="store_true", help="write M1 data in HistData layout (one file per New York month) instead of a bar CSV")
+    ap.add_argument("--histdata-fmt", default="mt", choices=["mt", "ascii"], help="mt: New York time, ascii: fixed EST")
+    ap.add_argument("--selftest", action="store_true", help="run the generator self-test (trend/mean-reversion bias, market-like statistics)")
+    a = ap.parse_args(argv)
+
+    if a.list:
+        print("\n".join(list_markets()))
+        return 0
+    if a.calibrate:
+        from .calibration import calibrate
+        p = calibrate(a.calibrate, market=a.market or "EURUSD", tz=a.tz)
+        if a.save_profile:
+            print("saved profile ->", p.save(a.save_profile))
+        p.describe()
+        return 0
+    if a.describe and a.market:
+        from .profiles import describe
+        describe(a.market)
+        return 0
+    if a.selftest:
+        from .diagnostics import selftest
+        rep = selftest(a.market or "EURUSD", profile=a.profile, seed=a.seed)
+        return 0 if rep.passed else 1
+    if not a.market and not a.multi:
+        ap.error("give a market (or --list)")
+    try:
+        at = float(a.scenario_at) if a.scenario_at.replace(".", "", 1).isdigit() else a.scenario_at
+        kw = dict(profile=a.profile, scenario=a.scenario, scenario_at=at,
+                  scenario_strength=a.scenario_strength)
+        if a.multi:
+            data = api.get_multi([m.strip() for m in a.multi.split(",")], a.timeframe, a.duration,
+                                 start=a.start, end=a.end, seed=a.seed, sparse=a.sparse,
+                                 return_labels=a.labels, **kw)
+            os.makedirs(a.out, exist_ok=True)
+            for sym, df in data.items():
+                path = os.path.join(a.out, f"{sym}_{a.timeframe}_{a.duration}_seed{df.attrs['seed']}.csv")
+                df.to_csv(path, date_format=api.CSV_DATE_FORMAT)
+                print("saved ->", path)
+            print("seed ->", next(iter(data.values())).attrs["seed"])
+            return 0
+        if a.histdata:
+            from .export import generate_histdata
+            df0 = api.get_data(a.market, "M1", a.duration, start=a.start, end=a.end, seed=a.seed,
+                               start_price=a.start_price, **kw)
+            from .export import export_histdata
+            paths = export_histdata(df0, a.out, a.market, a.histdata_fmt, None, True)
+            for p in paths:
+                print("saved ->", p)
+            print("seed ->", df0.attrs["seed"])
+            return 0
+        if a.ticks:
+            from .ticks import get_ticks
+            df = get_ticks(a.market, a.duration, start=a.start, end=a.end, seed=a.seed, **kw)
+            os.makedirs(a.out, exist_ok=True)
+            path = a.file or os.path.join(a.out, f"{a.market.upper()}_ticks_{a.duration}_seed{df.attrs['seed']}.csv")
+            df.to_csv(path, date_format="%Y-%m-%d %H:%M:%S.%f")
+        else:
+            df = api.get_data(a.market, a.timeframe, a.duration, start=a.start, end=a.end, seed=a.seed,
+                              start_price=a.start_price, sparse=a.sparse, return_labels=a.labels,
+                              daily_anchor=a.daily_anchor, **kw)
+            path = a.file or api.default_csv_path(a.market, a.timeframe, a.duration, df.attrs["seed"], a.out, a.end)
+            os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
+            df.to_csv(path, date_format=api.CSV_DATE_FORMAT)
+        print("saved ->", path)
+        print("seed ->", df.attrs["seed"])
+        return 0
+    except ValueError as e:
+        print(f"error: {e}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

forexgen/costs.py

@@ -0,0 +1,66 @@
+"""Trading-cost helpers for backtests: slippage, gap-aware stop fills, swap/rollover.
+
+All sizes are in pips of the market. These are MODELS (assumptions), not broker data.
+"""
+import numpy as np
+import pandas as pd
+
+from .marketcal import to_ny
+from .markets import get_market
+
+
+def slippage(df, market="EURUSD", order_type="market", model="vol", fixed_pips=0.2):
+    """Slippage in pips per bar.
+
+    model 'vol'  : 0.25 * spread + 0.10 * bar range  (grows with volatility; stops pay double)
+    model 'fixed': fixed_pips
+    model 'none' : 0
+    order_type   : 'market' | 'stop' | 'limit' (limit orders: 0 slippage, but may not fill)."""
+    mk = get_market(market)
+    if model == "none" or order_type == "limit":
+        return pd.Series(0.0, index=df.index)
+    if model == "fixed":
+        base = pd.Series(float(fixed_pips), index=df.index)
+    elif model == "vol":
+        rng_pips = (df["high"] - df["low"]) / mk.pip
+        base = 0.25 * df["spread_pips"] + 0.10 * rng_pips
+    else:
+        raise ValueError("model must be 'vol', 'fixed' or 'none'")
+    return base * (2.0 if order_type == "stop" else 1.0)
+
+
+def stop_fill(stop_price, side, bar_open, bar_high, bar_low, slip_pips=0.0, pip=1e-4):
+    """Fill of a stop order inside one bar. side: 'sell' (stop below the market, e.g. closing a
+    long) or 'buy' (stop above). Returns (filled: bool, price). If the bar OPENS beyond the stop
+    (a gap) the order fills at the OPEN, never at the stop price."""
+    if side == "sell":
+        if bar_low > stop_price:
+            return False, None
+        if bar_open <= stop_price:
+            return True, bar_open - slip_pips * pip
+        return True, stop_price - slip_pips * pip
+    if side == "buy":
+        if bar_high < stop_price:
+            return False, None
+        if bar_open >= stop_price:
+            return True, bar_open + slip_pips * pip
+        return True, stop_price + slip_pips * pip
+    raise ValueError("side must be 'sell' or 'buy'")
+
+
+def rollover_bars(index_utc):
+    """Boolean array: bars at the 17:00 New York rollover (Mon-Thu). Wednesday counts triple."""
+    ny = to_ny(index_utc)
+    at = (ny.hour == 17) & (ny.minute == 0) & (ny.weekday < 4)
+    return np.asarray(at), np.where(ny.weekday == 2, 3.0, 1.0)
+
+
+def swap_pips(index_utc, position, swap_long_pips, swap_short_pips):
+    """Swap earned (+) / paid (-) in pips, charged on the 17:00 NY bar for the position held
+    through the rollover; Wednesday is charged triple. `position` is a Series/array in [-1, 1]
+    aligned with the index (M1 or M5..H1 data; coarser bars may skip the 17:00 bar)."""
+    at, mult = rollover_bars(index_utc)
+    pos = np.asarray(position, float)
+    rate = np.where(pos > 0, swap_long_pips, swap_short_pips) * np.abs(pos)
+    out = np.where(at, rate * mult, 0.0)
+    return pd.Series(out, index=index_utc)