forexgen 2.1.0__tar.gz

forexgen-2.1.0/LICENSE

@@ -0,0 +1,17 @@
+MIT License
+
+Copyright (c) 2026 Henry Nyoike
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.

forexgen-2.1.0/PKG-INFO

@@ -0,0 +1,303 @@
+Metadata-Version: 2.4
+Name: forexgen
+Version: 2.1.0
+Summary: Offline synthetic forex and gold data generator for indicator testing and strategy backtesting
+Author: Henry Nyoike
+License: MIT License
+        
+        Copyright (c) 2026 Henry Nyoike
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+        
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Office/Business :: Financial :: Investment
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.26
+Requires-Dist: pandas>=2.2
+Dynamic: license-file
+
+# forexgen 2.1
+
+Offline synthetic **forex and gold** data generator for **indicator testing and strategy backtesting**
+(and, secondarily, producing training data). Pure Python, **numpy + pandas only**, no internet, no GUI.
+
+* 11 markets: EURUSD GBPUSD USDJPY AUDUSD NZDUSD USDCHF USDCAD EURGBP EURJPY GBPJPY XAUUSD (+ your own).
+* **Different data on every run by default**; the seed used is recorded so any run can be reproduced.
+* Real market structure: volatility clustering, fat tails, DST-aware sessions, scheduled news (NFP, CPI,
+  FOMC, ECB, BoE ...), regimes, weekend gaps, spreads that widen at news/rollover, intrabar ticks with bid/ask.
+* Several markets simulated **jointly** (shared factors, correlations that rise in crises); crosses derived
+  **exactly** from their legs.
+* Backtest-validation tools: luck-only ("null") paths, planted-edge paths, Monte Carlo of your strategy.
+* `calibrate()` fits a market to a real HistData/MetaTrader M1 file; `calibrate_many()` does it for several markets and
+  months at once and measures their correlations; `scorecard()` compares synthetic with real.
+* **New in 2.1:** `selftest()` (is the generator itself giving strategies a free edge?), `robustness_report(strategy)`
+  (one-page verdict), indicator edge-case `fixtures` + `indicator_stress_test`, daily bars cut at the **17:00 New York
+  close** (`daily_anchor="ny"`), **HistData/MetaTrader file writers** (`export_histdata`, `generate_histdata`),
+  correlation control (`set_correlation`), and shared microstructure noise so tightly coupled pairs (EURUSD/USDCHF)
+  correlate more like real markets.
+
+> **Read this first.** Only **EURUSD** was calibrated to real data (one real month, Aug 2026, 28,643 M1 bars).
+> Every other market is built from **documented assumptions** about its typical behaviour, scaled from the
+> EURUSD baseline, and says so in `forexgen.describe("GBPUSD")`. Synthetic data can test your **logic,
+> robustness and cost sensitivity**; it **cannot prove a strategy makes money live** (no real news,
+> central-bank behaviour or regime history). See [Limitations](#limitations).
+
+## Install
+
+```bash
+pip install forexgen
+python -m pytest tests   # optional
+```
+
+## Quick start
+
+```python
+import forexgen
+
+df = forexgen.get_data("EURUSD", "M15", "3m")          # different every call
+print(df.attrs["seed"])                                 # ... but reproducible
+df = forexgen.get_data("EURUSD", "M15", "3m", seed=df.attrs["seed"])
+
+forexgen.generate_csv("GBPUSD", "H1", "1y", seed=7, folder="data")     # data/GBPUSD_H1_1y_seed7.csv
+data = forexgen.get_multi(["EURUSD", "GBPUSD", "XAUUSD", "EURGBP"], "M15", "1y", seed=1)
+forexgen.describe("XAUUSD")                             # calibrated vs assumed
+```
+
+Command line: `python -m forexgen EURUSD M15 3m --seed 7 --out data` (`--list`, `--profile`, `--scenario`,
+`--sparse`, `--labels`, `--ticks`, `--multi EURUSD,GBPUSD`, `--describe`, `--calibrate FILE`). The seed used is printed.
+
+## Output format (unchanged from v1)
+
+`time` index/column = **naive UTC**, format `%Y-%m-%d %H:%M:%S`; columns `open, high, low, close, volume, spread_pips`
+(`volume` = tick volume, int). Prices are bid-style OHLC rounded to the market's digits. All timeframes are
+aggregated from the same M1 path, so they agree.
+
+## API
+
+### `get_data(market, timeframe="M15", duration="30d", start="2024-01-01 00:00", end=None, seed=None, start_price=None, **options)`
+
+| argument | meaning |
+|---|---|
+| `market` | e.g. `"EURUSD"`, `"XAUUSD"` (case/slash-insensitive: `"eur/usd"`). `list_markets()` |
+| `timeframe` | `M1 M5 M15 M30 H1 H4 D1 W1` (any `M<n>`/`H<n>`; also `15m`, `1h`, `daily`) |
+| `duration` | `30d`, `2w`, `3m`, `1y`, `12h`, `90min` (**market time**: 1w = 5 days, 1m = 21 days, 1y = 260 days) |
+| `start`, `end` | naive UTC timestamps; `end` overrides `duration` |
+| `seed` | `None` (default): fresh data every call. An integer reproduces the data exactly. Stored in `df.attrs["seed"]` |
+| `start_price` | override the first price (for crosses it adjusts the first leg) |
+| `profile` | `"typical"` (default), `"calm_aug2026"` (the real calm month's level), `"stressed"`, or a `Profile` |
+| `scenario`, `scenario_at`, `scenario_strength`, `scenario_direction`, `scenario_markets` | inject a stress scenario (below) |
+| `events` | `"default"` (rule calendar), `"none"`, or a DataFrame / CSV of your own events |
+| `sparse` | `True`: broker-like flat bars and missing minutes (measured EURUSD rates) |
+| `return_labels` | adds `regime`, `event_flag`, `true_vol` columns |
+| `edge` | `dict(kind="momentum"\|"mean_reversion", strength=0.02, horizon="1h")` plants a known edge |
+| `**model` | v1 options still accepted: `garch=(a,b)`, `t_df`, `jump_rate_per_day`, `jump_sigma`, `weekend_gap`, `kappa`, `drift`, `usd_factor`; plus `vol_scale` |
+
+How the v1 options map: `garch` sets the GARCH alpha/beta; `t_df` the Student-t tail; `jump_rate_per_day` /
+`jump_sigma` the **residual unscheduled** shock process (scheduled news now comes from the calendar);
+`weekend_gap` is the weekend-gap standard deviation in per-minute sigmas; `kappa` the mean-reversion speed;
+`drift` an annual log drift; `usd_factor=False` removes the cross-market factor.
+
+### `generate_csv(market, timeframe="M15", duration="30d", path=None, folder="forex_data", **kwargs)`
+Writes the CSV and returns its path. Default name `<folder>/<SYMBOL>_<TF>_<duration>_seed<seed>.csv`
+(`..._to-<digits of end>_...` when `end` is given). With `seed=None` the seed actually used is in the name.
+
+### `get_multi(markets, timeframe, duration, start, end, seed, profile, as_frame=False, **options)`
+Joint simulation. Returns `{symbol: DataFrame}` (or one MultiIndex-column DataFrame). **A market inside
+`get_multi` equals `get_data(market, seed=seed)`** for the same arguments. Crosses are derived exactly:
+`EURGBP = EURUSD / GBPUSD`, `EURJPY = EURUSD * USDJPY`, `GBPJPY = GBPUSD * USDJPY`.
+
+### `get_ticks(market, duration="1d", start, end, seed, profile, ticks_per_minute="auto", ...)`
+Bid/ask ticks inside every M1 bar on a Brownian-bridge path **conditioned on the bar's OHLC** (it visits the
+high and low in a defined order), so stop/target order inside a bar is decided by a path, not a guess. Bars
+rebuilt from the bid ticks equal `get_data(..., "M1")` for the same seed. ~100 µs per minute bar (off by default).
+
+### Events: `get_events`, `register_event`, `set_events`
+`get_events("EURUSD", start, end)` returns the calendar used for that market (with its sensitivity).
+`register_event(name, time_utc, importance=3, currencies=("USD",))` adds one event; `set_events(df_or_csv)`
+replaces the rule calendar (columns `time_utc, name, importance, currencies`); `set_events(None)` restores it.
+**All default dates are rule-based approximations, not the official calendar:**
+
+| event | rule (local time) | importance |
+|---|---|---|
+| US NFP | first Friday 08:30 NY | 5 |
+| US CPI | Wednesday in 11th-17th, 08:30 NY | 4 |
+| US FOMC | 8/yr, Wednesday 14:00 NY | 5 |
+| US retail sales / PCE / ISM / UMich | 08:30 or 10:00 NY | 2-3 |
+| Jackson Hole | Friday in 22nd-28th of August, 10:00 NY | 4 |
+| ECB | 8/yr, Thursday 14:15 Frankfurt | 4 |
+| BoE | 8/yr, Thursday 12:00 London | 4 |
+| BoJ, RBA, RBNZ, SNB, BoC | 3-8/yr, local release times | 3 |
+| US jobless claims | every Thursday 08:30 NY | 1 |
+
+Reaction (assumed shape; sizes anchored to the 8 events observed in the real file): vol x0.75 in the 30 minutes
+before, a random-sign heavy-tailed shock (80/12/5/3% over the first 4 minutes), power-law volatility decay for
+~4 hours, spread widening. USD events move every USD pair (opposite sign for USDxxx vs xxxUSD, x1.5 for gold);
+other currencies' events move the pairs that contain them.
+
+### Scenarios
+`scenario=` one of `flash_crash, peg_break, rate_shock, trend_up, trend_down, range, high_vol, low_liquidity,
+gap, intervention, crisis`, placed at `scenario_at` (timestamp, or a fraction 0..1 of the data) and scaled by
+`scenario_strength`. They are **parameter transformations of the base model** (jumps, volatility/spread
+multipliers, drift, mean reversion, forced global stress), documented in `forexgen/scenarios.py`.
+`crisis` also forces the global stress state, so correlations rise toward 1 in `get_multi`.
+`scenario_markets=[...]` restricts which markets get the scenario; `scenario_direction="up"|"down"|"auto"`.
+
+### Backtest validation tools
+```python
+null = forexgen.monte_carlo(strategy, "EURUSD", "M5", "1m", n=200, mode="null")    # luck only
+edge = forexgen.monte_carlo(strategy, "EURUSD", "M5", "1m", n=50, mode="edge")     # planted edge
+reg  = forexgen.monte_carlo(strategy, "EURUSD", "M5", "1m", n=50, mode="regular")
+forexgen.percentile_in_null(my_real_backtest_return, null)                          # >95 = unlikely luck
+paths = forexgen.get_null_paths("EURUSD", "H1", "3m", n=100)                        # the raw null paths
+```
+`strategy(df) -> pandas.Series` of target positions in [-1, 1]. Positions decided on bar *t*'s close are
+executed at bar *t+1*'s open and pay half the bar's spread per unit of position change (+ `commission_pips`).
+Returned metrics per path: `total_return, max_drawdown, sharpe, n_trades, cost_paid`.
+**Null paths** keep volatility clustering, tails, spreads and bar shapes but give every bar and every gap an
+independent random sign, so trends, momentum, mean reversion and drift cannot exist.
+`forexgen.backtest(df, positions, pip, ...)` is the same engine for your own paths.
+Costs: `slippage(df, market, order_type, model)`, `stop_fill(...)` (a stop that gaps fills at the gap, never at
+the stop price), `swap_pips(...)` (17:00 NY rollover, Wednesday triple).
+
+### Calibration and profiles
+```python
+profile = forexgen.calibrate("DAT_MT_GBPUSD_M1_202608.csv", market="GBPUSD", tz="America/New_York")
+forexgen.use_profile(profile)                 # or register_market(..., profile=profile)
+profile.save("gbpusd.json"); forexgen.load_profile("gbpusd.json")
+real = forexgen.read_histdata(path)           # UTC M1 DataFrame
+forexgen.scorecard(forexgen.get_data("GBPUSD", "M1", "6m", profile="calm_aug2026"), "GBPUSD", reference=real)
+```
+`calibrate` reads the HistData/MT layout (no header: `YYYY.MM.DD,HH:MM,o,h,l,c,v`), converts the **New York
+clock (DST-aware)** to UTC, and fits: volatility level, event-free seasonality, microstructure (lag-1 bounce and
+variance ratios), GARCH persistence (simulated method of moments), bar-range scale, weekend gap, flat-bar and
+missing-minute rates, and event sizes **only when >= 8 events of a class exist**. It cannot fit spreads or volume
+from a price-only file (kept as "assumed").
+
+## New in 2.1
+
+### Is the generator itself fair? `selftest()`
+```python
+report = forexgen.selftest("EURUSD", profile="typical")       # ~30 s
+print(report.passed, report.warnings)
+```
+Checks market-like statistics on M1 paths (variance ratios - **a trend-biased generator has VR > 1**, lag-1
+autocorrelation, volatility clustering, fat tails, OHLC validity) and a **free-edge test**: a plain trend strategy and a plain
+mean-reversion strategy run on ordinary paths and on luck-only paths; if ordinary paths pay either one significantly more
+than luck (z > 3.5) the generator is leaking an edge and the check FAILS. It also works as a guard for your own
+`profile=` / `calibrate()` results: `forexgen.selftest("GBPUSD", profile=my_profile)`. CLI: `python -m forexgen --selftest`.
+
+### One-page verdict for your strategy: `robustness_report(strategy, ...)`
+```python
+rep = forexgen.robustness_report(my_strategy, "EURUSD", "H1", "3m", n=30, commission_pips=0.3, real_result=0.04)
+rep.tables["costs"], rep.tables["stress"], rep.passed
+```
+Runs the strategy on regular paths, luck-only paths, planted momentum and mean-reversion edges, ten stress scenarios and a
+cost sweep (free / 1x / 2x / 3x spread), then prints a verdict: where your real result ranks among luck-only paths, whether the
+strategy can exploit a known edge, whether it is cost-fragile, and which scenarios hurt far more than normal conditions.
+It cannot prove the strategy works live.
+
+### Indicator edge-case fixtures
+```python
+from forexgen import fixtures
+report = forexgen.indicator_stress_test(my_rsi, timeframe="M5", bounds=(0, 100))   # one row per fixture
+fixtures.fixture("flat", "M15", 500)                                                # deterministic data
+```
+16 reproducible fixtures (`fixtures.FIXTURES`): flat, one-way trends, staircase, saw-tooth, huge gap, spike-and-reverse,
+zero-range bars, volatility burst, missing bars, tiny/huge prices, one/two bars, NaN inside the input. The tester reports
+exceptions, NaN **after** warm-up, inf, wrong output length and out-of-bounds values.
+
+### Daily bars at the New York close
+`get_data(..., "D1", daily_anchor="ny")` (and `W1`) cuts bars at **17:00 New York** (Sunday's session belongs to Monday's bar;
+5 bars per week; the timestamp is the bar's open in UTC, 21:00 or 22:00). Default stays `"utc"` (as in v1). CLI `--daily-anchor ny`.
+
+### HistData / MetaTrader writers
+```python
+forexgen.generate_histdata("EURUSD", "3m", folder="out", seed=7)          # DAT_MT_EURUSD_M1_202401.csv ... per New York month
+forexgen.export_histdata(df_m1, "out/x.csv", "EURUSD", fmt="mt")           # or fmt="ascii" (HistData ASCII, fixed EST)
+```
+`fmt="mt"`: `YYYY.MM.DD,HH:MM,o,h,l,c,v`, New York local time with DST (same as the real Aug-2026 file). `fmt="ascii"`:
+`YYYYMMDD HHMMSS;o;h;l;c;v`, fixed EST (UTC-5). Prices have digits+1 decimals; `volume="zero"` mimics the real MT files. Files
+read back exactly with `read_histdata` (both layouts are auto-detected, including across the November DST change). CLI: `--histdata`.
+
+### Calibrating many months / markets
+```python
+cs = forexgen.calibrate_many(glob.glob("real/DAT_MT_*_M1_*.csv"))   # symbol read from the file name, or {symbol: paths}
+cs.apply(); cs.save("my_calibration"); forexgen.CalibrationSet.load("my_calibration")
+forexgen.set_correlation(cs.corr); forexgen.reset_correlation()
+```
+Months of one market are pooled into ONE profile (pooled level, seasonality, microstructure; **no regime fitting**); the
+variance level is calm/typical automatically. The correlation matrix is measured from M15 returns of the base markets over
+their common period and installed by `apply()` (`set_correlation` scales your desired matrix by the attenuation found when
+tuning the defaults, so realised correlations follow it). Derived crosses are skipped (they follow their legs).
+
+### Tested environments
+Full test-suite (see REPORT.md) on **pandas 2.2.3 / numpy 1.26.4** and **pandas 3.0.2 / numpy 2.4.4**, Python 3.12.
+Older pandas/numpy are untested.
+
+## Markets: what is calibrated and what is assumed
+
+| | EURUSD | all other markets |
+|---|---|---|
+| calm volatility level (4.4%) | **calibrated** (Aug-2026) | assumed (0.63 x long-run) |
+| long-run volatility level | assumed (7%) | assumed (GBPUSD 8.5, AUDUSD 10, NZDUSD 10.5, USDJPY 9.5, USDCHF 8, USDCAD 6.5, XAUUSD 16 %) |
+| hourly (NY) seasonality | **calibrated** (events excluded) | assumed (EURUSD curve reshaped for each market's sessions) |
+| microstructure (bounce, variance ratios), GARCH, bar range, weekend gap, flat bars, missing minutes | **calibrated** | assumed (copied from EURUSD) |
+| event shock sizes | assumed, sanity-checked against 8 observed events | assumed (scaled by volatility ratio) |
+| spreads, volume | assumed (price-only file) | assumed |
+| regimes, scenarios, correlations | assumed | assumed (latent matrix tuned so realised M15 correlations land near typical values) |
+| EURGBP, EURJPY, GBPJPY | derived exactly from their legs (no separate parameters) | |
+
+`forexgen.describe("GBPUSD")` prints this per market. Upgrade any market with `calibrate()` on a real file.
+
+## Compatibility notes (changes from v1.0.0)
+
+Kept: every function name and signature, the CSV layout and timestamp format, the CLI, durations/timeframes, the
+11 markets, `register_market`, the v1 model options, and "same seed -> same first N days whatever the duration".
+Changed on purpose:
+* **`seed` now defaults to `None`** (v1: 42): data differs every call. Pass an integer to get v1-style repeatability.
+* **Market hours** are Sunday 17:00 -> Friday 17:00 **New York time, DST-aware** (v1: fixed 22:00 UTC).
+* The old random news jumps are replaced by the scheduled-event calendar plus a small residual shock process.
+* Default start prices are unchanged (e.g. EURUSD 1.0850); the calibrated *behaviour* comes from the Aug-2026 data.
+* Extra columns only appear when asked for (`return_labels=True`).
+* Daily bars are still cut at **UTC midnight** by default (as in v1); use `daily_anchor="ny"` for the 17:00 New York close.
+
+## Reproducibility rules
+An explicit `seed` gives identical output for identical arguments. Every random stream is keyed by
+(seed, stream, day-block) - and shared factors by absolute UTC time - so a 60-day request starts with exactly the
+same 30 days as a 30-day request, and a market alone equals the same market inside `get_multi`.
+Scenario positions given as a *fraction* depend on the duration; give a timestamp to keep prefix-stability.
+
+## Limitations
+* One real month of one market is the only real-data anchor. Non-EURUSD markets are plausible, not fitted;
+  `calibrate_many` has only been tested on synthetic multi-market files, never on real multi-month data.
+* A calm month cannot teach crashes or regime history; `stressed`, regimes and scenarios are assumptions.
+* Event dates are rule-based approximations; event sizes for importance classes are assumptions.
+* Realised cross-market correlations are close to, not exactly, the targets (EURUSD/USDCHF reaches about -0.83
+  against a -0.90 target); `scorecard`/your own checks report actuals.
+* Scheduled-event shock sizes use an uncapped lognormal, so the extreme tail is probably too fat: in the typical profile a
+  single 3-month path can contain a 40-95 bp one-minute move on an important release (real EURUSD rarely exceeds ~50 bp).
+* The model has no central-bank regime shifts, trending macro cycles, or order-book dynamics.
+* Spread and volume are models, not broker data. Tick generation is slow.
+
+## Overfitting warning
+Tuning a strategy until it looks good on synthetic data (or on a handful of paths) proves nothing. Always check a
+result against `monte_carlo(mode="null")`, keep out-of-sample paths you never tuned on, include costs, and validate
+on **real** data and a forward test before risking money.

forexgen-2.1.0/README.md

@@ -0,0 +1,267 @@
+# forexgen 2.1
+
+Offline synthetic **forex and gold** data generator for **indicator testing and strategy backtesting**
+(and, secondarily, producing training data). Pure Python, **numpy + pandas only**, no internet, no GUI.
+
+* 11 markets: EURUSD GBPUSD USDJPY AUDUSD NZDUSD USDCHF USDCAD EURGBP EURJPY GBPJPY XAUUSD (+ your own).
+* **Different data on every run by default**; the seed used is recorded so any run can be reproduced.
+* Real market structure: volatility clustering, fat tails, DST-aware sessions, scheduled news (NFP, CPI,
+  FOMC, ECB, BoE ...), regimes, weekend gaps, spreads that widen at news/rollover, intrabar ticks with bid/ask.
+* Several markets simulated **jointly** (shared factors, correlations that rise in crises); crosses derived
+  **exactly** from their legs.
+* Backtest-validation tools: luck-only ("null") paths, planted-edge paths, Monte Carlo of your strategy.
+* `calibrate()` fits a market to a real HistData/MetaTrader M1 file; `calibrate_many()` does it for several markets and
+  months at once and measures their correlations; `scorecard()` compares synthetic with real.
+* **New in 2.1:** `selftest()` (is the generator itself giving strategies a free edge?), `robustness_report(strategy)`
+  (one-page verdict), indicator edge-case `fixtures` + `indicator_stress_test`, daily bars cut at the **17:00 New York
+  close** (`daily_anchor="ny"`), **HistData/MetaTrader file writers** (`export_histdata`, `generate_histdata`),
+  correlation control (`set_correlation`), and shared microstructure noise so tightly coupled pairs (EURUSD/USDCHF)
+  correlate more like real markets.
+
+> **Read this first.** Only **EURUSD** was calibrated to real data (one real month, Aug 2026, 28,643 M1 bars).
+> Every other market is built from **documented assumptions** about its typical behaviour, scaled from the
+> EURUSD baseline, and says so in `forexgen.describe("GBPUSD")`. Synthetic data can test your **logic,
+> robustness and cost sensitivity**; it **cannot prove a strategy makes money live** (no real news,
+> central-bank behaviour or regime history). See [Limitations](#limitations).
+
+## Install
+
+```bash
+pip install forexgen
+python -m pytest tests   # optional
+```
+
+## Quick start
+
+```python
+import forexgen
+
+df = forexgen.get_data("EURUSD", "M15", "3m")          # different every call
+print(df.attrs["seed"])                                 # ... but reproducible
+df = forexgen.get_data("EURUSD", "M15", "3m", seed=df.attrs["seed"])
+
+forexgen.generate_csv("GBPUSD", "H1", "1y", seed=7, folder="data")     # data/GBPUSD_H1_1y_seed7.csv
+data = forexgen.get_multi(["EURUSD", "GBPUSD", "XAUUSD", "EURGBP"], "M15", "1y", seed=1)
+forexgen.describe("XAUUSD")                             # calibrated vs assumed
+```
+
+Command line: `python -m forexgen EURUSD M15 3m --seed 7 --out data` (`--list`, `--profile`, `--scenario`,
+`--sparse`, `--labels`, `--ticks`, `--multi EURUSD,GBPUSD`, `--describe`, `--calibrate FILE`). The seed used is printed.
+
+## Output format (unchanged from v1)
+
+`time` index/column = **naive UTC**, format `%Y-%m-%d %H:%M:%S`; columns `open, high, low, close, volume, spread_pips`
+(`volume` = tick volume, int). Prices are bid-style OHLC rounded to the market's digits. All timeframes are
+aggregated from the same M1 path, so they agree.
+
+## API
+
+### `get_data(market, timeframe="M15", duration="30d", start="2024-01-01 00:00", end=None, seed=None, start_price=None, **options)`
+
+| argument | meaning |
+|---|---|
+| `market` | e.g. `"EURUSD"`, `"XAUUSD"` (case/slash-insensitive: `"eur/usd"`). `list_markets()` |
+| `timeframe` | `M1 M5 M15 M30 H1 H4 D1 W1` (any `M<n>`/`H<n>`; also `15m`, `1h`, `daily`) |
+| `duration` | `30d`, `2w`, `3m`, `1y`, `12h`, `90min` (**market time**: 1w = 5 days, 1m = 21 days, 1y = 260 days) |
+| `start`, `end` | naive UTC timestamps; `end` overrides `duration` |
+| `seed` | `None` (default): fresh data every call. An integer reproduces the data exactly. Stored in `df.attrs["seed"]` |
+| `start_price` | override the first price (for crosses it adjusts the first leg) |
+| `profile` | `"typical"` (default), `"calm_aug2026"` (the real calm month's level), `"stressed"`, or a `Profile` |
+| `scenario`, `scenario_at`, `scenario_strength`, `scenario_direction`, `scenario_markets` | inject a stress scenario (below) |
+| `events` | `"default"` (rule calendar), `"none"`, or a DataFrame / CSV of your own events |
+| `sparse` | `True`: broker-like flat bars and missing minutes (measured EURUSD rates) |
+| `return_labels` | adds `regime`, `event_flag`, `true_vol` columns |
+| `edge` | `dict(kind="momentum"\|"mean_reversion", strength=0.02, horizon="1h")` plants a known edge |
+| `**model` | v1 options still accepted: `garch=(a,b)`, `t_df`, `jump_rate_per_day`, `jump_sigma`, `weekend_gap`, `kappa`, `drift`, `usd_factor`; plus `vol_scale` |
+
+How the v1 options map: `garch` sets the GARCH alpha/beta; `t_df` the Student-t tail; `jump_rate_per_day` /
+`jump_sigma` the **residual unscheduled** shock process (scheduled news now comes from the calendar);
+`weekend_gap` is the weekend-gap standard deviation in per-minute sigmas; `kappa` the mean-reversion speed;
+`drift` an annual log drift; `usd_factor=False` removes the cross-market factor.
+
+### `generate_csv(market, timeframe="M15", duration="30d", path=None, folder="forex_data", **kwargs)`
+Writes the CSV and returns its path. Default name `<folder>/<SYMBOL>_<TF>_<duration>_seed<seed>.csv`
+(`..._to-<digits of end>_...` when `end` is given). With `seed=None` the seed actually used is in the name.
+
+### `get_multi(markets, timeframe, duration, start, end, seed, profile, as_frame=False, **options)`
+Joint simulation. Returns `{symbol: DataFrame}` (or one MultiIndex-column DataFrame). **A market inside
+`get_multi` equals `get_data(market, seed=seed)`** for the same arguments. Crosses are derived exactly:
+`EURGBP = EURUSD / GBPUSD`, `EURJPY = EURUSD * USDJPY`, `GBPJPY = GBPUSD * USDJPY`.
+
+### `get_ticks(market, duration="1d", start, end, seed, profile, ticks_per_minute="auto", ...)`
+Bid/ask ticks inside every M1 bar on a Brownian-bridge path **conditioned on the bar's OHLC** (it visits the
+high and low in a defined order), so stop/target order inside a bar is decided by a path, not a guess. Bars
+rebuilt from the bid ticks equal `get_data(..., "M1")` for the same seed. ~100 µs per minute bar (off by default).
+
+### Events: `get_events`, `register_event`, `set_events`
+`get_events("EURUSD", start, end)` returns the calendar used for that market (with its sensitivity).
+`register_event(name, time_utc, importance=3, currencies=("USD",))` adds one event; `set_events(df_or_csv)`
+replaces the rule calendar (columns `time_utc, name, importance, currencies`); `set_events(None)` restores it.
+**All default dates are rule-based approximations, not the official calendar:**
+
+| event | rule (local time) | importance |
+|---|---|---|
+| US NFP | first Friday 08:30 NY | 5 |
+| US CPI | Wednesday in 11th-17th, 08:30 NY | 4 |
+| US FOMC | 8/yr, Wednesday 14:00 NY | 5 |
+| US retail sales / PCE / ISM / UMich | 08:30 or 10:00 NY | 2-3 |
+| Jackson Hole | Friday in 22nd-28th of August, 10:00 NY | 4 |
+| ECB | 8/yr, Thursday 14:15 Frankfurt | 4 |
+| BoE | 8/yr, Thursday 12:00 London | 4 |
+| BoJ, RBA, RBNZ, SNB, BoC | 3-8/yr, local release times | 3 |
+| US jobless claims | every Thursday 08:30 NY | 1 |
+
+Reaction (assumed shape; sizes anchored to the 8 events observed in the real file): vol x0.75 in the 30 minutes
+before, a random-sign heavy-tailed shock (80/12/5/3% over the first 4 minutes), power-law volatility decay for
+~4 hours, spread widening. USD events move every USD pair (opposite sign for USDxxx vs xxxUSD, x1.5 for gold);
+other currencies' events move the pairs that contain them.
+
+### Scenarios
+`scenario=` one of `flash_crash, peg_break, rate_shock, trend_up, trend_down, range, high_vol, low_liquidity,
+gap, intervention, crisis`, placed at `scenario_at` (timestamp, or a fraction 0..1 of the data) and scaled by
+`scenario_strength`. They are **parameter transformations of the base model** (jumps, volatility/spread
+multipliers, drift, mean reversion, forced global stress), documented in `forexgen/scenarios.py`.
+`crisis` also forces the global stress state, so correlations rise toward 1 in `get_multi`.
+`scenario_markets=[...]` restricts which markets get the scenario; `scenario_direction="up"|"down"|"auto"`.
+
+### Backtest validation tools
+```python
+null = forexgen.monte_carlo(strategy, "EURUSD", "M5", "1m", n=200, mode="null")    # luck only
+edge = forexgen.monte_carlo(strategy, "EURUSD", "M5", "1m", n=50, mode="edge")     # planted edge
+reg  = forexgen.monte_carlo(strategy, "EURUSD", "M5", "1m", n=50, mode="regular")
+forexgen.percentile_in_null(my_real_backtest_return, null)                          # >95 = unlikely luck
+paths = forexgen.get_null_paths("EURUSD", "H1", "3m", n=100)                        # the raw null paths
+```
+`strategy(df) -> pandas.Series` of target positions in [-1, 1]. Positions decided on bar *t*'s close are
+executed at bar *t+1*'s open and pay half the bar's spread per unit of position change (+ `commission_pips`).
+Returned metrics per path: `total_return, max_drawdown, sharpe, n_trades, cost_paid`.
+**Null paths** keep volatility clustering, tails, spreads and bar shapes but give every bar and every gap an
+independent random sign, so trends, momentum, mean reversion and drift cannot exist.
+`forexgen.backtest(df, positions, pip, ...)` is the same engine for your own paths.
+Costs: `slippage(df, market, order_type, model)`, `stop_fill(...)` (a stop that gaps fills at the gap, never at
+the stop price), `swap_pips(...)` (17:00 NY rollover, Wednesday triple).
+
+### Calibration and profiles
+```python
+profile = forexgen.calibrate("DAT_MT_GBPUSD_M1_202608.csv", market="GBPUSD", tz="America/New_York")
+forexgen.use_profile(profile)                 # or register_market(..., profile=profile)
+profile.save("gbpusd.json"); forexgen.load_profile("gbpusd.json")
+real = forexgen.read_histdata(path)           # UTC M1 DataFrame
+forexgen.scorecard(forexgen.get_data("GBPUSD", "M1", "6m", profile="calm_aug2026"), "GBPUSD", reference=real)
+```
+`calibrate` reads the HistData/MT layout (no header: `YYYY.MM.DD,HH:MM,o,h,l,c,v`), converts the **New York
+clock (DST-aware)** to UTC, and fits: volatility level, event-free seasonality, microstructure (lag-1 bounce and
+variance ratios), GARCH persistence (simulated method of moments), bar-range scale, weekend gap, flat-bar and
+missing-minute rates, and event sizes **only when >= 8 events of a class exist**. It cannot fit spreads or volume
+from a price-only file (kept as "assumed").
+
+## New in 2.1
+
+### Is the generator itself fair? `selftest()`
+```python
+report = forexgen.selftest("EURUSD", profile="typical")       # ~30 s
+print(report.passed, report.warnings)
+```
+Checks market-like statistics on M1 paths (variance ratios - **a trend-biased generator has VR > 1**, lag-1
+autocorrelation, volatility clustering, fat tails, OHLC validity) and a **free-edge test**: a plain trend strategy and a plain
+mean-reversion strategy run on ordinary paths and on luck-only paths; if ordinary paths pay either one significantly more
+than luck (z > 3.5) the generator is leaking an edge and the check FAILS. It also works as a guard for your own
+`profile=` / `calibrate()` results: `forexgen.selftest("GBPUSD", profile=my_profile)`. CLI: `python -m forexgen --selftest`.
+
+### One-page verdict for your strategy: `robustness_report(strategy, ...)`
+```python
+rep = forexgen.robustness_report(my_strategy, "EURUSD", "H1", "3m", n=30, commission_pips=0.3, real_result=0.04)
+rep.tables["costs"], rep.tables["stress"], rep.passed
+```
+Runs the strategy on regular paths, luck-only paths, planted momentum and mean-reversion edges, ten stress scenarios and a
+cost sweep (free / 1x / 2x / 3x spread), then prints a verdict: where your real result ranks among luck-only paths, whether the
+strategy can exploit a known edge, whether it is cost-fragile, and which scenarios hurt far more than normal conditions.
+It cannot prove the strategy works live.
+
+### Indicator edge-case fixtures
+```python
+from forexgen import fixtures
+report = forexgen.indicator_stress_test(my_rsi, timeframe="M5", bounds=(0, 100))   # one row per fixture
+fixtures.fixture("flat", "M15", 500)                                                # deterministic data
+```
+16 reproducible fixtures (`fixtures.FIXTURES`): flat, one-way trends, staircase, saw-tooth, huge gap, spike-and-reverse,
+zero-range bars, volatility burst, missing bars, tiny/huge prices, one/two bars, NaN inside the input. The tester reports
+exceptions, NaN **after** warm-up, inf, wrong output length and out-of-bounds values.
+
+### Daily bars at the New York close
+`get_data(..., "D1", daily_anchor="ny")` (and `W1`) cuts bars at **17:00 New York** (Sunday's session belongs to Monday's bar;
+5 bars per week; the timestamp is the bar's open in UTC, 21:00 or 22:00). Default stays `"utc"` (as in v1). CLI `--daily-anchor ny`.
+
+### HistData / MetaTrader writers
+```python
+forexgen.generate_histdata("EURUSD", "3m", folder="out", seed=7)          # DAT_MT_EURUSD_M1_202401.csv ... per New York month
+forexgen.export_histdata(df_m1, "out/x.csv", "EURUSD", fmt="mt")           # or fmt="ascii" (HistData ASCII, fixed EST)
+```
+`fmt="mt"`: `YYYY.MM.DD,HH:MM,o,h,l,c,v`, New York local time with DST (same as the real Aug-2026 file). `fmt="ascii"`:
+`YYYYMMDD HHMMSS;o;h;l;c;v`, fixed EST (UTC-5). Prices have digits+1 decimals; `volume="zero"` mimics the real MT files. Files
+read back exactly with `read_histdata` (both layouts are auto-detected, including across the November DST change). CLI: `--histdata`.
+
+### Calibrating many months / markets
+```python
+cs = forexgen.calibrate_many(glob.glob("real/DAT_MT_*_M1_*.csv"))   # symbol read from the file name, or {symbol: paths}
+cs.apply(); cs.save("my_calibration"); forexgen.CalibrationSet.load("my_calibration")
+forexgen.set_correlation(cs.corr); forexgen.reset_correlation()
+```
+Months of one market are pooled into ONE profile (pooled level, seasonality, microstructure; **no regime fitting**); the
+variance level is calm/typical automatically. The correlation matrix is measured from M15 returns of the base markets over
+their common period and installed by `apply()` (`set_correlation` scales your desired matrix by the attenuation found when
+tuning the defaults, so realised correlations follow it). Derived crosses are skipped (they follow their legs).
+
+### Tested environments
+Full test-suite (see REPORT.md) on **pandas 2.2.3 / numpy 1.26.4** and **pandas 3.0.2 / numpy 2.4.4**, Python 3.12.
+Older pandas/numpy are untested.
+
+## Markets: what is calibrated and what is assumed
+
+| | EURUSD | all other markets |
+|---|---|---|
+| calm volatility level (4.4%) | **calibrated** (Aug-2026) | assumed (0.63 x long-run) |
+| long-run volatility level | assumed (7%) | assumed (GBPUSD 8.5, AUDUSD 10, NZDUSD 10.5, USDJPY 9.5, USDCHF 8, USDCAD 6.5, XAUUSD 16 %) |
+| hourly (NY) seasonality | **calibrated** (events excluded) | assumed (EURUSD curve reshaped for each market's sessions) |
+| microstructure (bounce, variance ratios), GARCH, bar range, weekend gap, flat bars, missing minutes | **calibrated** | assumed (copied from EURUSD) |
+| event shock sizes | assumed, sanity-checked against 8 observed events | assumed (scaled by volatility ratio) |
+| spreads, volume | assumed (price-only file) | assumed |
+| regimes, scenarios, correlations | assumed | assumed (latent matrix tuned so realised M15 correlations land near typical values) |
+| EURGBP, EURJPY, GBPJPY | derived exactly from their legs (no separate parameters) | |
+
+`forexgen.describe("GBPUSD")` prints this per market. Upgrade any market with `calibrate()` on a real file.
+
+## Compatibility notes (changes from v1.0.0)
+
+Kept: every function name and signature, the CSV layout and timestamp format, the CLI, durations/timeframes, the
+11 markets, `register_market`, the v1 model options, and "same seed -> same first N days whatever the duration".
+Changed on purpose:
+* **`seed` now defaults to `None`** (v1: 42): data differs every call. Pass an integer to get v1-style repeatability.
+* **Market hours** are Sunday 17:00 -> Friday 17:00 **New York time, DST-aware** (v1: fixed 22:00 UTC).
+* The old random news jumps are replaced by the scheduled-event calendar plus a small residual shock process.
+* Default start prices are unchanged (e.g. EURUSD 1.0850); the calibrated *behaviour* comes from the Aug-2026 data.
+* Extra columns only appear when asked for (`return_labels=True`).
+* Daily bars are still cut at **UTC midnight** by default (as in v1); use `daily_anchor="ny"` for the 17:00 New York close.
+
+## Reproducibility rules
+An explicit `seed` gives identical output for identical arguments. Every random stream is keyed by
+(seed, stream, day-block) - and shared factors by absolute UTC time - so a 60-day request starts with exactly the
+same 30 days as a 30-day request, and a market alone equals the same market inside `get_multi`.
+Scenario positions given as a *fraction* depend on the duration; give a timestamp to keep prefix-stability.
+
+## Limitations
+* One real month of one market is the only real-data anchor. Non-EURUSD markets are plausible, not fitted;
+  `calibrate_many` has only been tested on synthetic multi-market files, never on real multi-month data.
+* A calm month cannot teach crashes or regime history; `stressed`, regimes and scenarios are assumptions.
+* Event dates are rule-based approximations; event sizes for importance classes are assumptions.
+* Realised cross-market correlations are close to, not exactly, the targets (EURUSD/USDCHF reaches about -0.83
+  against a -0.90 target); `scorecard`/your own checks report actuals.
+* Scheduled-event shock sizes use an uncapped lognormal, so the extreme tail is probably too fat: in the typical profile a
+  single 3-month path can contain a 40-95 bp one-minute move on an important release (real EURUSD rarely exceeds ~50 bp).
+* The model has no central-bank regime shifts, trending macro cycles, or order-book dynamics.
+* Spread and volume are models, not broker data. Tick generation is slow.
+
+## Overfitting warning
+Tuning a strategy until it looks good on synthetic data (or on a handful of paths) proves nothing. Always check a
+result against `monte_carlo(mode="null")`, keep out-of-sample paths you never tuned on, include costs, and validate
+on **real** data and a forward test before risking money.

forexgen-2.1.0/forexgen/__init__.py

@@ -0,0 +1,27 @@
+"""forexgen 2.0 - offline synthetic forex / gold data generator.
+
+Quick start:
+    import forexgen
+    df = forexgen.get_data("EURUSD", "M15", "3m")            # different every call; seed in df.attrs
+    df = forexgen.get_data("GBPUSD", "H1", "1y", seed=7)      # reproducible
+    forexgen.describe("XAUUSD")                               # calibrated vs assumed parameters
+"""
+from .api import (generate_csv, get_data, get_multi, list_markets, parse_duration,  # noqa: F401
+                  parse_timeframe, register_market)
+from .profiles import (Profile, describe, get_profile, load_profile, save_profile,  # noqa: F401
+                       use_profile, reset_profiles)
+from .events import get_events, register_event, set_events, clear_user_events      # noqa: F401
+from .scenarios import SCENARIOS                                                    # noqa: F401
+from .ticks import get_ticks                                                        # noqa: F401
+from .report import scorecard, stats_m1                                             # noqa: F401
+from .calibration import calibrate, calibrate_many, CalibrationSet, read_histdata                                     # noqa: F401
+from .validation import get_null_paths, monte_carlo, percentile_in_null, backtest   # noqa: F401
+from .costs import slippage, stop_fill, swap_pips                                   # noqa: F401
+
+from .factors import set_correlation, reset_correlation                             # noqa: F401
+from .export import export_histdata, generate_histdata, to_histdata_frame            # noqa: F401
+from .diagnostics import selftest, robustness_report, trend_strategy, mean_reversion_strategy  # noqa: F401
+from . import fixtures                                                              # noqa: F401
+from .fixtures import indicator_stress_test                                         # noqa: F401
+
+__version__ = "2.1.0"

forexgen-2.1.0/forexgen/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())