simudyne-pulse 0.7.0__tar.gz → 0.7.0.dev1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. {simudyne_pulse-0.7.0/src/simudyne_pulse.egg-info → simudyne_pulse-0.7.0.dev1}/PKG-INFO +1 -1
  2. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/pyproject.toml +1 -1
  3. simudyne_pulse-0.7.0.dev1/src/simudyne/resources/validation.py +411 -0
  4. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1/src/simudyne_pulse.egg-info}/PKG-INFO +1 -1
  5. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/src/simudyne_pulse.egg-info/SOURCES.txt +2 -1
  6. simudyne_pulse-0.7.0.dev1/tests/test_validation.py +158 -0
  7. simudyne_pulse-0.7.0/src/simudyne/resources/validation.py +0 -267
  8. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/LICENSE +0 -0
  9. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/README.md +0 -0
  10. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/setup.cfg +0 -0
  11. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/src/simudyne/__init__.py +0 -0
  12. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/src/simudyne/client.py +0 -0
  13. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/src/simudyne/exceptions.py +0 -0
  14. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/src/simudyne/resources/__init__.py +0 -0
  15. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/src/simudyne/resources/api_keys.py +0 -0
  16. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/src/simudyne/resources/data.py +0 -0
  17. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/src/simudyne/resources/historical.py +0 -0
  18. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/src/simudyne/resources/profile.py +0 -0
  19. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/src/simudyne/resources/simulation.py +0 -0
  20. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/src/simudyne/resources/simulator_gym.py +0 -0
  21. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/src/simudyne_pulse.egg-info/dependency_links.txt +0 -0
  22. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/src/simudyne_pulse.egg-info/requires.txt +0 -0
  23. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/src/simudyne_pulse.egg-info/top_level.txt +0 -0
  24. {simudyne_pulse-0.7.0 → simudyne_pulse-0.7.0.dev1}/tests/test_simulator_gym.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: simudyne-pulse
3
- Version: 0.7.0
3
+ Version: 0.7.0.dev1
4
4
  Summary: Python SDK for the Simudyne Pulse synthetic market data API
5
5
  Author-email: Simudyne <support@simudyne.com>
6
6
  License-Expression: MIT
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "simudyne-pulse"
7
- version = "0.7.0"
7
+ version = "0.7.0-dev.1"
8
8
  description = "Python SDK for the Simudyne Pulse synthetic market data API"
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -0,0 +1,411 @@
1
+ """
2
+ Validation Resource for the Pulse SDK.
3
+
4
+ This module provides methods for validating simulation quality by comparing
5
+ simulated LOB data against historical data using distributional metrics,
6
+ impact response analysis, stylised facts, and the MIND/FID inception distances
7
+ on DeepLOB embeddings.
8
+
9
+ Workflow:
10
+ 1. Submit a validation job with run() -> returns job_id
11
+ 2. Poll status with get_job(job_id) or use run_pipeline() for blocking
12
+ 3. View results including distances and plots
13
+ 4. List past jobs with list_jobs()
14
+
15
+ For the inception distances alone, inception_distances() is a one-call
16
+ shortcut that returns just the MIND and FID scores.
17
+
18
+ Tri-state run flags
19
+ -------------------
20
+ ``run_metrics`` / ``run_impact`` / ``run_stylised_facts`` / ``plot_data``
21
+ default to ``None``, meaning "use the default for my tier" — resolved
22
+ server-side. Demo turns everything on; other tiers get metrics and stylised
23
+ facts. Only flags you set explicitly are sent, so a demo key is not silently
24
+ opted out of the passes it is entitled to. Pass ``False`` to skip an expensive
25
+ pass, or ``True`` to force one on.
26
+
27
+ ``run_inception_distances`` is the exception: it defaults to ``True``, so MIND
28
+ and FID are computed unless you opt out. It maps to the API's ``run_fid``
29
+ config field, which gates both metrics because they share one DeepLOB
30
+ embedding pass. Note the scores are returned to the demo tier only — on any
31
+ other tier a ``True`` here spends the embedding pass and the response withholds
32
+ the numbers, so pass ``False`` if that is your situation.
33
+ """
34
+
35
+ import base64
36
+ import time
37
+
38
+
39
+ RUN_PATH = "/validation/run"
40
+ JOBS_PATH = "/validation/jobs"
41
+
42
+ #: Flags the API resolves from the caller's tier when left unset.
43
+ _TRI_STATE_FLAGS = (
44
+ "run_metrics",
45
+ "run_impact",
46
+ "run_stylised_facts",
47
+ "plot_data",
48
+ )
49
+
50
+ #: SDK name -> API config field. The API kept ``run_fid`` for compatibility;
51
+ #: the SDK spells out what it actually gates.
52
+ _INCEPTION_WIRE_FIELD = "run_fid"
53
+
54
+
55
+ class ValidationResource:
56
+ def __init__(self, client):
57
+ self._client = client
58
+
59
+ def run(
60
+ self,
61
+ symbol: str,
62
+ date: str,
63
+ sim_ids: list[str],
64
+ ticksize: float = 1.0,
65
+ run_metrics: bool = None,
66
+ run_impact: bool = None,
67
+ run_inception_distances: bool = True,
68
+ run_stylised_facts: bool = None,
69
+ plot_data: bool = None,
70
+ n_levels: int = 10,
71
+ l2_only: bool = False,
72
+ provider: str = None,
73
+ exchange: str = None,
74
+ ) -> dict:
75
+ """Submit a validation job.
76
+
77
+ Compares simulation output against historical market data using
78
+ distributional distance metrics (L1, Wasserstein), impact response
79
+ curves, Cont stylised facts, and the MIND/FID inception distances.
80
+
81
+ Historical data is fetched automatically based on symbol and date.
82
+ Simulation data is fetched from each sim_id's sim_data.parquet.
83
+
84
+ Args:
85
+ symbol: Trading symbol (e.g. "700.HK")
86
+ date: Calibration date in YYYY-MM-DD format (e.g. "2025-09-01")
87
+ sim_ids: List of simulation IDs to validate (max 25)
88
+ ticksize: Tick size for the symbol
89
+ run_metrics: Compute L1/Wasserstein distributional distances
90
+ (None = tier default)
91
+ run_impact: Compute impact response curves (None = tier default).
92
+ Only computed when plot_data is on — it has no verdict-only form.
93
+ run_inception_distances: Compute MIND *and* FID on DeepLOB
94
+ embeddings (default True). One flag gates both — they share a
95
+ single embedding pass. Sent as the API's ``run_fid`` field.
96
+ The scores reach the demo tier only, so on another tier this
97
+ spends the pass and the response withholds the numbers.
98
+ run_stylised_facts: Compute the 11 Cont stylised facts
99
+ (None = tier default)
100
+ plot_data: Store the raw data behind every plot — distribution
101
+ histograms, full stylised-fact payloads, impact curves.
102
+ **Demo tier only**; an explicit True from any other tier is
103
+ rejected with 403. When off, the job returns distances and the
104
+ per-fact verdicts only.
105
+ n_levels: Number of L2 book levels to use. The inception distances
106
+ need all 10.
107
+ l2_only: Restrict to metrics that need only bid/ask price+size.
108
+ Disables the impact response.
109
+ provider: Data provider (e.g. "omd"). Defaults to the prefix parsed
110
+ from sim_ids[0].
111
+ exchange: Exchange protocol (e.g. "hkex_securities"). Defaults to
112
+ the prefix parsed from sim_ids[0].
113
+
114
+ Returns:
115
+ dict with job_id, status, message
116
+ """
117
+ config = {
118
+ "n_levels": n_levels,
119
+ "l2_only": l2_only,
120
+ _INCEPTION_WIRE_FIELD: run_inception_distances,
121
+ }
122
+ for flag, value in zip(
123
+ _TRI_STATE_FLAGS,
124
+ (run_metrics, run_impact, run_stylised_facts, plot_data),
125
+ ):
126
+ # Omitted rather than sent as None: the API reads absence as "use my
127
+ # tier's default", and sending an explicit null would not do that.
128
+ if value is not None:
129
+ config[flag] = value
130
+
131
+ payload = {
132
+ "symbol": symbol,
133
+ "date": date,
134
+ "sim_ids": sim_ids,
135
+ "ticksize": ticksize,
136
+ "config": config,
137
+ }
138
+ if provider is not None:
139
+ payload["provider"] = provider
140
+ if exchange is not None:
141
+ payload["exchange"] = exchange
142
+
143
+ return self._client._request("POST", RUN_PATH, json=payload)
144
+
145
+ def get_job(self, job_id: str) -> dict:
146
+ """Get validation job status and results.
147
+
148
+ Args:
149
+ job_id: The job ID returned by run()
150
+
151
+ Returns:
152
+ dict with:
153
+ - status: "pending", "running", "completed", or "failed"
154
+ - distances: {metric: {l1: [...], w: [...]}} — every entitled tier
155
+ - stylised_fact_verdicts: {fact: {historical: bool | None,
156
+ simulated: [bool | None, ...]}} — every entitled tier
157
+ - mind_scores: one Monge Inception Distance per sim run, in sim_ids
158
+ order; None where a run could not be embedded. Demo tier only
159
+ - fid_scores: one Frechet Inception Distance per sim run, same
160
+ ordering and tier rule. Since pulse-check 1.8.0 this is the
161
+ embedding-space FID — not comparable with values stored by older
162
+ jobs
163
+ - distributions / impact_response / stylised_facts: the full
164
+ historical-derived payloads. Demo tier, plot_data jobs only
165
+ - plots: {distributions: [...], distances: [...],
166
+ impact_response: [...]} of {name, content_base64}
167
+ - metadata: dict with run parameters
168
+ - error: error message (when failed)
169
+
170
+ Lower MIND/FID = closer to the historical day. Neither is meaningful as
171
+ a bare number — see inception_distances() for how to read them.
172
+ """
173
+ return self._client._request("GET", f"{JOBS_PATH}/{job_id}")
174
+
175
+ def list_jobs(self, limit: int = 50) -> dict:
176
+ """List validation jobs for the current user.
177
+
178
+ Args:
179
+ limit: Max number of jobs to return (default 50, max 200)
180
+
181
+ Returns:
182
+ dict with jobs list and total count
183
+ """
184
+ return self._client._request("GET", JOBS_PATH, params={"limit": limit})
185
+
186
+ def run_pipeline(
187
+ self,
188
+ symbol: str,
189
+ date: str,
190
+ sim_ids: list[str],
191
+ ticksize: float = 1.0,
192
+ run_metrics: bool = None,
193
+ run_impact: bool = None,
194
+ run_inception_distances: bool = True,
195
+ run_stylised_facts: bool = None,
196
+ plot_data: bool = None,
197
+ n_levels: int = 10,
198
+ l2_only: bool = False,
199
+ provider: str = None,
200
+ exchange: str = None,
201
+ poll_interval: float = 3.0,
202
+ timeout: float = 600.0,
203
+ ) -> dict:
204
+ """Submit a validation job and block until it completes.
205
+
206
+ Combines run() + polling get_job() into a single call.
207
+ Prints progress to stderr.
208
+
209
+ Args:
210
+ symbol: Trading symbol (e.g. "700.HK")
211
+ date: Calibration date in YYYY-MM-DD format
212
+ sim_ids: List of simulation IDs to validate (max 25)
213
+ ticksize: Tick size for the symbol
214
+ run_metrics: Compute L1/Wasserstein distances (None = tier default)
215
+ run_impact: Compute impact response curves (None = tier default)
216
+ run_inception_distances: Compute MIND *and* FID on DeepLOB
217
+ embeddings (default True) — one flag gates both
218
+ run_stylised_facts: Compute the Cont stylised facts
219
+ (None = tier default)
220
+ plot_data: Store the raw plottable data (demo tier only)
221
+ n_levels: Number of L2 book levels to use
222
+ l2_only: Restrict to L2-only metrics
223
+ provider: Data provider; defaults to the sim_id prefix
224
+ exchange: Exchange protocol; defaults to the sim_id prefix
225
+ poll_interval: Seconds between status checks (default 3)
226
+ timeout: Max seconds to wait (default 600)
227
+
228
+ Returns:
229
+ dict with full validation results — see get_job() for the fields
230
+
231
+ Raises:
232
+ RuntimeError: If the validation job fails
233
+ TimeoutError: If the job doesn't complete within timeout
234
+ """
235
+ import sys
236
+
237
+ job = self.run(
238
+ symbol=symbol,
239
+ date=date,
240
+ sim_ids=sim_ids,
241
+ ticksize=ticksize,
242
+ run_metrics=run_metrics,
243
+ run_impact=run_impact,
244
+ run_inception_distances=run_inception_distances,
245
+ run_stylised_facts=run_stylised_facts,
246
+ plot_data=plot_data,
247
+ n_levels=n_levels,
248
+ l2_only=l2_only,
249
+ provider=provider,
250
+ exchange=exchange,
251
+ )
252
+ job_id = job["job_id"]
253
+ print(f"Validation job submitted: {job_id}", file=sys.stderr)
254
+
255
+ start = time.time()
256
+ while True:
257
+ result = self.get_job(job_id)
258
+ status = result["status"]
259
+
260
+ if status == "completed":
261
+ elapsed = time.time() - start
262
+ print(f"Completed in {elapsed:.1f}s", file=sys.stderr)
263
+ return result
264
+ elif status == "failed":
265
+ raise RuntimeError(f"Validation failed: {result.get('error')}")
266
+
267
+ if time.time() - start > timeout:
268
+ raise TimeoutError(
269
+ f"Validation job {job_id} timed out after {timeout}s"
270
+ )
271
+
272
+ time.sleep(poll_interval)
273
+
274
+ def inception_distances(
275
+ self,
276
+ symbol: str,
277
+ date: str,
278
+ sim_ids: list[str],
279
+ ticksize: float = 1.0,
280
+ n_levels: int = 10,
281
+ provider: str = None,
282
+ exchange: str = None,
283
+ poll_interval: float = 3.0,
284
+ timeout: float = 600.0,
285
+ ) -> dict:
286
+ """MIND and FID for each simulation, and nothing else.
287
+
288
+ A focused shortcut over run_pipeline(): keeps the inception distances
289
+ on and forces every other pass off, so the job does one DeepLOB embedding pass
290
+ and skips the metric, impact and stylised-fact work.
291
+
292
+ Both metrics are computed on 96-dim DeepLOB embeddings of 100-row L2
293
+ windows and need 10 book levels in the data.
294
+
295
+ Args:
296
+ symbol: Trading symbol (e.g. "700.HK")
297
+ date: Calibration date in YYYY-MM-DD format
298
+ sim_ids: List of simulation IDs to score (max 25)
299
+ ticksize: Tick size for the symbol
300
+ n_levels: Number of L2 book levels (10 required for the embeddings)
301
+ provider: Data provider; defaults to the sim_id prefix
302
+ exchange: Exchange protocol; defaults to the sim_id prefix
303
+ poll_interval: Seconds between status checks
304
+ timeout: Max seconds to wait
305
+
306
+ Returns:
307
+ dict with:
308
+ - mind: list of MIND scores, one per sim_id, None where a run could
309
+ not be embedded
310
+ - fid: list of FID scores, same ordering and convention
311
+ - sim_ids: the ids, so scores can be zipped back to their runs
312
+ - job_id: the underlying validation job
313
+
314
+ Raises:
315
+ RuntimeError: If the job fails, or if the scores come back empty —
316
+ which means the pipeline skipped them (missing torch, fewer
317
+ than 10 levels, unreachable checkpoint) or the key is not on
318
+ the demo tier, both of which are silent in the raw response.
319
+
320
+ Interpreting the scores:
321
+ Lower = closer to the historical day, but neither number means
322
+ anything on its own — only relative to a noise floor. Score a
323
+ real-vs-real control too (two slices of genuine market data) and
324
+ read a generator as a multiple of that floor. ~1x means the metric
325
+ cannot separate it from ordinary intraday variation.
326
+ """
327
+ result = self.run_pipeline(
328
+ symbol=symbol,
329
+ date=date,
330
+ sim_ids=sim_ids,
331
+ ticksize=ticksize,
332
+ run_inception_distances=True,
333
+ run_metrics=False,
334
+ run_impact=False,
335
+ run_stylised_facts=False,
336
+ n_levels=n_levels,
337
+ provider=provider,
338
+ exchange=exchange,
339
+ poll_interval=poll_interval,
340
+ timeout=timeout,
341
+ )
342
+
343
+ mind = result.get("mind_scores")
344
+ fid = result.get("fid_scores")
345
+ if not mind and not fid:
346
+ raise RuntimeError(
347
+ "no inception distances in the response. Either the pipeline "
348
+ "skipped them (torch missing, fewer than 10 book levels, or "
349
+ "the DeepLOB checkpoint unreachable) or this API key is not on "
350
+ "the demo tier, which is the only tier they are shared with."
351
+ )
352
+
353
+ return {
354
+ "mind": mind,
355
+ "fid": fid,
356
+ "sim_ids": list(sim_ids),
357
+ "job_id": result.get("job_id"),
358
+ }
359
+
360
+ def display_plots(self, result: dict) -> "PlotDisplay":
361
+ """Return a PlotDisplay object for displaying validation plots.
362
+
363
+ Usage:
364
+ plots = client.validation.display_plots(result)
365
+ plots.distributions() # show distribution histograms
366
+ plots.distances() # show spider plots
367
+ plots.impact_response() # show impact response plots
368
+
369
+ Args:
370
+ result: The result dict from run_pipeline() or get_job()
371
+ """
372
+ return PlotDisplay(result)
373
+
374
+
375
+ class PlotDisplay:
376
+ """Displays categorized validation plots inline in Jupyter notebooks."""
377
+
378
+ def __init__(self, result: dict):
379
+ plots = result.get("plots") or {}
380
+ self._distributions = plots.get("distributions", [])
381
+ self._distances = plots.get("distances", [])
382
+ self._impact_response = plots.get("impact_response", [])
383
+
384
+ def _show(self, plot_list, title):
385
+ from IPython.display import display, Image
386
+
387
+ if not plot_list:
388
+ print(f"No {title} plots available")
389
+ return
390
+
391
+ for plot in plot_list:
392
+ print(f"\n--- {plot['name']} ---")
393
+ display(Image(data=base64.b64decode(plot["content_base64"])))
394
+
395
+ def distributions(self):
396
+ """Display distribution histogram plots."""
397
+ self._show(self._distributions, "distribution")
398
+
399
+ def distances(self):
400
+ """Display spider plots (L1 and Wasserstein distances)."""
401
+ self._show(self._distances, "distance")
402
+
403
+ def impact_response(self):
404
+ """Display impact response plots."""
405
+ self._show(self._impact_response, "impact response")
406
+
407
+ def all(self):
408
+ """Display all plots."""
409
+ self.distances()
410
+ self.distributions()
411
+ self.impact_response()
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: simudyne-pulse
3
- Version: 0.7.0
3
+ Version: 0.7.0.dev1
4
4
  Summary: Python SDK for the Simudyne Pulse synthetic market data API
5
5
  Author-email: Simudyne <support@simudyne.com>
6
6
  License-Expression: MIT
@@ -17,4 +17,5 @@ src/simudyne_pulse.egg-info/SOURCES.txt
17
17
  src/simudyne_pulse.egg-info/dependency_links.txt
18
18
  src/simudyne_pulse.egg-info/requires.txt
19
19
  src/simudyne_pulse.egg-info/top_level.txt
20
- tests/test_simulator_gym.py
20
+ tests/test_simulator_gym.py
21
+ tests/test_validation.py
@@ -0,0 +1,158 @@
1
+ """Tests for the validation resource's request payloads and score handling.
2
+
3
+ No network: the client's _request is replaced with a recorder, so these assert
4
+ the exact payload the API receives — which is what actually broke when
5
+ pulse-check 1.8.0 dropped rescale_volumes/lot_size and made run_fid gate both
6
+ inception distances.
7
+ """
8
+
9
+ import pytest
10
+
11
+ from simudyne.resources.validation import ValidationResource
12
+
13
+
14
+ class FakeClient:
15
+ """Records calls and replays queued responses."""
16
+
17
+ def __init__(self, responses=None):
18
+ self.calls = []
19
+ self._responses = list(responses or [])
20
+
21
+ def _request(self, method, path, **kwargs):
22
+ self.calls.append((method, path, kwargs))
23
+ return self._responses.pop(0) if self._responses else {}
24
+
25
+
26
+ SIM_IDS = ["omd:hkex_securities:700.HK:2025-12-22:abc:normal:baseline:0000"]
27
+
28
+
29
+ class TestRunPayload:
30
+ def test_unset_flags_are_omitted_so_the_tier_default_applies(self):
31
+ client = FakeClient([{"job_id": "v1", "status": "pending"}])
32
+ ValidationResource(client).run("700.HK", "2025-12-22", SIM_IDS)
33
+
34
+ _, path, kwargs = client.calls[0]
35
+ config = kwargs["json"]["config"]
36
+ assert path == "/validation/run"
37
+ # Sending these as null would override the server-side tier default.
38
+ for flag in ("run_metrics", "run_impact",
39
+ "run_stylised_facts", "plot_data"):
40
+ assert flag not in config, f"{flag} must be omitted when unset"
41
+ # run_inception_distances is NOT tri-state: it defaults to True and is
42
+ # always sent, as the API's run_fid field.
43
+ assert config == {"n_levels": 10, "l2_only": False, "run_fid": True}
44
+
45
+ def test_explicit_flags_are_sent(self):
46
+ client = FakeClient([{"job_id": "v1"}])
47
+ ValidationResource(client).run(
48
+ "700.HK", "2025-12-22", SIM_IDS, run_impact=False
49
+ )
50
+
51
+ config = client.calls[0][2]["json"]["config"]
52
+ # An explicit False must survive — it is how a demo user skips a pass.
53
+ assert config["run_impact"] is False
54
+
55
+ def test_inception_distances_can_be_opted_out(self):
56
+ """Non-demo tiers should be able to skip the embedding pass."""
57
+ client = FakeClient([{"job_id": "v1"}])
58
+ ValidationResource(client).run(
59
+ "700.HK", "2025-12-22", SIM_IDS, run_inception_distances=False
60
+ )
61
+ assert client.calls[0][2]["json"]["config"]["run_fid"] is False
62
+
63
+ def test_old_run_fid_kwarg_is_rejected(self):
64
+ """Renamed to run_inception_distances — the old spelling must fail loudly."""
65
+ with pytest.raises(TypeError):
66
+ ValidationResource(FakeClient([{}])).run(
67
+ "700.HK", "2025-12-22", SIM_IDS, run_fid=True
68
+ )
69
+
70
+ def test_removed_params_are_not_accepted(self):
71
+ """rescale_volumes/lot_size went away with pulse-check 1.8.0."""
72
+ resource = ValidationResource(FakeClient([{}]))
73
+ with pytest.raises(TypeError):
74
+ resource.run("700.HK", "2025-12-22", SIM_IDS, rescale_volumes=True)
75
+ with pytest.raises(TypeError):
76
+ resource.run("700.HK", "2025-12-22", SIM_IDS, lot_size=100)
77
+
78
+ def test_provider_and_exchange_omitted_unless_given(self):
79
+ client = FakeClient([{}, {}])
80
+ r = ValidationResource(client)
81
+ r.run("700.HK", "2025-12-22", SIM_IDS)
82
+ assert "provider" not in client.calls[0][2]["json"]
83
+
84
+ r.run("700.HK", "2025-12-22", SIM_IDS, provider="omd", exchange="lse")
85
+ payload = client.calls[1][2]["json"]
86
+ assert payload["provider"] == "omd" and payload["exchange"] == "lse"
87
+
88
+
89
+ class TestInceptionDistances:
90
+ def _completed(self, **extra):
91
+ return {"job_id": "v1", "status": "completed", **extra}
92
+
93
+ def test_forces_fid_on_and_everything_else_off(self):
94
+ client = FakeClient([
95
+ {"job_id": "v1", "status": "pending"},
96
+ self._completed(mind_scores=[12.5], fid_scores=[8.25]),
97
+ ])
98
+ out = ValidationResource(client).inception_distances(
99
+ "700.HK", "2025-12-22", SIM_IDS, poll_interval=0
100
+ )
101
+
102
+ config = client.calls[0][2]["json"]["config"]
103
+ assert config["run_fid"] is True # wire name for run_inception_distances
104
+ assert config["run_metrics"] is False
105
+ assert config["run_impact"] is False
106
+ assert config["run_stylised_facts"] is False
107
+
108
+ assert out["mind"] == [12.5]
109
+ assert out["fid"] == [8.25]
110
+ assert out["sim_ids"] == SIM_IDS
111
+ assert out["job_id"] == "v1"
112
+
113
+ def test_none_scores_are_preserved_to_keep_indexing_aligned(self):
114
+ two = SIM_IDS * 2
115
+ client = FakeClient([
116
+ {"job_id": "v1", "status": "pending"},
117
+ self._completed(mind_scores=[12.5, None], fid_scores=[8.25, None]),
118
+ ])
119
+ out = ValidationResource(client).inception_distances(
120
+ "700.HK", "2025-12-22", two, poll_interval=0
121
+ )
122
+ assert out["mind"] == [12.5, None]
123
+ assert len(out["mind"]) == len(out["sim_ids"])
124
+
125
+ def test_missing_scores_raise_rather_than_returning_empty(self):
126
+ """A skipped pass or a non-demo key is silent in the raw response."""
127
+ client = FakeClient([
128
+ {"job_id": "v1", "status": "pending"},
129
+ self._completed(mind_scores=None, fid_scores=None),
130
+ ])
131
+ with pytest.raises(RuntimeError, match="no inception distances"):
132
+ ValidationResource(client).inception_distances(
133
+ "700.HK", "2025-12-22", SIM_IDS, poll_interval=0
134
+ )
135
+
136
+
137
+ class TestRunPipeline:
138
+ def test_raises_on_failed_job(self):
139
+ client = FakeClient([
140
+ {"job_id": "v1", "status": "pending"},
141
+ {"job_id": "v1", "status": "failed", "error": "boom"},
142
+ ])
143
+ with pytest.raises(RuntimeError, match="boom"):
144
+ ValidationResource(client).run_pipeline(
145
+ "700.HK", "2025-12-22", SIM_IDS, poll_interval=0
146
+ )
147
+
148
+ def test_returns_the_completed_result(self):
149
+ client = FakeClient([
150
+ {"job_id": "v1", "status": "pending"},
151
+ {"job_id": "v1", "status": "running"},
152
+ {"job_id": "v1", "status": "completed", "distances": {"spread": {}}},
153
+ ])
154
+ result = ValidationResource(client).run_pipeline(
155
+ "700.HK", "2025-12-22", SIM_IDS, poll_interval=0
156
+ )
157
+ assert result["status"] == "completed"
158
+ assert "spread" in result["distances"]
@@ -1,267 +0,0 @@
1
- """
2
- Validation Resource for the Pulse SDK.
3
-
4
- This module provides methods for validating simulation quality by comparing
5
- simulated LOB data against historical data using distributional metrics,
6
- impact response analysis, and FID scores.
7
-
8
- Workflow:
9
- 1. Submit a validation job with run() -> returns job_id
10
- 2. Poll status with get_job(job_id) or use run_pipeline() for blocking
11
- 3. View results including distances and plots
12
- 4. List past jobs with list_jobs()
13
- """
14
-
15
- import time
16
- import base64
17
-
18
-
19
- RUN_PATH = "/validation/run"
20
- JOBS_PATH = "/validation/jobs"
21
-
22
-
23
- class ValidationResource:
24
- def __init__(self, client):
25
- self._client = client
26
-
27
- def run(
28
- self,
29
- symbol: str,
30
- date: str,
31
- sim_ids: list[str],
32
- ticksize: float = 1.0,
33
- run_metrics: bool | None = None,
34
- run_impact: bool | None = None,
35
- run_fid: bool | None = None,
36
- run_stylised_facts: bool | None = None,
37
- n_levels: int = 10,
38
- rescale_volumes: bool = True,
39
- lot_size: int = 1,
40
- ) -> dict:
41
- """Submit a validation job.
42
-
43
- Compares simulation output against historical market data using
44
- distributional distance metrics (L1, Wasserstein), impact response
45
- curves, and FID scores.
46
-
47
- Historical data is fetched automatically from GCS based on symbol and date.
48
- Simulation data is fetched from each sim_id's sim_data.parquet in GCS.
49
-
50
- Args:
51
- symbol: Trading symbol (e.g. "700.HK")
52
- date: Calibration date in YYYY-MM-DD format (e.g. "2025-09-01")
53
- sim_ids: List of simulation IDs to validate (max 25)
54
- ticksize: Tick size for the symbol
55
- run_metrics: Compute L1/Wasserstein distributional distances.
56
- Leave unset to use your tier's default.
57
- run_impact: Compute impact response curves. Leave unset to use your
58
- tier's default.
59
- run_fid: Compute Frechet Inception Distance. Leave unset to use your
60
- tier's default.
61
- run_stylised_facts: Compute stylised facts (autocorrelation, heavy
62
- tails, volatility clustering). Leave unset to use your tier's
63
- default.
64
- n_levels: Number of L2 book levels to use
65
- rescale_volumes: Multiply simulated L2 size columns by lot_size
66
- lot_size: Lot size multiplier for volume rescaling
67
-
68
- Returns:
69
- dict with job_id, status, message
70
- """
71
- # A run_* flag left unset is omitted so the API applies your tier's
72
- # default rather than an SDK-side one. Sending False for the optional
73
- # passes would silently opt demo accounts out of the extra results that
74
- # tier is meant to return by default.
75
- config = {
76
- "n_levels": n_levels,
77
- "rescale_volumes": rescale_volumes,
78
- "lot_size": lot_size,
79
- }
80
- for flag, value in (
81
- ("run_metrics", run_metrics),
82
- ("run_impact", run_impact),
83
- ("run_fid", run_fid),
84
- ("run_stylised_facts", run_stylised_facts),
85
- ):
86
- if value is not None:
87
- config[flag] = value
88
-
89
- payload = {
90
- "symbol": symbol,
91
- "date": date,
92
- "sim_ids": sim_ids,
93
- "ticksize": ticksize,
94
- "config": config,
95
- }
96
- return self._client._request("POST", RUN_PATH, json=payload)
97
-
98
- def get_job(self, job_id: str) -> dict:
99
- """Get validation job status and results.
100
-
101
- Args:
102
- job_id: The job ID returned by run()
103
-
104
- Returns:
105
- dict with:
106
- - status: "pending", "running", "completed", or "failed"
107
- - distances: dict of {metric: {l1: [...], w: [...]}} (when completed)
108
- - metadata: dict with run parameters, including which passes ran
109
- - error: error message (when failed)
110
-
111
- Demo-tier accounts additionally receive the numbers derived from the
112
- historical data, which are withheld at the pro tier:
113
-
114
- - distributions: per-metric historical vs simulated histograms
115
- - impact_response: impact response curves as numbers, historical and
116
- one block per sim run
117
- - stylised_facts: historical and one block per sim run
118
- - fid_scores: one score per sim run (None where not computable)
119
- """
120
- return self._client._request("GET", f"{JOBS_PATH}/{job_id}")
121
-
122
- def list_jobs(self, limit: int = 50) -> dict:
123
- """List validation jobs for the current user.
124
-
125
- Args:
126
- limit: Max number of jobs to return (default 50, max 200)
127
-
128
- Returns:
129
- dict with jobs list and total count
130
- """
131
- return self._client._request("GET", JOBS_PATH, params={"limit": limit})
132
-
133
- def run_pipeline(
134
- self,
135
- symbol: str,
136
- date: str,
137
- sim_ids: list[str],
138
- ticksize: float = 1.0,
139
- run_metrics: bool | None = None,
140
- run_impact: bool | None = None,
141
- run_fid: bool | None = None,
142
- run_stylised_facts: bool | None = None,
143
- n_levels: int = 10,
144
- rescale_volumes: bool = True,
145
- lot_size: int = 1,
146
- poll_interval: float = 3.0,
147
- timeout: float = 600.0,
148
- ) -> dict:
149
- """Submit a validation job and block until it completes.
150
-
151
- Combines run() + polling get_job() into a single call.
152
- Prints progress to stderr.
153
-
154
- Args:
155
- symbol: Trading symbol (e.g. "700.HK")
156
- date: Calibration date in YYYY-MM-DD format
157
- sim_ids: List of simulation IDs to validate (max 25)
158
- ticksize: Tick size for the symbol
159
- run_metrics: Compute L1/Wasserstein distributional distances.
160
- Leave unset to use your tier's default.
161
- run_impact: Compute impact response curves. Leave unset to use your
162
- tier's default.
163
- run_fid: Compute Frechet Inception Distance. Leave unset to use your
164
- tier's default.
165
- run_stylised_facts: Compute stylised facts (autocorrelation, heavy
166
- tails, volatility clustering). Leave unset to use your tier's
167
- default.
168
- n_levels: Number of L2 book levels to use
169
- rescale_volumes: Multiply simulated L2 size columns by lot_size
170
- lot_size: Lot size multiplier for volume rescaling
171
- poll_interval: Seconds between status checks (default 3)
172
- timeout: Max seconds to wait (default 600)
173
-
174
- Returns:
175
- dict with full validation results (distances, plots, metadata)
176
-
177
- Raises:
178
- RuntimeError: If the validation job fails
179
- TimeoutError: If the job doesn't complete within timeout
180
- """
181
- import sys
182
-
183
- job = self.run(
184
- symbol=symbol,
185
- date=date,
186
- sim_ids=sim_ids,
187
- ticksize=ticksize,
188
- run_metrics=run_metrics,
189
- run_impact=run_impact,
190
- run_fid=run_fid,
191
- run_stylised_facts=run_stylised_facts,
192
- n_levels=n_levels,
193
- rescale_volumes=rescale_volumes,
194
- lot_size=lot_size,
195
- )
196
- job_id = job["job_id"]
197
- print(f"Validation job submitted: {job_id}", file=sys.stderr)
198
-
199
- start = time.time()
200
- while True:
201
- result = self.get_job(job_id)
202
- status = result["status"]
203
-
204
- if status == "completed":
205
- elapsed = time.time() - start
206
- print(f"Completed in {elapsed:.1f}s", file=sys.stderr)
207
- return result
208
- elif status == "failed":
209
- raise RuntimeError(f"Validation failed: {result.get('error')}")
210
-
211
- if time.time() - start > timeout:
212
- raise TimeoutError(f"Validation job {job_id} timed out after {timeout}s")
213
-
214
- time.sleep(poll_interval)
215
-
216
- def display_plots(self, result: dict) -> "PlotDisplay":
217
- """Return a PlotDisplay object for displaying validation plots.
218
-
219
- Usage:
220
- plots = client.validation.display_plots(result)
221
- plots.distributions() # show distribution histograms
222
- plots.distances() # show spider plots
223
- plots.impact_response() # show impact response plots
224
-
225
- Args:
226
- result: The result dict from run_pipeline() or get_job()
227
- """
228
- return PlotDisplay(result)
229
-
230
-
231
- class PlotDisplay:
232
- """Displays categorized validation plots inline in Jupyter notebooks."""
233
-
234
- def __init__(self, result: dict):
235
- plots = result.get("plots") or {}
236
- self._distributions = plots.get("distributions", [])
237
- self._distances = plots.get("distances", [])
238
- self._impact_response = plots.get("impact_response", [])
239
-
240
- def _show(self, plot_list, title):
241
- from IPython.display import display, Image
242
-
243
- if not plot_list:
244
- print(f"No {title} plots available")
245
- return
246
-
247
- for plot in plot_list:
248
- print(f"\n--- {plot['name']} ---")
249
- display(Image(data=base64.b64decode(plot["content_base64"])))
250
-
251
- def distributions(self):
252
- """Display distribution histogram plots."""
253
- self._show(self._distributions, "distribution")
254
-
255
- def distances(self):
256
- """Display spider plots (L1 and Wasserstein distances)."""
257
- self._show(self._distances, "distance")
258
-
259
- def impact_response(self):
260
- """Display impact response plots."""
261
- self._show(self._impact_response, "impact response")
262
-
263
- def all(self):
264
- """Display all plots."""
265
- self.distances()
266
- self.distributions()
267
- self.impact_response()