pineforge-data 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pineforge_data/__init__.py +124 -0
- pineforge_data/backtest.py +552 -0
- pineforge_data/cli/__init__.py +1 -0
- pineforge_data/cli/backtest.py +354 -0
- pineforge_data/compile_cache.py +171 -0
- pineforge_data/docker_runtime.py +226 -0
- pineforge_data/engine.py +185 -0
- pineforge_data/errors.py +5 -0
- pineforge_data/models.py +199 -0
- pineforge_data/providers/__init__.py +69 -0
- pineforge_data/providers/base.py +51 -0
- pineforge_data/providers/ccxt.py +411 -0
- pineforge_data/providers/local.py +207 -0
- pineforge_data/providers/registry.py +252 -0
- pineforge_data/providers/sqlalchemy.py +138 -0
- pineforge_data/providers/tabular.py +530 -0
- pineforge_data/py.typed +1 -0
- pineforge_data/release_contract.py +153 -0
- pineforge_data/requests.py +87 -0
- pineforge_data/server.py +646 -0
- pineforge_data/server_client.py +142 -0
- pineforge_data-0.2.0.dist-info/METADATA +364 -0
- pineforge_data-0.2.0.dist-info/RECORD +26 -0
- pineforge_data-0.2.0.dist-info/WHEEL +4 -0
- pineforge_data-0.2.0.dist-info/entry_points.txt +3 -0
- pineforge_data-0.2.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""Standard-library client for the PineForge FastAPI backtest service."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from collections.abc import Mapping, Sequence
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from urllib.error import HTTPError, URLError
|
|
9
|
+
from urllib.parse import urlsplit
|
|
10
|
+
from urllib.request import Request, urlopen
|
|
11
|
+
from uuid import uuid4
|
|
12
|
+
|
|
13
|
+
from .backtest import BacktestOptions, JsonValue
|
|
14
|
+
from .models import Bar, Instrument
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class BacktestServerError(RuntimeError):
|
|
18
|
+
"""The configured FastAPI server rejected or failed a backtest."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _scalar_inputs(values: Mapping[str, JsonValue] | None, field: str) -> dict[str, object]:
|
|
22
|
+
result: dict[str, object] = {}
|
|
23
|
+
for key, value in (values or {}).items():
|
|
24
|
+
if not isinstance(value, (str, int, float, bool)):
|
|
25
|
+
raise ValueError(f"{field}.{key} must be a scalar value")
|
|
26
|
+
result[key] = value
|
|
27
|
+
return result
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True, slots=True)
|
|
31
|
+
class FastApiBacktestClient:
|
|
32
|
+
"""Submit normalized bars to a bounded remote PineForge runtime."""
|
|
33
|
+
|
|
34
|
+
base_url: str
|
|
35
|
+
timeout_seconds: float = 330.0
|
|
36
|
+
api_key: str | None = None
|
|
37
|
+
max_response_bytes: int = 128 * 1_024 * 1_024
|
|
38
|
+
|
|
39
|
+
def __post_init__(self) -> None:
|
|
40
|
+
parsed = urlsplit(self.base_url)
|
|
41
|
+
if parsed.scheme not in ("http", "https") or not parsed.netloc:
|
|
42
|
+
raise ValueError("server URL must be an absolute http:// or https:// URL")
|
|
43
|
+
if self.timeout_seconds <= 0 or self.max_response_bytes <= 0:
|
|
44
|
+
raise ValueError("client timeout and response limit must be positive")
|
|
45
|
+
|
|
46
|
+
def run(
|
|
47
|
+
self,
|
|
48
|
+
pine_source: str,
|
|
49
|
+
bars: Sequence[Bar],
|
|
50
|
+
*,
|
|
51
|
+
instrument: Instrument,
|
|
52
|
+
source: str,
|
|
53
|
+
options: BacktestOptions,
|
|
54
|
+
strategy_params: Mapping[str, JsonValue] | None = None,
|
|
55
|
+
strategy_overrides: Mapping[str, JsonValue] | None = None,
|
|
56
|
+
) -> dict[str, object]:
|
|
57
|
+
payload = {
|
|
58
|
+
"pine_source": pine_source,
|
|
59
|
+
"bars": [
|
|
60
|
+
{
|
|
61
|
+
"timestamp_ms": bar.timestamp_ms,
|
|
62
|
+
"open": bar.open,
|
|
63
|
+
"high": bar.high,
|
|
64
|
+
"low": bar.low,
|
|
65
|
+
"close": bar.close,
|
|
66
|
+
"volume": bar.volume,
|
|
67
|
+
}
|
|
68
|
+
for bar in bars
|
|
69
|
+
],
|
|
70
|
+
"instrument": {
|
|
71
|
+
"symbol": instrument.symbol,
|
|
72
|
+
"venue": instrument.venue,
|
|
73
|
+
"timezone": instrument.timezone,
|
|
74
|
+
"session": instrument.session,
|
|
75
|
+
"volume_unit": instrument.volume_unit,
|
|
76
|
+
},
|
|
77
|
+
"source": source,
|
|
78
|
+
"options": {
|
|
79
|
+
"input_timeframe": options.input_timeframe,
|
|
80
|
+
"script_timeframe": options.script_timeframe,
|
|
81
|
+
"bar_magnifier": options.bar_magnifier,
|
|
82
|
+
"magnifier_samples": options.magnifier_samples,
|
|
83
|
+
"magnifier_distribution": options.magnifier_distribution.name.lower(),
|
|
84
|
+
"trace_enabled": options.trace_enabled,
|
|
85
|
+
"chart_timezone": options.chart_timezone,
|
|
86
|
+
"trade_start_time_ms": options.trade_start_time_ms,
|
|
87
|
+
},
|
|
88
|
+
"strategy_params": _scalar_inputs(strategy_params, "strategy_params"),
|
|
89
|
+
"strategy_overrides": _scalar_inputs(strategy_overrides, "strategy_overrides"),
|
|
90
|
+
}
|
|
91
|
+
headers = {
|
|
92
|
+
"Content-Type": "application/json",
|
|
93
|
+
"Accept": "application/json",
|
|
94
|
+
"X-Request-ID": uuid4().hex,
|
|
95
|
+
}
|
|
96
|
+
if self.api_key:
|
|
97
|
+
headers["Authorization"] = f"Bearer {self.api_key}"
|
|
98
|
+
request = Request(
|
|
99
|
+
f"{self.base_url.rstrip('/')}/v1/backtests",
|
|
100
|
+
data=json.dumps(payload, separators=(",", ":"), allow_nan=False).encode(),
|
|
101
|
+
headers=headers,
|
|
102
|
+
method="POST",
|
|
103
|
+
)
|
|
104
|
+
try:
|
|
105
|
+
with urlopen(request, timeout=self.timeout_seconds) as response:
|
|
106
|
+
body = response.read(self.max_response_bytes + 1)
|
|
107
|
+
except HTTPError as exc:
|
|
108
|
+
body = exc.read(self.max_response_bytes + 1)
|
|
109
|
+
detail = _error_message(body) or f"HTTP {exc.code}"
|
|
110
|
+
raise BacktestServerError(detail) from exc
|
|
111
|
+
except URLError as exc:
|
|
112
|
+
raise BacktestServerError(f"backtest server is unavailable: {exc.reason}") from exc
|
|
113
|
+
if len(body) > self.max_response_bytes:
|
|
114
|
+
raise BacktestServerError("backtest server response exceeded the configured limit")
|
|
115
|
+
try:
|
|
116
|
+
value = json.loads(body)
|
|
117
|
+
except json.JSONDecodeError as exc:
|
|
118
|
+
raise BacktestServerError("backtest server returned invalid JSON") from exc
|
|
119
|
+
if not isinstance(value, dict):
|
|
120
|
+
raise BacktestServerError("backtest server response must be a JSON object")
|
|
121
|
+
if value.get("schema_version") != 1:
|
|
122
|
+
raise BacktestServerError("unsupported backtest server response schema")
|
|
123
|
+
if not isinstance(value.get("runtime"), dict) or not isinstance(
|
|
124
|
+
value.get("backtest"), dict
|
|
125
|
+
):
|
|
126
|
+
raise BacktestServerError("backtest server response is missing runtime data")
|
|
127
|
+
return value
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _error_message(body: bytes) -> str:
|
|
131
|
+
try:
|
|
132
|
+
value = json.loads(body)
|
|
133
|
+
except json.JSONDecodeError:
|
|
134
|
+
return body.decode("utf-8", "replace").strip()
|
|
135
|
+
if not isinstance(value, dict) or not isinstance(value.get("error"), dict):
|
|
136
|
+
return str(value)
|
|
137
|
+
error = value["error"]
|
|
138
|
+
phase = error.get("phase", "server")
|
|
139
|
+
message = error.get("message", "request failed")
|
|
140
|
+
request_id = error.get("request_id")
|
|
141
|
+
suffix = f" (request {request_id})" if request_id else ""
|
|
142
|
+
return f"{phase}: {message}{suffix}"
|
|
@@ -0,0 +1,364 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pineforge-data
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Provider-neutral market and macro data adapters for PineForge
|
|
5
|
+
Project-URL: Homepage, https://github.com/pineforge-4pass/pineforge-data
|
|
6
|
+
Project-URL: Documentation, https://pineforge-4pass.github.io/pineforge-data/
|
|
7
|
+
Project-URL: Issues, https://github.com/pineforge-4pass/pineforge-data/issues
|
|
8
|
+
Project-URL: Source, https://github.com/pineforge-4pass/pineforge-data
|
|
9
|
+
Author: PineForge contributors
|
|
10
|
+
License: Apache-2.0
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: backtesting,macro-data,market-data,pinescript
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.11
|
|
21
|
+
Provides-Extra: ccxt
|
|
22
|
+
Requires-Dist: ccxt<5,>=4.5.64; extra == 'ccxt'
|
|
23
|
+
Provides-Extra: database
|
|
24
|
+
Requires-Dist: sqlalchemy<3,>=2.0; extra == 'database'
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
27
|
+
Requires-Dist: fastapi<0.140,>=0.139; extra == 'dev'
|
|
28
|
+
Requires-Dist: httpx2<2.6,>=2.5; extra == 'dev'
|
|
29
|
+
Requires-Dist: mypy>=1.13; extra == 'dev'
|
|
30
|
+
Requires-Dist: pytest>=8.3; extra == 'dev'
|
|
31
|
+
Requires-Dist: ruff>=0.8; extra == 'dev'
|
|
32
|
+
Requires-Dist: sqlalchemy<3,>=2.0; extra == 'dev'
|
|
33
|
+
Requires-Dist: uvicorn<0.52,>=0.51; extra == 'dev'
|
|
34
|
+
Provides-Extra: docs
|
|
35
|
+
Requires-Dist: mkdocs-material<10,>=9.7; extra == 'docs'
|
|
36
|
+
Requires-Dist: mkdocs<2,>=1.6; extra == 'docs'
|
|
37
|
+
Requires-Dist: mkdocstrings-python<3,>=2.0; extra == 'docs'
|
|
38
|
+
Requires-Dist: ruff>=0.8; extra == 'docs'
|
|
39
|
+
Provides-Extra: release
|
|
40
|
+
Requires-Dist: build<2,>=1.2; extra == 'release'
|
|
41
|
+
Requires-Dist: twine<7,>=6.2; extra == 'release'
|
|
42
|
+
Provides-Extra: server
|
|
43
|
+
Requires-Dist: fastapi<0.140,>=0.139; extra == 'server'
|
|
44
|
+
Requires-Dist: uvicorn<0.52,>=0.51; extra == 'server'
|
|
45
|
+
Description-Content-Type: text/markdown
|
|
46
|
+
|
|
47
|
+
# PineForge Data
|
|
48
|
+
|
|
49
|
+
Provider-neutral market and macro data adapters for PineForge.
|
|
50
|
+
|
|
51
|
+
`pineforge-data` is the boundary between third-party services and the
|
|
52
|
+
deterministic [`pineforge-engine`](https://github.com/pineforge-4pass/pineforge-engine)
|
|
53
|
+
runtime:
|
|
54
|
+
|
|
55
|
+
```text
|
|
56
|
+
provider APIs → pineforge-data normalization → PineForge C ABI
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
The engine does not import or link this package. Provider transport,
|
|
60
|
+
authentication, retries, caching, symbol mapping, and vendor schemas stay here;
|
|
61
|
+
the engine receives only normalized bars and ordered trades.
|
|
62
|
+
|
|
63
|
+
## Documentation
|
|
64
|
+
|
|
65
|
+
- [Documentation home](https://pineforge-4pass.github.io/pineforge-data/) — architecture, guarantees, and guide map.
|
|
66
|
+
- [Getting started](https://pineforge-4pass.github.io/pineforge-data/getting-started/) — installation, first provider
|
|
67
|
+
request, and local or remote backtest.
|
|
68
|
+
- [Python API reference](https://pineforge-4pass.github.io/pineforge-data/api/) — generated signatures, types, and docstrings.
|
|
69
|
+
- [Normalized data model](https://pineforge-4pass.github.io/pineforge-data/data-model/) — instruments, contracts, bars,
|
|
70
|
+
live trades, macro vintages, and validation rules.
|
|
71
|
+
- [Provider catalog](https://pineforge-4pass.github.io/pineforge-data/providers/) — shared lifecycle and second-level API
|
|
72
|
+
guides for CCXT, CSV, SQLite, and SQLAlchemy.
|
|
73
|
+
- [Backtesting](https://pineforge-4pass.github.io/pineforge-data/backtesting/) — CLI options, configuration files, runtime
|
|
74
|
+
channels, report schema, and reproducibility.
|
|
75
|
+
- [FastAPI server](https://pineforge-4pass.github.io/pineforge-data/server/) — concurrency, authentication, timeouts,
|
|
76
|
+
compile cache, and deployment.
|
|
77
|
+
- [Provider contract](https://pineforge-4pass.github.io/pineforge-data/provider-contract/) — implementing and testing a
|
|
78
|
+
community exchange or broker adapter.
|
|
79
|
+
|
|
80
|
+
## Why Python first
|
|
81
|
+
|
|
82
|
+
Provider integrations are dominated by HTTP, WebSockets, JSON, credentials,
|
|
83
|
+
and asynchronous I/O. Python makes those integrations accessible to community
|
|
84
|
+
contributors. Engine throughput remains native: normalized records are packed
|
|
85
|
+
into contiguous C ABI arrays and submitted in one call.
|
|
86
|
+
|
|
87
|
+
## Initial contracts
|
|
88
|
+
|
|
89
|
+
- `Instrument` — normalized symbol, provider-native ID, venue, asset/market type,
|
|
90
|
+
currencies, contract terms, timezone, session, and volume units.
|
|
91
|
+
- `MarketListing` and `MarketQuery` — catalog discovery without vendor schemas.
|
|
92
|
+
- `Bar` — confirmed OHLCV with source provenance.
|
|
93
|
+
- `TradeTick` — the provider-neutral four-field engine payload plus provenance.
|
|
94
|
+
- `MacroObservation` — observation period, first release, and vintage timestamps
|
|
95
|
+
to prevent revised-data lookahead.
|
|
96
|
+
- `MarketCatalogProvider`, `HistoricalBarProvider`, `LiveTradeProvider`, and
|
|
97
|
+
`MacroDataProvider` — small structural protocols that community adapters
|
|
98
|
+
implement.
|
|
99
|
+
- `ProviderRegistry` — built-in and installed broker adapters selected by name.
|
|
100
|
+
- `BarColumnMapping` and `TabularSchema` — runtime discovery and safe OHLCV
|
|
101
|
+
mappings for user-owned files, tables, and views.
|
|
102
|
+
- `PfBar`, `PfTradeTick`, and `EngineStreamSink` — dependency-free `ctypes`
|
|
103
|
+
interoperability with PineForge strategy libraries.
|
|
104
|
+
|
|
105
|
+
## CCXT adapter
|
|
106
|
+
|
|
107
|
+
The first community adapter uses CCXT's unified async API for exchange-neutral
|
|
108
|
+
crypto data. It paginates OHLCV, removes duplicate timestamps, excludes the
|
|
109
|
+
currently forming candle, and polls public trades into a strictly increasing
|
|
110
|
+
per-stream sequence.
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
pip install 'pineforge-data[ccxt]'
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
from pineforge_data import BarRequest, CcxtProvider, Instrument
|
|
118
|
+
|
|
119
|
+
instrument = Instrument("BTC/USDT", venue="kraken")
|
|
120
|
+
request = BarRequest(
|
|
121
|
+
instrument,
|
|
122
|
+
timeframe="1m",
|
|
123
|
+
start_ms=1_767_225_600_000,
|
|
124
|
+
end_ms=1_767_312_000_000,
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
async with CcxtProvider("kraken") as provider:
|
|
128
|
+
listing = await provider.resolve_market("BTC/USDT")
|
|
129
|
+
confirmed_bars = await provider.fetch_bars(request)
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
`Instrument.symbol` uses CCXT's exact unified spelling. Do not infer a market
|
|
133
|
+
by parsing it: `resolve_market()` uses CCXT's catalog fields to distinguish
|
|
134
|
+
spot, swap, future, and option listings and captures `base`, `quote`, `settle`,
|
|
135
|
+
raw exchange ID, contract size, linear/inverse settlement, expiry, strike, and
|
|
136
|
+
option type. For example, `BTC/USDT` and `BTC/USDT:USDT` are separate markets.
|
|
137
|
+
|
|
138
|
+
Exchange credentials and exchange-specific options can be passed through
|
|
139
|
+
`config`, while endpoint options remain isolated in `market_params`,
|
|
140
|
+
`ohlcv_params`, and `trade_params`. Realtime public trades use REST polling in
|
|
141
|
+
this bootstrap; a WebSocket transport can implement the same
|
|
142
|
+
`LiveTradeProvider` contract later.
|
|
143
|
+
|
|
144
|
+
`TradeSubscription.start_ms` can pin the live handoff to the next timestamp
|
|
145
|
+
after an engine warmup. `start_sequence` is the last accepted sequence, so the
|
|
146
|
+
adapter emits `start_sequence + 1` next.
|
|
147
|
+
|
|
148
|
+
## Local files and databases
|
|
149
|
+
|
|
150
|
+
Built-in `csv`, `sqlite`, and `sqlalchemy` providers let users backtest their
|
|
151
|
+
own data without adopting a fixed PineForge DDL. They inspect file headers or
|
|
152
|
+
database reflection metadata at runtime, infer common OHLCV names, and accept
|
|
153
|
+
partial mappings for arbitrary names. Ambiguous schemas fail instead of being
|
|
154
|
+
guessed.
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
from pineforge_data import SqliteBarProvider
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
provider = SqliteBarProvider(
|
|
161
|
+
"warehouse.sqlite3",
|
|
162
|
+
table="price candles",
|
|
163
|
+
mapping={
|
|
164
|
+
"timestamp": "epoch seconds",
|
|
165
|
+
"open": "first px",
|
|
166
|
+
"high": "top px",
|
|
167
|
+
"low": "bottom px",
|
|
168
|
+
"close": "last px",
|
|
169
|
+
"volume": "traded qty",
|
|
170
|
+
},
|
|
171
|
+
timestamp_unit="seconds",
|
|
172
|
+
)
|
|
173
|
+
schema = await provider.inspect_schema()
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
SQL identifiers are validated against reflected metadata; filter values are
|
|
177
|
+
bound parameters. For complex transformations, expose a database view rather
|
|
178
|
+
than putting raw SQL in harness configuration. See the complete
|
|
179
|
+
[provider catalog](https://pineforge-4pass.github.io/pineforge-data/providers/),
|
|
180
|
+
with dedicated API guides for
|
|
181
|
+
[CSV](https://pineforge-4pass.github.io/pineforge-data/providers/csv/),
|
|
182
|
+
[SQLite](https://pineforge-4pass.github.io/pineforge-data/providers/sqlite/), and
|
|
183
|
+
[SQLAlchemy](https://pineforge-4pass.github.io/pineforge-data/providers/sqlalchemy/).
|
|
184
|
+
|
|
185
|
+
## Direct backtest harness
|
|
186
|
+
|
|
187
|
+
The public backtest input is raw PineScript v6. `pineforge-backtest` fetches
|
|
188
|
+
confirmed OHLCV through a data provider and runs this pinned pipeline:
|
|
189
|
+
|
|
190
|
+
```text
|
|
191
|
+
raw .pine + provider OHLCV
|
|
192
|
+
↓ local read-only mount or FastAPI request
|
|
193
|
+
pineforge-release → generated C++ → cached/compiled strategy → JSON report
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
Docker is a prerequisite. A host C++ compiler and a precompiled strategy
|
|
197
|
+
library are not required. Install the package without cloning engine or codegen
|
|
198
|
+
repositories:
|
|
199
|
+
|
|
200
|
+
```bash
|
|
201
|
+
pip install 'pineforge-data[ccxt]'
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
pineforge-backtest \
|
|
206
|
+
--pine strategy.pine \
|
|
207
|
+
--provider ccxt \
|
|
208
|
+
--venue kraken \
|
|
209
|
+
--symbol BTC/USD \
|
|
210
|
+
--timeframe 15m \
|
|
211
|
+
--start 2026-07-01T00:00:00Z \
|
|
212
|
+
--end 2026-07-08T00:00:00Z \
|
|
213
|
+
--warmup-bars 500 \
|
|
214
|
+
--output report.json \
|
|
215
|
+
--pretty
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
`--warmup-bars` fetches additional source bars before `--start` to initialize
|
|
219
|
+
indicators and higher-timeframe state. Order execution remains disabled until
|
|
220
|
+
`--start`, and the report records both requested and loaded warmup counts.
|
|
221
|
+
|
|
222
|
+
The first local invocation pulls an immutable, multi-architecture
|
|
223
|
+
`pineforge-release` image pinned by both version and OCI digest. It never builds
|
|
224
|
+
engine or codegen locally. Use `--pull-policy never` for offline runs or opt in
|
|
225
|
+
to the rolling channel with:
|
|
226
|
+
|
|
227
|
+
```bash
|
|
228
|
+
pineforge-backtest ... \
|
|
229
|
+
--runtime-image ghcr.io/pineforge-4pass/pineforge-release:latest \
|
|
230
|
+
--pull-policy always
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
`latest` is convenient for development but not deterministic. The report
|
|
234
|
+
records the resolved image digest and component versions when Docker exposes
|
|
235
|
+
them.
|
|
236
|
+
|
|
237
|
+
Compilation and execution run as a non-root user with networking disabled, all
|
|
238
|
+
Linux capabilities dropped, a read-only root filesystem, and only a read-only
|
|
239
|
+
temporary input mount.
|
|
240
|
+
|
|
241
|
+
The JSON report contains provider and market provenance, the release runtime
|
|
242
|
+
identity and fingerprint, processed-bar counts, every closed trade,
|
|
243
|
+
all/long/short statistics, equity statistics, diagnostics, and the complete
|
|
244
|
+
equity curve. Unix millisecond timestamps can be used instead of ISO-8601
|
|
245
|
+
values. The pinned `pineforge-release` does not currently expose trace
|
|
246
|
+
collection; `--trace` fails explicitly rather than silently omitting it.
|
|
247
|
+
|
|
248
|
+
Use `--provider-config config.json` for CCXT constructor options and
|
|
249
|
+
`--strategy-params inputs.json` for Pine inputs. Use `--strategy-overrides` for
|
|
250
|
+
`strategy()` header overrides. The provider config file may contain
|
|
251
|
+
credentials, so keep it outside version control.
|
|
252
|
+
|
|
253
|
+
The generated C++ hash and exact engine/codegen versions are recorded in the
|
|
254
|
+
release fingerprint. The combined runtime and its component licensing are
|
|
255
|
+
owned by [`pineforge-release`](https://github.com/pineforge-4pass/pineforge-release),
|
|
256
|
+
not vendored into this repository.
|
|
257
|
+
|
|
258
|
+
Provider implementations in this repository are Python-only. The compiled C++
|
|
259
|
+
strategy and engine stay behind the Docker/runtime boundary; broker SDKs and
|
|
260
|
+
provider-specific types do not cross into `pineforge-engine`.
|
|
261
|
+
|
|
262
|
+
## Concurrent FastAPI server
|
|
263
|
+
|
|
264
|
+
The server image derives from the same pinned `pineforge-release` image. It
|
|
265
|
+
admits a bounded number of compiler/backtest processes, keeps a bounded queue,
|
|
266
|
+
isolates every request in its own temporary directory, and optionally requires
|
|
267
|
+
a bearer token.
|
|
268
|
+
|
|
269
|
+
```bash
|
|
270
|
+
docker build -f docker/server.Dockerfile -t pineforge-data-server .
|
|
271
|
+
docker volume create pineforge-compile-cache
|
|
272
|
+
docker run --rm -p 127.0.0.1:8000:8000 \
|
|
273
|
+
--read-only \
|
|
274
|
+
--tmpfs /tmp:rw,exec,nosuid,nodev,size=512m \
|
|
275
|
+
--cap-drop ALL \
|
|
276
|
+
--security-opt no-new-privileges \
|
|
277
|
+
--mount type=volume,src=pineforge-compile-cache,dst=/cache \
|
|
278
|
+
-e PINEFORGE_SERVER_API_KEY=change-me \
|
|
279
|
+
pineforge-data-server
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
Point the same harness at it without putting the token on the command line:
|
|
283
|
+
|
|
284
|
+
```bash
|
|
285
|
+
export PINEFORGE_SERVER_URL=http://127.0.0.1:8000
|
|
286
|
+
export PINEFORGE_SERVER_API_KEY=change-me
|
|
287
|
+
pineforge-backtest --pine strategy.pine --venue kraken --symbol BTC/USD \
|
|
288
|
+
--timeframe 15m --start 2026-07-01T00:00:00Z --end 2026-07-08T00:00:00Z
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
The server always transpiles Pine deterministically and hashes the generated
|
|
292
|
+
C++. Its cache stores the compiled `.so` under a key containing that C++ hash
|
|
293
|
+
plus the release, engine, architecture, and compile flags. Concurrent misses
|
|
294
|
+
for the same key compile once; subsequent requests skip compilation. Cache
|
|
295
|
+
hit/key/hash are included in response provenance. See the
|
|
296
|
+
[server guide](https://pineforge-4pass.github.io/pineforge-data/server/) for endpoints, limits, deployment, and cache
|
|
297
|
+
settings.
|
|
298
|
+
|
|
299
|
+
## Contributing
|
|
300
|
+
|
|
301
|
+
Community providers, market-model improvements, server/runtime work, tests, and
|
|
302
|
+
documentation are welcome. Provider integrations are Python-only; engine and
|
|
303
|
+
codegen changes belong in their upstream repositories and are consumed here
|
|
304
|
+
through `pineforge-release`.
|
|
305
|
+
|
|
306
|
+
### Choose the right contribution path
|
|
307
|
+
|
|
308
|
+
| Contribution | Primary location | Start here |
|
|
309
|
+
|---|---|---|
|
|
310
|
+
| Exchange or broker adapter | `src/pineforge_data/providers/` | [Provider contract](https://pineforge-4pass.github.io/pineforge-data/provider-contract/) |
|
|
311
|
+
| Market, contract, bar, or request model | `src/pineforge_data/models.py`, `src/pineforge_data/requests.py`, `src/pineforge_data/providers/base.py` | Existing public models and protocols |
|
|
312
|
+
| Backtest harness or HTTP client | `src/pineforge_data/cli/backtest.py`, `src/pineforge_data/server_client.py` | Harness unit tests |
|
|
313
|
+
| FastAPI concurrency or compile cache | `src/pineforge_data/server.py`, `src/pineforge_data/compile_cache.py` | [Server guide](https://pineforge-4pass.github.io/pineforge-data/server/) |
|
|
314
|
+
| Release-container integration | `src/pineforge_data/release_contract.py`, `src/pineforge_data/docker_runtime.py` | Pinned release contract and Docker tests |
|
|
315
|
+
| Documentation or examples | `README.md`, `docs/` | A focused documentation PR |
|
|
316
|
+
|
|
317
|
+
For a new provider, implement the smallest applicable structural protocols,
|
|
318
|
+
register its factory, keep its SDK in an optional dependency extra, and add
|
|
319
|
+
offline fixture tests. Resolve exact upstream markets through their catalog;
|
|
320
|
+
do not parse symbols to infer base, quote, settlement, or contract terms.
|
|
321
|
+
Provider-specific fields stay in this repository and must not leak into
|
|
322
|
+
`pineforge-engine`.
|
|
323
|
+
|
|
324
|
+
### Development setup
|
|
325
|
+
|
|
326
|
+
```bash
|
|
327
|
+
git clone https://github.com/pineforge-4pass/pineforge-data.git
|
|
328
|
+
cd pineforge-data
|
|
329
|
+
python3 -m venv .venv
|
|
330
|
+
.venv/bin/pip install -e '.[dev,ccxt,server]'
|
|
331
|
+
```
|
|
332
|
+
|
|
333
|
+
No Git submodules are required. Docker is needed only for release-runtime and
|
|
334
|
+
end-to-end backtest work.
|
|
335
|
+
|
|
336
|
+
### Before opening a pull request
|
|
337
|
+
|
|
338
|
+
1. Keep the change focused and document any public API, report-schema, provider,
|
|
339
|
+
runtime-image, or cache-key compatibility impact.
|
|
340
|
+
2. Add deterministic offline tests; CI must not require credentials or live
|
|
341
|
+
provider access.
|
|
342
|
+
3. Keep credentials out of fixtures, logs, exception messages, and committed
|
|
343
|
+
configuration.
|
|
344
|
+
4. Run the standard checks:
|
|
345
|
+
|
|
346
|
+
```bash
|
|
347
|
+
.venv/bin/ruff format --check src tests
|
|
348
|
+
.venv/bin/ruff check .
|
|
349
|
+
.venv/bin/mypy src
|
|
350
|
+
.venv/bin/pytest
|
|
351
|
+
.venv/bin/python -m build
|
|
352
|
+
```
|
|
353
|
+
|
|
354
|
+
5. For Docker, FastAPI server, cache, or release-contract changes, also run:
|
|
355
|
+
|
|
356
|
+
```bash
|
|
357
|
+
PINEFORGE_DOCKER_TEST=1 .venv/bin/pytest tests/test_docker_integration.py
|
|
358
|
+
```
|
|
359
|
+
|
|
360
|
+
Read the [documentation home](https://pineforge-4pass.github.io/pineforge-data/)
|
|
361
|
+
and [CONTRIBUTING.md](https://github.com/pineforge-4pass/pineforge-data/blob/main/CONTRIBUTING.md) for provider requirements,
|
|
362
|
+
determinism rules, external provider entry points, and the complete checklist.
|
|
363
|
+
For broad changes to public models or the report contract, open an issue first
|
|
364
|
+
so providers and runtime consumers can agree on the shape before implementation.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
pineforge_data/__init__.py,sha256=__cVJ5xJIbGQfSmFxBApC9OkSuiMXnkPPZPCu1lRo5U,2991
|
|
2
|
+
pineforge_data/backtest.py,sha256=XEuTzE7gSmJGKG_AgC1kB_pGblcwxf2PFIFAkpVTdOQ,21119
|
|
3
|
+
pineforge_data/compile_cache.py,sha256=47x-H618QesZhDLghxHM2-dXTGNm5W19izTElYvaA0w,5645
|
|
4
|
+
pineforge_data/docker_runtime.py,sha256=ggMpNXhqZ4M_lvAAXYvY4NtriAP1I-g89lEGUTx-ZEg,7859
|
|
5
|
+
pineforge_data/engine.py,sha256=djCqdyEYbjrbi7eK6dKmTvLhajI8OQThJbsx8RW9Qxo,6721
|
|
6
|
+
pineforge_data/errors.py,sha256=qGT66oufMrj_a_aHIL4mv5t3XK5yvFo9RWH4MndUi0w,153
|
|
7
|
+
pineforge_data/models.py,sha256=X5I818hS68SEkTG-q9uU2qdc9UJGXWhOzXDIppFeCkI,6279
|
|
8
|
+
pineforge_data/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
9
|
+
pineforge_data/release_contract.py,sha256=wEflU-605RsdNgVI5UaTB1kAIngXX7b-lEA1MmPaOZM,5538
|
|
10
|
+
pineforge_data/requests.py,sha256=iX7wEiKMLU3dfwTOZqXoFVIjP_qf3k1V8PJB9GQOat0,3197
|
|
11
|
+
pineforge_data/server.py,sha256=id48AtB9rn_qT7AlCtcjP-hnTnXG4mUs2ZDdv3-wX8Q,23701
|
|
12
|
+
pineforge_data/server_client.py,sha256=weJnFdoeEM5y81tlJejtP_hWNru6J7bS9fsLdrTr-ls,5708
|
|
13
|
+
pineforge_data/cli/__init__.py,sha256=lCHfy77tj8312uwexeJG6-fJYlqY2gDkn4P5knKPgiI,58
|
|
14
|
+
pineforge_data/cli/backtest.py,sha256=-voKKnDPAdhiGNuFbI8vsm1lH8_VS5eE3lXkFa1y8Wg,13388
|
|
15
|
+
pineforge_data/providers/__init__.py,sha256=bdIdX0vAh4OInYBz19xHF5tg-zS_oJnjZhPtw-veLW8,1566
|
|
16
|
+
pineforge_data/providers/base.py,sha256=Oo_FImXWG6epF1dcyQCmlfXbPCQlT3PXEbZmscjKK8o,1430
|
|
17
|
+
pineforge_data/providers/ccxt.py,sha256=XVM5eAEgiEmcspw958tOAQ8_8HkG8URfNkdu-duykUE,15552
|
|
18
|
+
pineforge_data/providers/local.py,sha256=AiMHWI5RHm3wo-TQod7c8T8KABirLood9S0IrTEcxLY,7743
|
|
19
|
+
pineforge_data/providers/registry.py,sha256=A7Bip2jE7Hrr4uwG9Vuu_J903RaadLrimv32KO2MrYo,9565
|
|
20
|
+
pineforge_data/providers/sqlalchemy.py,sha256=1m8huLOMm_kxTVLZutwU2qnTBCZmDLGotWMcjGouuic,5081
|
|
21
|
+
pineforge_data/providers/tabular.py,sha256=1NjPy99Qly1PTGeLVYkVbH7jVo1kmwPMmkeICQGWDQ4,20813
|
|
22
|
+
pineforge_data-0.2.0.dist-info/METADATA,sha256=K6hPhQYVodRHbx8i9fLxj-oqwSNWTyb2p1dS_cA61cY,15750
|
|
23
|
+
pineforge_data-0.2.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
24
|
+
pineforge_data-0.2.0.dist-info/entry_points.txt,sha256=ecEJlP8Fd9hEH5kWhmpiBOqzm9Wu7gffn6oPrxyIdsQ,127
|
|
25
|
+
pineforge_data-0.2.0.dist-info/licenses/LICENSE,sha256=e48PSlRg3fD_D7eYddPiSQd1AWenf1AuQ149pF6sUrg,11352
|
|
26
|
+
pineforge_data-0.2.0.dist-info/RECORD,,
|