pineforge-data 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. pineforge_data-0.3.0/.github/workflows/ci.yml +90 -0
  2. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/CONTRIBUTING.md +14 -0
  3. pineforge_data-0.3.0/PKG-INFO +217 -0
  4. pineforge_data-0.3.0/README.md +167 -0
  5. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docker/server.Dockerfile +1 -1
  6. pineforge_data-0.3.0/docs/api/backtesting.md +50 -0
  7. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/api/server.md +5 -3
  8. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/backtesting.md +35 -10
  9. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/data-model.md +2 -2
  10. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/getting-started.md +3 -3
  11. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/index.md +3 -3
  12. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/providers/ccxt.md +3 -2
  13. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/providers/csv.md +0 -1
  14. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/providers/sqlalchemy.md +25 -1
  15. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/providers/tabular-schema.md +4 -3
  16. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/server.md +7 -3
  17. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/mkdocs.yml +1 -1
  18. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/pyproject.toml +9 -4
  19. pineforge_data-0.3.0/scripts/run_database_e2e.sh +33 -0
  20. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/backtest.py +48 -6
  21. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/release_contract.py +3 -3
  22. pineforge_data-0.3.0/tests/database/docker-compose.yml +33 -0
  23. pineforge_data-0.3.0/tests/fixtures/CORPUS.md +17 -0
  24. pineforge_data-0.3.0/tests/fixtures/corpus_ohlcv_ETH-USDT-USDT_1m_sample.csv +257 -0
  25. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/tests/fixtures/sma_cross.pine +1 -1
  26. pineforge_data-0.3.0/tests/in_process_release_check.py +283 -0
  27. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/tests/test_backtest.py +40 -4
  28. pineforge_data-0.3.0/tests/test_database_e2e.py +245 -0
  29. pineforge_data-0.3.0/tests/test_docker_integration.py +279 -0
  30. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/tests/test_docker_runtime.py +1 -1
  31. pineforge_data-0.2.0/.github/workflows/ci.yml +0 -43
  32. pineforge_data-0.2.0/PKG-INFO +0 -364
  33. pineforge_data-0.2.0/README.md +0 -318
  34. pineforge_data-0.2.0/docs/api/backtesting.md +0 -31
  35. pineforge_data-0.2.0/tests/test_docker_integration.py +0 -165
  36. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/.dockerignore +0 -0
  37. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/.github/release.yml +0 -0
  38. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/.github/workflows/docs.yml +0 -0
  39. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/.github/workflows/release.yml +0 -0
  40. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/.gitignore +0 -0
  41. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/LICENSE +0 -0
  42. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/api/index.md +0 -0
  43. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/api/models.md +0 -0
  44. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/api/providers.md +0 -0
  45. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/api/requests.md +0 -0
  46. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/api/streaming.md +0 -0
  47. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/provider-contract.md +0 -0
  48. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/providers/sqlite.md +0 -0
  49. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/providers.md +0 -0
  50. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/docs/releasing.md +0 -0
  51. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/scripts/backtest.py +0 -0
  52. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/scripts/bump_version.py +0 -0
  53. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/scripts/verify_release.py +0 -0
  54. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/__init__.py +0 -0
  55. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/cli/__init__.py +0 -0
  56. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/cli/backtest.py +0 -0
  57. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/compile_cache.py +0 -0
  58. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/docker_runtime.py +0 -0
  59. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/engine.py +0 -0
  60. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/errors.py +0 -0
  61. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/models.py +0 -0
  62. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/providers/__init__.py +0 -0
  63. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/providers/base.py +0 -0
  64. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/providers/ccxt.py +0 -0
  65. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/providers/local.py +0 -0
  66. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/providers/registry.py +0 -0
  67. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/providers/sqlalchemy.py +0 -0
  68. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/providers/tabular.py +0 -0
  69. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/py.typed +0 -0
  70. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/requests.py +0 -0
  71. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/server.py +0 -0
  72. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/src/pineforge_data/server_client.py +0 -0
  73. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/tests/test_ccxt.py +0 -0
  74. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/tests/test_compile_cache.py +0 -0
  75. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/tests/test_documentation.py +0 -0
  76. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/tests/test_engine.py +0 -0
  77. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/tests/test_models.py +0 -0
  78. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/tests/test_providers.py +0 -0
  79. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/tests/test_release.py +0 -0
  80. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/tests/test_release_contract.py +0 -0
  81. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/tests/test_server.py +0 -0
  82. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/tests/test_server_client.py +0 -0
  83. {pineforge_data-0.2.0 → pineforge_data-0.3.0}/tests/test_tabular_providers.py +0 -0
@@ -0,0 +1,90 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ permissions:
9
+ contents: read
10
+
11
+ jobs:
12
+ test:
13
+ runs-on: ubuntu-latest
14
+ strategy:
15
+ fail-fast: false
16
+ matrix:
17
+ python-version: ["3.11", "3.12", "3.13"]
18
+ steps:
19
+ - uses: actions/checkout@v7
20
+ - uses: actions/setup-python@v6
21
+ with:
22
+ python-version: ${{ matrix.python-version }}
23
+ cache: pip
24
+ - run: python -m pip install --upgrade pip
25
+ - run: python -m pip install -e '.[dev,ccxt,release]'
26
+ - run: ruff check .
27
+ - run: mypy src
28
+ - run: pytest
29
+ - if: matrix.python-version == '3.13'
30
+ run: python -m build
31
+ - if: matrix.python-version == '3.13'
32
+ run: python -m twine check --strict dist/*
33
+
34
+ docker:
35
+ runs-on: ubuntu-latest
36
+ steps:
37
+ - uses: actions/checkout@v7
38
+ - uses: actions/setup-python@v6
39
+ with:
40
+ python-version: "3.13"
41
+ cache: pip
42
+ - run: python -m pip install -e '.[dev]'
43
+ - run: PINEFORGE_DOCKER_TEST=1 pytest tests/test_docker_integration.py
44
+
45
+ database-e2e:
46
+ runs-on: ubuntu-latest
47
+ services:
48
+ mysql:
49
+ image: mysql:8.4
50
+ env:
51
+ MYSQL_DATABASE: pineforge
52
+ MYSQL_USER: pineforge
53
+ MYSQL_PASSWORD: pineforge
54
+ MYSQL_ROOT_PASSWORD: pineforge-root
55
+ ports:
56
+ - 3306:3306
57
+ options: >-
58
+ --health-cmd="mysqladmin ping --host=127.0.0.1 --user=pineforge --password=pineforge --silent"
59
+ --health-interval=2s
60
+ --health-timeout=5s
61
+ --health-retries=30
62
+ --health-start-period=15s
63
+ postgres:
64
+ image: postgres:17
65
+ env:
66
+ POSTGRES_DB: pineforge
67
+ POSTGRES_USER: pineforge
68
+ POSTGRES_PASSWORD: pineforge
69
+ ports:
70
+ - 5432:5432
71
+ options: >-
72
+ --health-cmd="pg_isready --username=pineforge --dbname=pineforge"
73
+ --health-interval=2s
74
+ --health-timeout=5s
75
+ --health-retries=30
76
+ --health-start-period=10s
77
+ env:
78
+ PINEFORGE_DATABASE_E2E: "1"
79
+ PINEFORGE_CORPUS_OHLCV: tests/fixtures/corpus_ohlcv_ETH-USDT-USDT_1m_sample.csv
80
+ PINEFORGE_MYSQL_URL: mysql+pymysql://pineforge:pineforge@127.0.0.1:3306/pineforge
81
+ PINEFORGE_POSTGRES_URL: postgresql+psycopg://pineforge:pineforge@127.0.0.1:5432/pineforge
82
+ steps:
83
+ - uses: actions/checkout@v7
84
+ - uses: actions/setup-python@v6
85
+ with:
86
+ python-version: "3.13"
87
+ cache: pip
88
+ - run: python -m pip install --upgrade pip
89
+ - run: python -m pip install -e '.[dev,database-e2e]'
90
+ - run: pytest tests/test_database_e2e.py
@@ -77,6 +77,20 @@ python -m twine check dist/*
77
77
  PINEFORGE_DOCKER_TEST=1 pytest tests/test_docker_integration.py
78
78
  ```
79
79
 
80
+ Database-provider changes must also pass the corpus-backed database suite:
81
+
82
+ ```bash
83
+ python -m pip install -e '.[dev,database-e2e]'
84
+ ./scripts/run_database_e2e.sh
85
+ ```
86
+
87
+ The harness starts disposable MySQL and PostgreSQL containers and creates a
88
+ temporary SQLite database. If the sibling engine corpus exists, it streams the
89
+ first requested rows from
90
+ `../pineforge-engine/corpus/data/ohlcv_ETH-USDT-USDT_1m.csv`; otherwise it uses
91
+ the provenance-pinned sample under `tests/fixtures/`. Override the source or
92
+ row count with `PINEFORGE_CORPUS_OHLCV` and `PINEFORGE_DATABASE_E2E_ROWS`.
93
+
80
94
  Maintainers should follow the version, tag, artifact, and Trusted Publishing
81
95
  steps in the [release guide](docs/releasing.md). Contributors cannot publish a
82
96
  package from a pull request.
@@ -0,0 +1,217 @@
1
+ Metadata-Version: 2.5
2
+ Name: pineforge-data
3
+ Version: 0.3.0
4
+ Summary: Provider-neutral market data adapters and raw-PineScript backtest harness for PineForge
5
+ Project-URL: Homepage, https://github.com/pineforge-4pass/pineforge-data
6
+ Project-URL: Documentation, https://pineforge-4pass.github.io/pineforge-data/
7
+ Project-URL: Issues, https://github.com/pineforge-4pass/pineforge-data/issues
8
+ Project-URL: Source, https://github.com/pineforge-4pass/pineforge-data
9
+ Author: PineForge contributors
10
+ License: Apache-2.0
11
+ License-File: LICENSE
12
+ Keywords: backtesting,market-data,pinescript
13
+ Classifier: Development Status :: 3 - Alpha
14
+ Classifier: License :: OSI Approved :: Apache Software License
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Typing :: Typed
20
+ Requires-Python: >=3.11
21
+ Provides-Extra: ccxt
22
+ Requires-Dist: ccxt<5,>=4.5.64; extra == 'ccxt'
23
+ Provides-Extra: database
24
+ Requires-Dist: sqlalchemy<3,>=2.0; extra == 'database'
25
+ Provides-Extra: database-e2e
26
+ Requires-Dist: psycopg[binary]<4,>=3.2; extra == 'database-e2e'
27
+ Requires-Dist: pymysql<2,>=1.1; extra == 'database-e2e'
28
+ Requires-Dist: sqlalchemy<3,>=2.0; extra == 'database-e2e'
29
+ Provides-Extra: dev
30
+ Requires-Dist: build>=1.2; extra == 'dev'
31
+ Requires-Dist: fastapi<0.140,>=0.139; extra == 'dev'
32
+ Requires-Dist: httpx2<2.6,>=2.5; extra == 'dev'
33
+ Requires-Dist: mypy>=1.13; extra == 'dev'
34
+ Requires-Dist: pytest>=8.3; extra == 'dev'
35
+ Requires-Dist: ruff>=0.8; extra == 'dev'
36
+ Requires-Dist: sqlalchemy<3,>=2.0; extra == 'dev'
37
+ Requires-Dist: uvicorn<0.52,>=0.51; extra == 'dev'
38
+ Provides-Extra: docs
39
+ Requires-Dist: mkdocs-material<10,>=9.7; extra == 'docs'
40
+ Requires-Dist: mkdocs<2,>=1.6; extra == 'docs'
41
+ Requires-Dist: mkdocstrings-python<3,>=2.0; extra == 'docs'
42
+ Requires-Dist: ruff>=0.8; extra == 'docs'
43
+ Provides-Extra: release
44
+ Requires-Dist: build<2,>=1.2; extra == 'release'
45
+ Requires-Dist: twine<8,>=7; extra == 'release'
46
+ Provides-Extra: server
47
+ Requires-Dist: fastapi<0.140,>=0.139; extra == 'server'
48
+ Requires-Dist: uvicorn<0.52,>=0.51; extra == 'server'
49
+ Description-Content-Type: text/markdown
50
+
51
+ # PineForge Data
52
+
53
+ [![PyPI](https://img.shields.io/pypi/v/pineforge-data.svg)](https://pypi.org/project/pineforge-data/)
54
+ [![Python](https://img.shields.io/pypi/pyversions/pineforge-data.svg)](https://pypi.org/project/pineforge-data/)
55
+ [![CI](https://github.com/pineforge-4pass/pineforge-data/actions/workflows/ci.yml/badge.svg)](https://github.com/pineforge-4pass/pineforge-data/actions/workflows/ci.yml)
56
+ [![Documentation](https://img.shields.io/badge/docs-GitHub%20Pages-blue)](https://pineforge-4pass.github.io/pineforge-data/)
57
+
58
+ Fetch normalized market data and run raw PineScript strategies through
59
+ PineForge.
60
+
61
+ PineForge Data provides Python adapters for exchanges, CSV files, SQLite, and
62
+ SQLAlchemy-compatible databases. Its backtest harness sends normalized OHLCV
63
+ to the `pineforge-release` container, which transpiles the PineScript, compiles
64
+ it, and runs `pineforge-engine`.
65
+
66
+ ```text
67
+ exchange / CSV / database
68
+ ↓
69
+ pineforge-data
70
+ ↓
71
+ pineforge-release container
72
+ ↓
73
+ backtest report
74
+ ```
75
+
76
+ The provider layer is Python-only. You do not need a C++ compiler, Git
77
+ submodules, or local engine and codegen checkouts.
78
+
79
+ ## Quick start
80
+
81
+ You need Python 3.11 or newer. Docker is required only for backtests; fetching
82
+ and normalizing data does not use Docker.
83
+
84
+ ### 1. Install an exchange provider
85
+
86
+ ```bash
87
+ python -m pip install 'pineforge-data[ccxt]'
88
+ ```
89
+
90
+ ### 2. Fetch confirmed OHLCV
91
+
92
+ ```python
93
+ import asyncio
94
+
95
+ from pineforge_data import BarRequest, CcxtProvider
96
+
97
+
98
+ async def main() -> None:
99
+ async with CcxtProvider("bitstamp") as provider:
100
+ market = await provider.resolve_market("BTC/USD")
101
+ bars = await provider.fetch_bars(
102
+ BarRequest(
103
+ instrument=market.instrument,
104
+ timeframe="15m",
105
+ start_ms=1_751_328_000_000,
106
+ end_ms=1_751_414_400_000,
107
+ )
108
+ )
109
+ print(bars[-1])
110
+
111
+
112
+ asyncio.run(main())
113
+ ```
114
+
115
+ Symbols are resolved through the provider catalog. For example, CCXT spot
116
+ `BTC/USDT` and linear swap `BTC/USDT:USDT` are distinct instruments with
117
+ different contract metadata.
118
+
119
+ ### 3. Backtest raw PineScript
120
+
121
+ Save a strategy as `strategy.pine`:
122
+
123
+ ```pinescript
124
+ //@version=6
125
+ strategy("SMA cross", initial_capital=10000, default_qty_type=strategy.percent_of_equity, default_qty_value=10)
126
+
127
+ fast = ta.sma(close, 2)
128
+ slow = ta.sma(close, 4)
129
+
130
+ if ta.crossover(fast, slow)
131
+ strategy.entry("Long", strategy.long)
132
+
133
+ if ta.crossunder(fast, slow)
134
+ strategy.close("Long")
135
+ ```
136
+
137
+ Then run:
138
+
139
+ ```bash
140
+ pineforge-backtest \
141
+ --pine strategy.pine \
142
+ --provider ccxt \
143
+ --venue bitstamp \
144
+ --symbol BTC/USD \
145
+ --timeframe 15m \
146
+ --start 2025-07-01T00:00:00Z \
147
+ --end 2025-07-08T00:00:00Z \
148
+ --warmup-bars 100 \
149
+ --output report.json \
150
+ --pretty
151
+ ```
152
+
153
+ The first run pulls the digest-pinned `pineforge-release` 1.0.0 image (engine
154
+ 1.0.0, codegen 1.0.0). Results can differ between releases, notably for scripts
155
+ that omit their capital or order size; the
156
+ [runtime image policy](https://pineforge-4pass.github.io/pineforge-data/backtesting/#runtime-image-policy)
157
+ explains why and how to pin another release with `--runtime-image`.
158
+ The JSON report includes trades, performance
159
+ statistics, the equity curve, data provenance, and exact runtime versions. The
160
+ strategy is compiled inside an isolated Docker container; the provider and its
161
+ credentials remain on the host.
162
+
163
+ ## Bring your own data
164
+
165
+ PineForge does not require a fixed table definition. Local providers inspect
166
+ headers or reflected database columns at runtime, infer common OHLCV names,
167
+ and accept an explicit mapping when your schema uses different names.
168
+
169
+ | Source | Install | Guide |
170
+ |---|---|---|
171
+ | CSV | `pip install pineforge-data` | [CSV API](https://pineforge-4pass.github.io/pineforge-data/providers/csv/) |
172
+ | SQLite | `pip install pineforge-data` | [SQLite API](https://pineforge-4pass.github.io/pineforge-data/providers/sqlite/) |
173
+ | SQLAlchemy database | `pip install 'pineforge-data[database]'` plus a database driver | [SQLAlchemy API](https://pineforge-4pass.github.io/pineforge-data/providers/sqlalchemy/) |
174
+ | CCXT exchange | `pip install 'pineforge-data[ccxt]'` | [CCXT API](https://pineforge-4pass.github.io/pineforge-data/providers/ccxt/) |
175
+
176
+ The same `pineforge-backtest` command supports `--provider csv`, `sqlite`, and
177
+ `sqlalchemy`. See the [backtesting guide](https://pineforge-4pass.github.io/pineforge-data/backtesting/)
178
+ for provider configuration, warmup behavior, strategy inputs, reports, and
179
+ local versus remote execution.
180
+
181
+ ## Documentation
182
+
183
+ | I want to… | Read… |
184
+ |---|---|
185
+ | Install and run the first request | [Getting started](https://pineforge-4pass.github.io/pineforge-data/getting-started/) |
186
+ | Look up Python classes and signatures | [API reference](https://pineforge-4pass.github.io/pineforge-data/api/) |
187
+ | Choose or configure a provider | [Provider catalog](https://pineforge-4pass.github.io/pineforge-data/providers/) |
188
+ | Understand instruments, contracts, and bars | [Data model](https://pineforge-4pass.github.io/pineforge-data/data-model/) |
189
+ | Run and reproduce backtests | [Backtesting](https://pineforge-4pass.github.io/pineforge-data/backtesting/) |
190
+ | Deploy the concurrent compile/backtest service | [FastAPI server](https://pineforge-4pass.github.io/pineforge-data/server/) |
191
+ | Implement a broker or exchange adapter | [Provider contract](https://pineforge-4pass.github.io/pineforge-data/provider-contract/) |
192
+
193
+ ## Contributing
194
+
195
+ Community providers are welcome. Keep vendor SDKs and schemas inside the
196
+ adapter, resolve instruments through the upstream market catalog, normalize
197
+ timestamps to Unix milliseconds, and add deterministic offline tests.
198
+
199
+ ```bash
200
+ git clone https://github.com/pineforge-4pass/pineforge-data.git
201
+ cd pineforge-data
202
+ python -m pip install -e '.[dev,ccxt,database-e2e,server,docs,release]'
203
+ ruff check .
204
+ mypy src
205
+ pytest
206
+ mkdocs build --strict
207
+ ```
208
+
209
+ Run `./scripts/run_database_e2e.sh` to seed corpus OHLCV into disposable
210
+ SQLite, MySQL, and PostgreSQL databases and verify the public provider API
211
+ against all three.
212
+
213
+ Read [CONTRIBUTING.md](https://github.com/pineforge-4pass/pineforge-data/blob/main/CONTRIBUTING.md)
214
+ for the provider contract, entry points, security rules, and full validation
215
+ checklist.
216
+
217
+ Apache-2.0 licensed.
@@ -0,0 +1,167 @@
1
+ # PineForge Data
2
+
3
+ [![PyPI](https://img.shields.io/pypi/v/pineforge-data.svg)](https://pypi.org/project/pineforge-data/)
4
+ [![Python](https://img.shields.io/pypi/pyversions/pineforge-data.svg)](https://pypi.org/project/pineforge-data/)
5
+ [![CI](https://github.com/pineforge-4pass/pineforge-data/actions/workflows/ci.yml/badge.svg)](https://github.com/pineforge-4pass/pineforge-data/actions/workflows/ci.yml)
6
+ [![Documentation](https://img.shields.io/badge/docs-GitHub%20Pages-blue)](https://pineforge-4pass.github.io/pineforge-data/)
7
+
8
+ Fetch normalized market data and run raw PineScript strategies through
9
+ PineForge.
10
+
11
+ PineForge Data provides Python adapters for exchanges, CSV files, SQLite, and
12
+ SQLAlchemy-compatible databases. Its backtest harness sends normalized OHLCV
13
+ to the `pineforge-release` container, which transpiles the PineScript, compiles
14
+ it, and runs `pineforge-engine`.
15
+
16
+ ```text
17
+ exchange / CSV / database
18
+ ↓
19
+ pineforge-data
20
+ ↓
21
+ pineforge-release container
22
+ ↓
23
+ backtest report
24
+ ```
25
+
26
+ The provider layer is Python-only. You do not need a C++ compiler, Git
27
+ submodules, or local engine and codegen checkouts.
28
+
29
+ ## Quick start
30
+
31
+ You need Python 3.11 or newer. Docker is required only for backtests; fetching
32
+ and normalizing data does not use Docker.
33
+
34
+ ### 1. Install an exchange provider
35
+
36
+ ```bash
37
+ python -m pip install 'pineforge-data[ccxt]'
38
+ ```
39
+
40
+ ### 2. Fetch confirmed OHLCV
41
+
42
+ ```python
43
+ import asyncio
44
+
45
+ from pineforge_data import BarRequest, CcxtProvider
46
+
47
+
48
+ async def main() -> None:
49
+ async with CcxtProvider("bitstamp") as provider:
50
+ market = await provider.resolve_market("BTC/USD")
51
+ bars = await provider.fetch_bars(
52
+ BarRequest(
53
+ instrument=market.instrument,
54
+ timeframe="15m",
55
+ start_ms=1_751_328_000_000,
56
+ end_ms=1_751_414_400_000,
57
+ )
58
+ )
59
+ print(bars[-1])
60
+
61
+
62
+ asyncio.run(main())
63
+ ```
64
+
65
+ Symbols are resolved through the provider catalog. For example, CCXT spot
66
+ `BTC/USDT` and linear swap `BTC/USDT:USDT` are distinct instruments with
67
+ different contract metadata.
68
+
69
+ ### 3. Backtest raw PineScript
70
+
71
+ Save a strategy as `strategy.pine`:
72
+
73
+ ```pinescript
74
+ //@version=6
75
+ strategy("SMA cross", initial_capital=10000, default_qty_type=strategy.percent_of_equity, default_qty_value=10)
76
+
77
+ fast = ta.sma(close, 2)
78
+ slow = ta.sma(close, 4)
79
+
80
+ if ta.crossover(fast, slow)
81
+ strategy.entry("Long", strategy.long)
82
+
83
+ if ta.crossunder(fast, slow)
84
+ strategy.close("Long")
85
+ ```
86
+
87
+ Then run:
88
+
89
+ ```bash
90
+ pineforge-backtest \
91
+ --pine strategy.pine \
92
+ --provider ccxt \
93
+ --venue bitstamp \
94
+ --symbol BTC/USD \
95
+ --timeframe 15m \
96
+ --start 2025-07-01T00:00:00Z \
97
+ --end 2025-07-08T00:00:00Z \
98
+ --warmup-bars 100 \
99
+ --output report.json \
100
+ --pretty
101
+ ```
102
+
103
+ The first run pulls the digest-pinned `pineforge-release` 1.0.0 image (engine
104
+ 1.0.0, codegen 1.0.0). Results can differ between releases, notably for scripts
105
+ that omit their capital or order size; the
106
+ [runtime image policy](https://pineforge-4pass.github.io/pineforge-data/backtesting/#runtime-image-policy)
107
+ explains why and how to pin another release with `--runtime-image`.
108
+ The JSON report includes trades, performance
109
+ statistics, the equity curve, data provenance, and exact runtime versions. The
110
+ strategy is compiled inside an isolated Docker container; the provider and its
111
+ credentials remain on the host.
112
+
113
+ ## Bring your own data
114
+
115
+ PineForge does not require a fixed table definition. Local providers inspect
116
+ headers or reflected database columns at runtime, infer common OHLCV names,
117
+ and accept an explicit mapping when your schema uses different names.
118
+
119
+ | Source | Install | Guide |
120
+ |---|---|---|
121
+ | CSV | `pip install pineforge-data` | [CSV API](https://pineforge-4pass.github.io/pineforge-data/providers/csv/) |
122
+ | SQLite | `pip install pineforge-data` | [SQLite API](https://pineforge-4pass.github.io/pineforge-data/providers/sqlite/) |
123
+ | SQLAlchemy database | `pip install 'pineforge-data[database]'` plus a database driver | [SQLAlchemy API](https://pineforge-4pass.github.io/pineforge-data/providers/sqlalchemy/) |
124
+ | CCXT exchange | `pip install 'pineforge-data[ccxt]'` | [CCXT API](https://pineforge-4pass.github.io/pineforge-data/providers/ccxt/) |
125
+
126
+ The same `pineforge-backtest` command supports `--provider csv`, `sqlite`, and
127
+ `sqlalchemy`. See the [backtesting guide](https://pineforge-4pass.github.io/pineforge-data/backtesting/)
128
+ for provider configuration, warmup behavior, strategy inputs, reports, and
129
+ local versus remote execution.
130
+
131
+ ## Documentation
132
+
133
+ | I want to… | Read… |
134
+ |---|---|
135
+ | Install and run the first request | [Getting started](https://pineforge-4pass.github.io/pineforge-data/getting-started/) |
136
+ | Look up Python classes and signatures | [API reference](https://pineforge-4pass.github.io/pineforge-data/api/) |
137
+ | Choose or configure a provider | [Provider catalog](https://pineforge-4pass.github.io/pineforge-data/providers/) |
138
+ | Understand instruments, contracts, and bars | [Data model](https://pineforge-4pass.github.io/pineforge-data/data-model/) |
139
+ | Run and reproduce backtests | [Backtesting](https://pineforge-4pass.github.io/pineforge-data/backtesting/) |
140
+ | Deploy the concurrent compile/backtest service | [FastAPI server](https://pineforge-4pass.github.io/pineforge-data/server/) |
141
+ | Implement a broker or exchange adapter | [Provider contract](https://pineforge-4pass.github.io/pineforge-data/provider-contract/) |
142
+
143
+ ## Contributing
144
+
145
+ Community providers are welcome. Keep vendor SDKs and schemas inside the
146
+ adapter, resolve instruments through the upstream market catalog, normalize
147
+ timestamps to Unix milliseconds, and add deterministic offline tests.
148
+
149
+ ```bash
150
+ git clone https://github.com/pineforge-4pass/pineforge-data.git
151
+ cd pineforge-data
152
+ python -m pip install -e '.[dev,ccxt,database-e2e,server,docs,release]'
153
+ ruff check .
154
+ mypy src
155
+ pytest
156
+ mkdocs build --strict
157
+ ```
158
+
159
+ Run `./scripts/run_database_e2e.sh` to seed corpus OHLCV into disposable
160
+ SQLite, MySQL, and PostgreSQL databases and verify the public provider API
161
+ against all three.
162
+
163
+ Read [CONTRIBUTING.md](https://github.com/pineforge-4pass/pineforge-data/blob/main/CONTRIBUTING.md)
164
+ for the provider contract, entry points, security rules, and full validation
165
+ checklist.
166
+
167
+ Apache-2.0 licensed.
@@ -1,6 +1,6 @@
1
1
  # syntax=docker/dockerfile:1@sha256:87999aa3d42bdc6bea60565083ee17e86d1f3339802f543c0d03998580f9cb89
2
2
 
3
- ARG PINEFORGE_RELEASE_IMAGE=ghcr.io/pineforge-4pass/pineforge-release:0.1.12@sha256:312b9d908390b828484617472c749d5815feb75507da87eae2f6902cfe3d47b1
3
+ ARG PINEFORGE_RELEASE_IMAGE=ghcr.io/pineforge-4pass/pineforge-release:1.0.0@sha256:b271873d0e91a9eeeb4595daaded9bdfbdbcd3925d3ec1300a833395e2004c86
4
4
 
5
5
  FROM python:3.11-slim-bookworm@sha256:f5cf0344c9886ff24d34797578d5d7dd6e8911ae0fe5962bb55d0f89603ec361 AS server-builder
6
6
  RUN python -m pip install --no-cache-dir --target /opt/pineforge-server-deps \
@@ -0,0 +1,50 @@
1
+ # Backtesting
2
+
3
+ ## Runtime options and reports
4
+
5
+ ::: pineforge_data.backtest.BacktestOptions
6
+
7
+ ::: pineforge_data.backtest.MagnifierDistribution
8
+
9
+ ::: pineforge_data.backtest.BacktestReport
10
+
11
+ `PineForgeBacktestRunner` runs a compiled strategy library in-process. It
12
+ accepts libraries that report PineForge C ABI 4, which engine 1.x builds. It
13
+ refuses any other library before any strategy call, with an
14
+ `EngineBacktestError` that says what to do. Libraries without
15
+ `pf_abi_version` (engine v0.10.1 or earlier), ABI 2 (engine v0.10.2 through
16
+ v0.12.3) and ABI 3 (engine v0.13.x) need a rebuild with engine 1.x;
17
+ pineforge-data 0.2.0 still reads ABI 2. A newer ABI needs a newer
18
+ pineforge-data. The release container and the FastAPI server do not use this
19
+ class.
20
+
21
+ A trade's `max_runup` and `max_drawdown` are whole-trade excursions in account
22
+ currency, net of entry fees, as engine 1.0's `pineforge.h` documents them.
23
+ Headers before 1.0 described them as price travel per unit of quantity, but the
24
+ values did not change: on the README quick start, engines 0.11.0 and 1.0.0
25
+ report identical excursions for the 88 trades both report. A trade whose
26
+ `open_at_end` is true closes a position still open after the final bar, at that
27
+ bar's close rounded to the tick size, without slippage. Engine 0.11.0 reports no
28
+ such row, so the README quick start has 88 trades on it and 89 on engine 1.0.0.
29
+
30
+ ::: pineforge_data.backtest.PineForgeBacktestRunner
31
+
32
+ ::: pineforge_data.backtest.EngineBacktestError
33
+
34
+ ## Published release container
35
+
36
+ ::: pineforge_data.docker_runtime.DockerBacktestRuntime
37
+
38
+ ::: pineforge_data.docker_runtime.DockerPrerequisiteError
39
+
40
+ ::: pineforge_data.docker_runtime.DockerExecutionError
41
+
42
+ ::: pineforge_data.release_contract.DEFAULT_RELEASE_IMAGE
43
+
44
+ ::: pineforge_data.release_contract.ReleaseContractError
45
+
46
+ ## Remote FastAPI client
47
+
48
+ ::: pineforge_data.server_client.FastApiBacktestClient
49
+
50
+ ::: pineforge_data.server_client.BacktestServerError
@@ -1,8 +1,10 @@
1
1
  # Server components
2
2
 
3
- Most users run the packaged `pineforge-backtest-server` command or submit work
4
- through `FastApiBacktestClient`. These objects support applications that embed
5
- the service or manage its compiled-strategy cache directly.
3
+ Most users run the server image built from `docker/server.Dockerfile` or submit
4
+ work through `FastApiBacktestClient`. The packaged `pineforge-backtest-server`
5
+ command reports ready only where the `pineforge-release` toolchain is
6
+ installed, as in that image. These objects support applications that embed the
7
+ service or manage its compiled-strategy cache directly.
6
8
 
7
9
  ::: pineforge_data.server.create_app
8
10
 
@@ -29,7 +29,7 @@ are documented in the [server guide](server.md).
29
29
  pineforge-backtest \
30
30
  --pine strategy.pine \
31
31
  --provider ccxt \
32
- --venue kraken \
32
+ --venue bitstamp \
33
33
  --symbol BTC/USD \
34
34
  --timeframe 15m \
35
35
  --start 2025-07-01T00:00:00Z \
@@ -60,7 +60,7 @@ strategy to place orders during that earlier interval:
60
60
  pineforge-backtest \
61
61
  --pine strategy.pine \
62
62
  --provider ccxt \
63
- --venue kraken \
63
+ --venue bitstamp \
64
64
  --symbol BTC/USD \
65
65
  --timeframe 15m \
66
66
  --start 2025-07-01T00:00:00Z \
@@ -132,9 +132,29 @@ guides.
132
132
 
133
133
  ## Runtime image policy
134
134
 
135
- The package default is an exact `pineforge-release` version and OCI digest. The
136
- `missing` pull policy downloads it only when absent; `never` supports offline
137
- runs; `always` refreshes a tag before running.
135
+ The package default is an exact `pineforge-release` version and OCI digest:
136
+ 1.0.0, which bundles engine 1.0.0 and codegen 1.0.0. The `missing` pull
137
+ policy downloads it only when absent; `never` supports offline runs; `always`
138
+ refreshes a tag before running.
139
+
140
+ The previous default, release 0.1.12 (engine 0.11.0, codegen 0.9.0), accepts the
141
+ same inputs. To reproduce results made with it, pin it by digest:
142
+
143
+ ```bash
144
+ --runtime-image ghcr.io/pineforge-4pass/pineforge-release:0.1.12@sha256:312b9d908390b828484617472c749d5815feb75507da87eae2f6902cfe3d47b1
145
+ ```
146
+
147
+ Results can differ between releases: the README quick start reports 89 trades
148
+ on 1.0.0 and 88 on 0.1.12. The first 88 are the same trades; 1.0.0 adds the
149
+ close of the position still open after the last bar, marked `open_at_end`.
150
+
151
+ Order sizes differ too when a script omits `initial_capital`,
152
+ `default_qty_type` or `default_qty_value`. Codegen 1.0 gives them TradingView's
153
+ Pine v6 defaults: 100000, `strategy.percent_of_equity` and 100. Release 0.1.12
154
+ used 1000000, `strategy.fixed` and 1. At 100% of equity, the default 100% margin
155
+ refuses an entry that fills above its signal price, so such a script can trade
156
+ less on 1.0.0. Declare all three settings, as the README strategy does, to size
157
+ orders the same way on both releases.
138
158
 
139
159
  The rolling channel is explicit:
140
160
 
@@ -155,13 +175,13 @@ The harness combines provider provenance with the release report:
155
175
  "schema_version": 1,
156
176
  "request_id": null,
157
177
  "provider": {
158
- "name": "ccxt:kraken",
178
+ "name": "ccxt:bitstamp",
159
179
  "adapter": "ccxt",
160
- "venue": "kraken",
180
+ "venue": "bitstamp",
161
181
  "source_timeframe": "15m",
162
182
  "market": {
163
183
  "symbol": "BTC/USD",
164
- "provider_id": "XXBTZUSD",
184
+ "provider_id": "btcusd",
165
185
  "market_type": "spot",
166
186
  "contract": null
167
187
  }
@@ -169,8 +189,8 @@ The harness combines provider provenance with the release report:
169
189
  "data": {
170
190
  "requested_start_ms": 1751328000000,
171
191
  "requested_end_ms": 1751932800000,
172
- "provider_start_ms": 1746828000000,
173
- "first_bar_ms": 1746828000000,
192
+ "provider_start_ms": 1750878000000,
193
+ "first_bar_ms": 1750878000000,
174
194
  "last_bar_ms": 1751931900000,
175
195
  "bars": 1172,
176
196
  "requested_bars": 672,
@@ -178,6 +198,11 @@ The harness combines provider provenance with the release report:
178
198
  "warmup_bars_loaded": 500,
179
199
  "trade_start_time_ms": 1751328000000
180
200
  },
201
+ "strategy": {
202
+ "pine": "/path/to/strategy.pine",
203
+ "input_timeframe": "15",
204
+ "script_timeframe": "15"
205
+ },
181
206
  "runtime": {
182
207
  "mode": "local-container",
183
208
  "release_image": "ghcr.io/...@sha256:..."
@@ -61,8 +61,8 @@ swap = Instrument(
61
61
  ```
62
62
 
63
63
  `ContractSpec` can also carry `expiry_ms`, `strike`, and `option_type`. A
64
- contract cannot be both linear and inverse, and numeric contract terms must be
65
- finite and positive.
64
+ contract cannot be both linear and inverse, and contract size and strike must
65
+ be finite and positive.
66
66
 
67
67
  ## Market listings and discovery
68
68