convexpi-lab 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- convexpi_lab-0.1.0/.github/workflows/anomalies.yml +49 -0
- convexpi_lab-0.1.0/.github/workflows/ci.yml +30 -0
- convexpi_lab-0.1.0/.github/workflows/forward.yml +31 -0
- convexpi_lab-0.1.0/.github/workflows/publish.yml +29 -0
- convexpi_lab-0.1.0/.gitignore +13 -0
- convexpi_lab-0.1.0/LICENSE +21 -0
- convexpi_lab-0.1.0/PKG-INFO +91 -0
- convexpi_lab-0.1.0/README.md +58 -0
- convexpi_lab-0.1.0/deploy/Dockerfile.grader +21 -0
- convexpi_lab-0.1.0/deploy/compute_anomaly_stats.py +57 -0
- convexpi_lab-0.1.0/deploy/forward_runner.py +264 -0
- convexpi_lab-0.1.0/deploy/grader_worker.py +419 -0
- convexpi_lab-0.1.0/deploy/seed_demo_cohort.py +184 -0
- convexpi_lab-0.1.0/examples/lab_demo.py +42 -0
- convexpi_lab-0.1.0/pyproject.toml +56 -0
- convexpi_lab-0.1.0/src/convexpi/lab/__init__.py +62 -0
- convexpi_lab-0.1.0/src/convexpi/lab/anomalies.py +342 -0
- convexpi_lab-0.1.0/src/convexpi/lab/backtest.py +336 -0
- convexpi_lab-0.1.0/src/convexpi/lab/grader.py +361 -0
- convexpi_lab-0.1.0/src/convexpi/lab/real_data.py +570 -0
- convexpi_lab-0.1.0/src/convexpi/lab/strategies.py +791 -0
- convexpi_lab-0.1.0/src/convexpi/lab/synth.py +352 -0
- convexpi_lab-0.1.0/tests/__init__.py +1 -0
- convexpi_lab-0.1.0/tests/integration/__init__.py +1 -0
- convexpi_lab-0.1.0/tests/integration/test_grader_worker.py +247 -0
- convexpi_lab-0.1.0/tests/lab/__init__.py +0 -0
- convexpi_lab-0.1.0/tests/lab/test_anomalies.py +172 -0
- convexpi_lab-0.1.0/tests/lab/test_backtest.py +268 -0
- convexpi_lab-0.1.0/tests/lab/test_grader.py +193 -0
- convexpi_lab-0.1.0/tests/lab/test_real_data.py +312 -0
- convexpi_lab-0.1.0/tests/lab/test_strategies.py +276 -0
- convexpi_lab-0.1.0/tests/lab/test_synth.py +170 -0
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
name: Refresh anomaly stats
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
schedule:
|
|
5
|
+
- cron: "0 4 1 * *" # 1st of each month at 04:00 UTC
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
refresh:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
permissions:
|
|
12
|
+
contents: write
|
|
13
|
+
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
with:
|
|
17
|
+
repository: convexpi/platform
|
|
18
|
+
token: ${{ secrets.PLATFORM_REPO_TOKEN }}
|
|
19
|
+
path: platform
|
|
20
|
+
|
|
21
|
+
- uses: actions/checkout@v4
|
|
22
|
+
with:
|
|
23
|
+
path: lab
|
|
24
|
+
|
|
25
|
+
- uses: actions/setup-python@v5
|
|
26
|
+
with:
|
|
27
|
+
python-version: "3.12"
|
|
28
|
+
cache: pip
|
|
29
|
+
|
|
30
|
+
- name: Install convexpi-lab
|
|
31
|
+
working-directory: lab
|
|
32
|
+
run: pip install -e ".[dev]" --quiet
|
|
33
|
+
|
|
34
|
+
- name: Compute anomaly stats
|
|
35
|
+
working-directory: lab
|
|
36
|
+
run: python deploy/compute_anomaly_stats.py --output ../platform/web/public/anomaly-stats.json
|
|
37
|
+
|
|
38
|
+
- name: Commit to platform repo if changed
|
|
39
|
+
working-directory: platform
|
|
40
|
+
run: |
|
|
41
|
+
git config user.name "github-actions[bot]"
|
|
42
|
+
git config user.email "github-actions[bot]@users.noreply.github.com"
|
|
43
|
+
git add web/public/anomaly-stats.json
|
|
44
|
+
if git diff --cached --quiet; then
|
|
45
|
+
echo "No changes to anomaly stats"
|
|
46
|
+
else
|
|
47
|
+
git commit -m "chore: refresh anomaly stats $(date -u +%Y-%m)"
|
|
48
|
+
git push
|
|
49
|
+
fi
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
name: Python ${{ matrix.python-version }}
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
strategy:
|
|
14
|
+
fail-fast: false
|
|
15
|
+
matrix:
|
|
16
|
+
python-version: ["3.10", "3.11", "3.12"]
|
|
17
|
+
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
|
|
21
|
+
- uses: actions/setup-python@v5
|
|
22
|
+
with:
|
|
23
|
+
python-version: ${{ matrix.python-version }}
|
|
24
|
+
cache: pip
|
|
25
|
+
|
|
26
|
+
- name: Install package and dev dependencies
|
|
27
|
+
run: pip install -e ".[dev]"
|
|
28
|
+
|
|
29
|
+
- name: Run tests
|
|
30
|
+
run: pytest --tb=short -q
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
name: Forward paper-trading
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
schedule:
|
|
5
|
+
- cron: '0 6 * * *' # 06:00 UTC daily
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
forward-run:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
timeout-minutes: 60
|
|
12
|
+
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
|
|
16
|
+
- uses: actions/setup-python@v5
|
|
17
|
+
with:
|
|
18
|
+
python-version: '3.12'
|
|
19
|
+
|
|
20
|
+
- name: Install convexpi-lab
|
|
21
|
+
run: pip install -e ".[deploy]"
|
|
22
|
+
|
|
23
|
+
- name: Run forward evaluator
|
|
24
|
+
env:
|
|
25
|
+
SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
|
|
26
|
+
SUPABASE_SERVICE_KEY: ${{ secrets.SUPABASE_SERVICE_KEY }}
|
|
27
|
+
MARKET_SEED_BASE: ${{ secrets.MARKET_SEED_BASE }}
|
|
28
|
+
DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}
|
|
29
|
+
FORWARD_WINDOW_DAYS: "60"
|
|
30
|
+
FORWARD_N_STOCKS: "100"
|
|
31
|
+
run: python deploy/forward_runner.py
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- 'v*'
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
build-and-publish:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
environment: pypi
|
|
12
|
+
permissions:
|
|
13
|
+
id-token: write
|
|
14
|
+
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
|
|
18
|
+
- uses: actions/setup-python@v5
|
|
19
|
+
with:
|
|
20
|
+
python-version: '3.12'
|
|
21
|
+
|
|
22
|
+
- name: Install build tools
|
|
23
|
+
run: pip install hatch
|
|
24
|
+
|
|
25
|
+
- name: Build package
|
|
26
|
+
run: hatch build
|
|
27
|
+
|
|
28
|
+
- name: Publish to PyPI
|
|
29
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Shane Conway
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: convexpi-lab
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: ConvexPi Lab — synthetic equity panel, backtester, and anti-overfitting grader
|
|
5
|
+
Project-URL: Homepage, https://convexpi.ai
|
|
6
|
+
Project-URL: Repository, https://github.com/convexpi/lab
|
|
7
|
+
Project-URL: Documentation, https://convexpi.ai/docs
|
|
8
|
+
Project-URL: Bug Tracker, https://github.com/convexpi/lab/issues
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: alpha,backtesting,education,factor-model,finance,quant
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Education
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Topic :: Office/Business :: Financial
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Requires-Dist: numpy>=1.24
|
|
20
|
+
Requires-Dist: pandas>=2.0
|
|
21
|
+
Requires-Dist: scipy>=1.10
|
|
22
|
+
Provides-Extra: deploy
|
|
23
|
+
Requires-Dist: sentry-sdk>=2.0; extra == 'deploy'
|
|
24
|
+
Requires-Dist: supabase>=2.0; extra == 'deploy'
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: hatch; extra == 'dev'
|
|
27
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
28
|
+
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
29
|
+
Provides-Extra: real-data
|
|
30
|
+
Requires-Dist: pandas-datareader>=0.10; extra == 'real-data'
|
|
31
|
+
Requires-Dist: yfinance>=0.2; extra == 'real-data'
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
# convexpi-lab
|
|
35
|
+
|
|
36
|
+
Synthetic equity panel generator, walk-forward backtester, and anti-overfitting grader for quantitative finance education and research.
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install convexpi-lab
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Part of the [ConvexPi](https://convexpi.ai) platform. See also [convexpi-arena](https://github.com/convexpi/arena) for the live exchange simulator.
|
|
43
|
+
|
|
44
|
+
## Quick start
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from convexpi.lab import SyntheticMarket, Backtest, LongShortRank
|
|
48
|
+
|
|
49
|
+
market = SyntheticMarket(n_stocks=50, n_days=756, seed=42)
|
|
50
|
+
result = Backtest(market).run(LongShortRank(feature='mom_1m'))
|
|
51
|
+
print(f"OOS Sharpe: {result.oos_sharpe:.3f}")
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Graded submission
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from convexpi.lab import Strategy, Grader
|
|
58
|
+
import numpy as np
|
|
59
|
+
|
|
60
|
+
class MyStrategy(Strategy):
|
|
61
|
+
def on_day(self, day, features, prices, portfolio):
|
|
62
|
+
sig = features['mom_1m']
|
|
63
|
+
total = np.abs(sig).sum()
|
|
64
|
+
return sig / total if total > 0 else np.zeros(len(prices))
|
|
65
|
+
|
|
66
|
+
report = Grader().grade(MyStrategy)
|
|
67
|
+
print(f"IS Sharpe: {report.is_sharpe:.3f} OOS Sharpe: {report.oos_sharpe:.3f}")
|
|
68
|
+
print(f"Overfitting ratio: {report.overfitting_ratio:.2%}")
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Features
|
|
72
|
+
|
|
73
|
+
- Synthetic equity panel with planted alpha signals of known strength
|
|
74
|
+
- Walk-forward backtester with transaction costs and turnover limits
|
|
75
|
+
- Hidden-holdout grader — OOS data never seen during development
|
|
76
|
+
- Alpha discovery detection — did you find the planted signal or fit noise?
|
|
77
|
+
- 19 canonical strategy implementations (momentum, value, quality, size, risk-based)
|
|
78
|
+
- Real-data mode: Ken French factors, FRED macro, yfinance prices (optional)
|
|
79
|
+
- Anomaly graveyard: pre/post-publication Sharpe decay for 6 canonical factors
|
|
80
|
+
- Forward paper-trading scorer (nightly, via GitHub Actions)
|
|
81
|
+
|
|
82
|
+
## Optional dependencies
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
pip install "convexpi-lab[real-data]" # yfinance + pandas-datareader
|
|
86
|
+
pip install "convexpi-lab[deploy]" # supabase + sentry (grader worker)
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## License
|
|
90
|
+
|
|
91
|
+
MIT © Shane Conway
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# convexpi-lab
|
|
2
|
+
|
|
3
|
+
Synthetic equity panel generator, walk-forward backtester, and anti-overfitting grader for quantitative finance education and research.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install convexpi-lab
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
Part of the [ConvexPi](https://convexpi.ai) platform. See also [convexpi-arena](https://github.com/convexpi/arena) for the live exchange simulator.
|
|
10
|
+
|
|
11
|
+
## Quick start
|
|
12
|
+
|
|
13
|
+
```python
|
|
14
|
+
from convexpi.lab import SyntheticMarket, Backtest, LongShortRank
|
|
15
|
+
|
|
16
|
+
market = SyntheticMarket(n_stocks=50, n_days=756, seed=42)
|
|
17
|
+
result = Backtest(market).run(LongShortRank(feature='mom_1m'))
|
|
18
|
+
print(f"OOS Sharpe: {result.oos_sharpe:.3f}")
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
## Graded submission
|
|
22
|
+
|
|
23
|
+
```python
|
|
24
|
+
from convexpi.lab import Strategy, Grader
|
|
25
|
+
import numpy as np
|
|
26
|
+
|
|
27
|
+
class MyStrategy(Strategy):
|
|
28
|
+
def on_day(self, day, features, prices, portfolio):
|
|
29
|
+
sig = features['mom_1m']
|
|
30
|
+
total = np.abs(sig).sum()
|
|
31
|
+
return sig / total if total > 0 else np.zeros(len(prices))
|
|
32
|
+
|
|
33
|
+
report = Grader().grade(MyStrategy)
|
|
34
|
+
print(f"IS Sharpe: {report.is_sharpe:.3f} OOS Sharpe: {report.oos_sharpe:.3f}")
|
|
35
|
+
print(f"Overfitting ratio: {report.overfitting_ratio:.2%}")
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Features
|
|
39
|
+
|
|
40
|
+
- Synthetic equity panel with planted alpha signals of known strength
|
|
41
|
+
- Walk-forward backtester with transaction costs and turnover limits
|
|
42
|
+
- Hidden-holdout grader — OOS data never seen during development
|
|
43
|
+
- Alpha discovery detection — did you find the planted signal or fit noise?
|
|
44
|
+
- 19 canonical strategy implementations (momentum, value, quality, size, risk-based)
|
|
45
|
+
- Real-data mode: Ken French factors, FRED macro, yfinance prices (optional)
|
|
46
|
+
- Anomaly graveyard: pre/post-publication Sharpe decay for 6 canonical factors
|
|
47
|
+
- Forward paper-trading scorer (nightly, via GitHub Actions)
|
|
48
|
+
|
|
49
|
+
## Optional dependencies
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install "convexpi-lab[real-data]" # yfinance + pandas-datareader
|
|
53
|
+
pip install "convexpi-lab[deploy]" # supabase + sentry (grader worker)
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## License
|
|
57
|
+
|
|
58
|
+
MIT © Shane Conway
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
FROM python:3.12-slim
|
|
2
|
+
|
|
3
|
+
WORKDIR /app
|
|
4
|
+
COPY pyproject.toml .
|
|
5
|
+
COPY src/ src/
|
|
6
|
+
COPY deploy/grader_worker.py deploy/grader_worker.py
|
|
7
|
+
RUN pip install --no-cache-dir -e .
|
|
8
|
+
|
|
9
|
+
# Optional: install Docker CLI so the grader can launch sandboxed subcontainers.
|
|
10
|
+
# Set GRADER_DOCKER_IMAGE=<this image tag> in Railway env vars to enable Docker-in-Docker mode.
|
|
11
|
+
# Without it the grader falls back to a plain subprocess (less secure but functional).
|
|
12
|
+
# RUN apt-get update && apt-get install -y docker.io && rm -rf /var/lib/apt/lists/*
|
|
13
|
+
|
|
14
|
+
# Required env vars (set in Railway):
|
|
15
|
+
# SUPABASE_URL, SUPABASE_SERVICE_KEY, MARKET_SEED
|
|
16
|
+
# Optional:
|
|
17
|
+
# MARKET_N_STOCKS, MARKET_N_DAYS, POLL_SECONDS, GRADE_TIMEOUT
|
|
18
|
+
# GRADER_DOCKER_IMAGE — set to enable Docker sandbox isolation
|
|
19
|
+
# SENTRY_DSN — set to enable Sentry error monitoring
|
|
20
|
+
|
|
21
|
+
CMD ["python", "deploy/grader_worker.py"]
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
compute_anomaly_stats.py — Fetch French factor data and write anomaly-stats.json.
|
|
4
|
+
|
|
5
|
+
Run this to refresh web/public/anomaly-stats.json. The file is committed to
|
|
6
|
+
the repo and read by the /anomalies web page at build time. GitHub Actions
|
|
7
|
+
runs this monthly so the OOS sample grows automatically.
|
|
8
|
+
|
|
9
|
+
Usage:
|
|
10
|
+
python deploy/compute_anomaly_stats.py
|
|
11
|
+
python deploy/compute_anomaly_stats.py --no-monthly # skip sparkline data
|
|
12
|
+
python deploy/compute_anomaly_stats.py --out path/to/anomaly-stats.json
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import json
|
|
17
|
+
import sys
|
|
18
|
+
from datetime import datetime, timezone
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
|
22
|
+
|
|
23
|
+
from convexpi.lab.anomalies import compute_all
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def main():
|
|
27
|
+
p = argparse.ArgumentParser(description="Compute anomaly stats and write JSON")
|
|
28
|
+
p.add_argument(
|
|
29
|
+
"--out",
|
|
30
|
+
default=str(Path(__file__).parent.parent / "web" / "public" / "anomaly-stats.json"),
|
|
31
|
+
)
|
|
32
|
+
p.add_argument("--no-monthly", action="store_true",
|
|
33
|
+
help="Skip monthly sparkline data (faster)")
|
|
34
|
+
args = p.parse_args()
|
|
35
|
+
|
|
36
|
+
print("Fetching French factor data…")
|
|
37
|
+
stats = compute_all(include_monthly=not args.no_monthly)
|
|
38
|
+
|
|
39
|
+
payload = {
|
|
40
|
+
"updated_at": datetime.now(tz=timezone.utc).isoformat(),
|
|
41
|
+
"source": "Kenneth French Data Library (mba.tuck.dartmouth.edu)",
|
|
42
|
+
"anomalies": stats,
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
out = Path(args.out)
|
|
46
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
47
|
+
out.write_text(json.dumps(payload, indent=2))
|
|
48
|
+
print(f"Written {len(stats)} anomalies to {out}")
|
|
49
|
+
|
|
50
|
+
for s in stats:
|
|
51
|
+
decay_str = f"{s['decay_pct']:+.1f}%" if s['decay_pct'] != 0 else "N/A"
|
|
52
|
+
print(f" {s['name']:<22} IS={s['is_sharpe']:>6.3f} "
|
|
53
|
+
f"OOS={s['oos_sharpe']:>6.3f} decay={decay_str} [{s['status']}]")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
if __name__ == "__main__":
|
|
57
|
+
main()
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
"""
|
|
2
|
+
forward_runner.py — Nightly forward paper-trading evaluator.
|
|
3
|
+
|
|
4
|
+
For each completed submission, runs the strategy on a fresh synthetic market
|
|
5
|
+
window (seeded by today's date) and upserts one row into forward_scores.
|
|
6
|
+
The leaderboard then shows rolling forward Sharpe alongside backtest Sharpe.
|
|
7
|
+
|
|
8
|
+
Run once per day (cron, Railway cron service, or GitHub Actions schedule):
|
|
9
|
+
python deploy/forward_runner.py
|
|
10
|
+
|
|
11
|
+
Required env vars (same as grader_worker):
|
|
12
|
+
SUPABASE_URL — your project URL
|
|
13
|
+
SUPABASE_SERVICE_KEY — service role key (bypasses RLS)
|
|
14
|
+
MARKET_SEED_BASE — integer base seed (default 1000); daily seed = base + days_since_epoch
|
|
15
|
+
|
|
16
|
+
Optional:
|
|
17
|
+
FORWARD_WINDOW_DAYS — trading days per evaluation window (default 60)
|
|
18
|
+
FORWARD_N_STOCKS — stocks in each daily market (default 100; smaller = faster)
|
|
19
|
+
DISCORD_WEBHOOK_URL — post a daily summary embed
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import json
|
|
25
|
+
import logging
|
|
26
|
+
import os
|
|
27
|
+
import subprocess
|
|
28
|
+
import sys
|
|
29
|
+
import tempfile
|
|
30
|
+
import textwrap
|
|
31
|
+
import time
|
|
32
|
+
import traceback
|
|
33
|
+
import urllib.request
|
|
34
|
+
import urllib.error
|
|
35
|
+
from datetime import date, timedelta
|
|
36
|
+
from pathlib import Path
|
|
37
|
+
|
|
38
|
+
logging.basicConfig(
|
|
39
|
+
level=logging.INFO,
|
|
40
|
+
format="%(asctime)s %(levelname)s %(message)s",
|
|
41
|
+
datefmt="%Y-%m-%dT%H:%M:%SZ",
|
|
42
|
+
stream=sys.stdout,
|
|
43
|
+
)
|
|
44
|
+
log = logging.getLogger("forward_runner")
|
|
45
|
+
|
|
46
|
+
SUPABASE_URL = os.environ["SUPABASE_URL"]
|
|
47
|
+
SUPABASE_KEY = os.environ["SUPABASE_SERVICE_KEY"]
|
|
48
|
+
SEED_BASE = int(os.environ.get("MARKET_SEED_BASE", "1000"))
|
|
49
|
+
WINDOW_DAYS = int(os.environ.get("FORWARD_WINDOW_DAYS", "60"))
|
|
50
|
+
N_STOCKS = int(os.environ.get("FORWARD_N_STOCKS", "100"))
|
|
51
|
+
TIMEOUT_SECS = int(os.environ.get("GRADE_TIMEOUT", "120"))
|
|
52
|
+
DISCORD_WEBHOOK = os.environ.get("DISCORD_WEBHOOK_URL", "")
|
|
53
|
+
|
|
54
|
+
PKG_PATH = str(Path(__file__).parent.parent / "src")
|
|
55
|
+
|
|
56
|
+
# Seed is deterministic per calendar date so re-runs are idempotent
|
|
57
|
+
EPOCH = date(2020, 1, 1)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def date_seed(run_date: date) -> int:
|
|
61
|
+
return SEED_BASE + (run_date - EPOCH).days
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
# ---------------------------------------------------------------------------
|
|
65
|
+
# Supabase helpers
|
|
66
|
+
# ---------------------------------------------------------------------------
|
|
67
|
+
|
|
68
|
+
def _headers() -> dict:
|
|
69
|
+
return {
|
|
70
|
+
"apikey": SUPABASE_KEY,
|
|
71
|
+
"Authorization": f"Bearer {SUPABASE_KEY}",
|
|
72
|
+
"Content-Type": "application/json",
|
|
73
|
+
"Prefer": "return=representation",
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _req(method: str, path: str, body: dict | None = None):
|
|
78
|
+
url = f"{SUPABASE_URL}/rest/v1{path}"
|
|
79
|
+
data = json.dumps(body).encode() if body else None
|
|
80
|
+
req = urllib.request.Request(url, data=data, headers=_headers(), method=method)
|
|
81
|
+
try:
|
|
82
|
+
with urllib.request.urlopen(req, timeout=15) as resp:
|
|
83
|
+
return json.loads(resp.read())
|
|
84
|
+
except urllib.error.HTTPError as e:
|
|
85
|
+
log.error("supabase %s %s → %s: %s", method, path, e.code, e.read().decode()[:200])
|
|
86
|
+
return None
|
|
87
|
+
except Exception as exc:
|
|
88
|
+
log.error("supabase %s %s → %s", method, path, exc)
|
|
89
|
+
return None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def fetch_completed_submissions() -> list[dict]:
|
|
93
|
+
result = _req("GET", "/submissions?status=eq.completed&select=id,cohort_id,code&limit=500")
|
|
94
|
+
return result or []
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def already_scored(submission_id: str, run_date: date) -> bool:
|
|
98
|
+
result = _req("GET", f"/forward_scores?submission_id=eq.{submission_id}&run_date=eq.{run_date}&select=id")
|
|
99
|
+
return bool(result)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def upsert_score(submission_id: str, run_date: date, seed: int, metrics: dict) -> None:
|
|
103
|
+
_req("POST", "/forward_scores", {
|
|
104
|
+
"submission_id": submission_id,
|
|
105
|
+
"run_date": str(run_date),
|
|
106
|
+
"forward_sharpe": metrics.get("oos_sharpe"),
|
|
107
|
+
"forward_return": metrics.get("oos_annual_return"),
|
|
108
|
+
"forward_max_dd": metrics.get("oos_max_dd"),
|
|
109
|
+
"market_seed": seed,
|
|
110
|
+
"window_days": WINDOW_DAYS,
|
|
111
|
+
})
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# ---------------------------------------------------------------------------
|
|
115
|
+
# Runner template (reuses grader runner pattern)
|
|
116
|
+
# ---------------------------------------------------------------------------
|
|
117
|
+
|
|
118
|
+
RUNNER_TEMPLATE = textwrap.dedent("""\
|
|
119
|
+
import sys, json
|
|
120
|
+
sys.path.insert(0, {pkg_path!r})
|
|
121
|
+
|
|
122
|
+
from convexpi.lab.synth import SyntheticMarket
|
|
123
|
+
from convexpi.lab.grader import Grader
|
|
124
|
+
|
|
125
|
+
{user_code}
|
|
126
|
+
|
|
127
|
+
market = SyntheticMarket(
|
|
128
|
+
n_stocks={n_stocks},
|
|
129
|
+
n_days={n_days},
|
|
130
|
+
seed={seed},
|
|
131
|
+
)
|
|
132
|
+
report = Grader(market).evaluate(MyStrategy())
|
|
133
|
+
result = {{
|
|
134
|
+
"oos_sharpe": report.oos_sharpe,
|
|
135
|
+
"oos_annual_return": report.oos_annual_return,
|
|
136
|
+
"oos_max_dd": report.oos_max_dd,
|
|
137
|
+
}}
|
|
138
|
+
print("__RESULT__:" + json.dumps(result))
|
|
139
|
+
""")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def evaluate_submission(sub_id: str, code: str, seed: int) -> dict | None:
|
|
143
|
+
runner = RUNNER_TEMPLATE.format(
|
|
144
|
+
pkg_path=PKG_PATH,
|
|
145
|
+
user_code=code,
|
|
146
|
+
n_stocks=N_STOCKS,
|
|
147
|
+
n_days=WINDOW_DAYS + 20, # small buffer for IS warmup
|
|
148
|
+
seed=seed,
|
|
149
|
+
)
|
|
150
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
151
|
+
script = Path(tmpdir) / "forward_runner.py"
|
|
152
|
+
script.write_text(runner)
|
|
153
|
+
try:
|
|
154
|
+
result = subprocess.run(
|
|
155
|
+
[sys.executable, str(script)],
|
|
156
|
+
capture_output=True, text=True,
|
|
157
|
+
timeout=TIMEOUT_SECS,
|
|
158
|
+
cwd=tmpdir,
|
|
159
|
+
)
|
|
160
|
+
except subprocess.TimeoutExpired:
|
|
161
|
+
log.warning("submission=%s forward eval timed out", sub_id[:8])
|
|
162
|
+
return None
|
|
163
|
+
|
|
164
|
+
if result.returncode != 0 or "__RESULT__:" not in result.stdout:
|
|
165
|
+
log.warning("submission=%s forward eval error: %s", sub_id[:8], result.stderr[:200])
|
|
166
|
+
return None
|
|
167
|
+
|
|
168
|
+
try:
|
|
169
|
+
json_str = result.stdout.split("__RESULT__:")[1].strip().split("\n")[0]
|
|
170
|
+
return json.loads(json_str)
|
|
171
|
+
except Exception:
|
|
172
|
+
log.error("submission=%s could not parse forward output", sub_id[:8])
|
|
173
|
+
return None
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
# ---------------------------------------------------------------------------
|
|
177
|
+
# Discord summary
|
|
178
|
+
# ---------------------------------------------------------------------------
|
|
179
|
+
|
|
180
|
+
def _discord_post(payload: dict) -> None:
|
|
181
|
+
if not DISCORD_WEBHOOK:
|
|
182
|
+
return
|
|
183
|
+
data = json.dumps(payload).encode()
|
|
184
|
+
req = urllib.request.Request(
|
|
185
|
+
DISCORD_WEBHOOK, data=data,
|
|
186
|
+
headers={"Content-Type": "application/json"}, method="POST",
|
|
187
|
+
)
|
|
188
|
+
try:
|
|
189
|
+
with urllib.request.urlopen(req, timeout=5):
|
|
190
|
+
pass
|
|
191
|
+
except Exception as exc:
|
|
192
|
+
log.warning("discord webhook failed: %s", exc)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def post_daily_summary(run_date: date, n_scored: int, n_skipped: int, n_failed: int,
|
|
196
|
+
best: dict | None) -> None:
|
|
197
|
+
lines = [
|
|
198
|
+
f"Scored **{n_scored}** submissions skipped {n_skipped} failed {n_failed}",
|
|
199
|
+
]
|
|
200
|
+
if best:
|
|
201
|
+
lines.append(
|
|
202
|
+
f"Best forward Sharpe: **{best['sharpe']:.3f}** (submission `{best['sub_id'][:8]}`)"
|
|
203
|
+
)
|
|
204
|
+
_discord_post({
|
|
205
|
+
"embeds": [{
|
|
206
|
+
"title": f"Forward paper-trading — {run_date}",
|
|
207
|
+
"color": 0x5865F2,
|
|
208
|
+
"description": "\n".join(lines),
|
|
209
|
+
"footer": {"text": "ConvexPi forward runner"},
|
|
210
|
+
}]
|
|
211
|
+
})
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
# ---------------------------------------------------------------------------
|
|
215
|
+
# Main
|
|
216
|
+
# ---------------------------------------------------------------------------
|
|
217
|
+
|
|
218
|
+
def main() -> None:
|
|
219
|
+
run_date = date.today()
|
|
220
|
+
seed = date_seed(run_date)
|
|
221
|
+
log.info("forward runner started date=%s seed=%d window=%d stocks=%d",
|
|
222
|
+
run_date, seed, WINDOW_DAYS, N_STOCKS)
|
|
223
|
+
|
|
224
|
+
submissions = fetch_completed_submissions()
|
|
225
|
+
log.info("found %d completed submissions", len(submissions))
|
|
226
|
+
|
|
227
|
+
n_scored = n_skipped = n_failed = 0
|
|
228
|
+
best: dict | None = None
|
|
229
|
+
|
|
230
|
+
for sub in submissions:
|
|
231
|
+
sub_id = sub["id"]
|
|
232
|
+
code = sub.get("code", "")
|
|
233
|
+
|
|
234
|
+
if already_scored(sub_id, run_date):
|
|
235
|
+
log.info("submission=%s already scored for %s — skipping", sub_id[:8], run_date)
|
|
236
|
+
n_skipped += 1
|
|
237
|
+
continue
|
|
238
|
+
|
|
239
|
+
t0 = time.monotonic()
|
|
240
|
+
metrics = evaluate_submission(sub_id, code, seed)
|
|
241
|
+
elapsed = time.monotonic() - t0
|
|
242
|
+
|
|
243
|
+
if metrics is None:
|
|
244
|
+
n_failed += 1
|
|
245
|
+
continue
|
|
246
|
+
|
|
247
|
+
upsert_score(sub_id, run_date, seed, metrics)
|
|
248
|
+
sharpe = metrics.get("oos_sharpe") or 0.0
|
|
249
|
+
log.info("submission=%s forward_sharpe=%.3f elapsed=%.1fs", sub_id[:8], sharpe, elapsed)
|
|
250
|
+
n_scored += 1
|
|
251
|
+
|
|
252
|
+
if best is None or sharpe > best["sharpe"]:
|
|
253
|
+
best = {"sub_id": sub_id, "sharpe": sharpe}
|
|
254
|
+
|
|
255
|
+
log.info("done scored=%d skipped=%d failed=%d", n_scored, n_skipped, n_failed)
|
|
256
|
+
post_daily_summary(run_date, n_scored, n_skipped, n_failed, best)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
if __name__ == "__main__":
|
|
260
|
+
try:
|
|
261
|
+
main()
|
|
262
|
+
except Exception:
|
|
263
|
+
log.error("forward runner crashed:\n%s", traceback.format_exc())
|
|
264
|
+
sys.exit(1)
|