precog-mcp 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- precog_mcp-1.0.0/.gitignore +25 -0
- precog_mcp-1.0.0/PKG-INFO +59 -0
- precog_mcp-1.0.0/README.md +30 -0
- precog_mcp-1.0.0/pyproject.toml +47 -0
- precog_mcp-1.0.0/src/precog_mcp/__init__.py +5 -0
- precog_mcp-1.0.0/src/precog_mcp/__main__.py +42 -0
- precog_mcp-1.0.0/src/precog_mcp/adapter.py +391 -0
- precog_mcp-1.0.0/src/precog_mcp/capabilities.py +107 -0
- precog_mcp-1.0.0/src/precog_mcp/config.py +27 -0
- precog_mcp-1.0.0/src/precog_mcp/errors.py +226 -0
- precog_mcp-1.0.0/src/precog_mcp/models.py +341 -0
- precog_mcp-1.0.0/src/precog_mcp/observability.py +23 -0
- precog_mcp-1.0.0/src/precog_mcp/server.py +161 -0
- precog_mcp-1.0.0/src/precog_mcp/tracing.py +32 -0
- precog_mcp-1.0.0/tests/test_adapter.py +335 -0
- precog_mcp-1.0.0/tests/test_backtest.py +213 -0
- precog_mcp-1.0.0/tests/test_capabilities.py +159 -0
- precog_mcp-1.0.0/tests/test_contract.py +362 -0
- precog_mcp-1.0.0/tests/test_integration_semantic.py +385 -0
- precog_mcp-1.0.0/tests/test_mcp.py +470 -0
- precog_mcp-1.0.0/tests/test_models.py +281 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
.venv/
|
|
5
|
+
.uv/
|
|
6
|
+
*.egg-info/
|
|
7
|
+
.pytest_cache/
|
|
8
|
+
.mypy_cache/
|
|
9
|
+
.ruff_cache/
|
|
10
|
+
|
|
11
|
+
# Environment
|
|
12
|
+
.env
|
|
13
|
+
.env.*
|
|
14
|
+
|
|
15
|
+
# Helm packaging
|
|
16
|
+
dist/
|
|
17
|
+
|
|
18
|
+
# Node / TypeScript
|
|
19
|
+
node_modules/
|
|
20
|
+
|
|
21
|
+
# Model weights and caches — never commit these (see PREC-9 / THIRD_PARTY_NOTICES)
|
|
22
|
+
models/
|
|
23
|
+
*.safetensors
|
|
24
|
+
*.gguf
|
|
25
|
+
.cache/huggingface/
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: precog-mcp
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: MCP server exposing Precog forecasts as tools.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Albe83/precog
|
|
6
|
+
Project-URL: Repository, https://github.com/Albe83/precog
|
|
7
|
+
Project-URL: Issues, https://github.com/Albe83/precog/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/Albe83/precog/blob/main/CHANGELOG.md
|
|
9
|
+
License: MIT
|
|
10
|
+
Keywords: forecasting,mcp,time-series,timesfm
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
15
|
+
Requires-Python: >=3.12
|
|
16
|
+
Requires-Dist: httpx>=0.27
|
|
17
|
+
Requires-Dist: mcp>=2.2
|
|
18
|
+
Requires-Dist: precog-client<0.2.0,>=0.1.0
|
|
19
|
+
Requires-Dist: precog-schemas<0.2.0,>=0.1.0
|
|
20
|
+
Requires-Dist: prometheus-client>=0.20
|
|
21
|
+
Requires-Dist: pydantic-settings>=2.4
|
|
22
|
+
Requires-Dist: pydantic>=2.7
|
|
23
|
+
Requires-Dist: uvicorn[standard]>=0.30
|
|
24
|
+
Provides-Extra: otel
|
|
25
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.27; extra == 'otel'
|
|
26
|
+
Requires-Dist: opentelemetry-instrumentation-httpx>=0.48b0; extra == 'otel'
|
|
27
|
+
Requires-Dist: opentelemetry-sdk>=1.27; extra == 'otel'
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# precog-mcp
|
|
31
|
+
|
|
32
|
+
MCP server exposing Precog zero-shot forecasting to agents as the semantic
|
|
33
|
+
`forecast` and `backtest` tools plus the `precog://capabilities` resource.
|
|
34
|
+
|
|
35
|
+
It talks to the Precog **REST Execution API** over HTTP through the official
|
|
36
|
+
`precog-client` and contains **no model weights**.
|
|
37
|
+
|
|
38
|
+
## Install
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install precog-mcp
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## Run
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
# stdio transport
|
|
48
|
+
PRECOG_API_URL=http://localhost:8000 precog-mcp
|
|
49
|
+
|
|
50
|
+
# HTTP transport (endpoint: http://localhost:8765/mcp)
|
|
51
|
+
PRECOG_MCP_TRANSPORT=http PRECOG_MCP_PORT=8765 \
|
|
52
|
+
PRECOG_API_URL=http://localhost:8000 precog-mcp
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
The API must be reachable; see `precog-api` or the container/Helm deployment.
|
|
56
|
+
|
|
57
|
+
## License
|
|
58
|
+
|
|
59
|
+
MIT.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# precog-mcp
|
|
2
|
+
|
|
3
|
+
MCP server exposing Precog zero-shot forecasting to agents as the semantic
|
|
4
|
+
`forecast` and `backtest` tools plus the `precog://capabilities` resource.
|
|
5
|
+
|
|
6
|
+
It talks to the Precog **REST Execution API** over HTTP through the official
|
|
7
|
+
`precog-client` and contains **no model weights**.
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install precog-mcp
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Run
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
# stdio transport
|
|
19
|
+
PRECOG_API_URL=http://localhost:8000 precog-mcp
|
|
20
|
+
|
|
21
|
+
# HTTP transport (endpoint: http://localhost:8765/mcp)
|
|
22
|
+
PRECOG_MCP_TRANSPORT=http PRECOG_MCP_PORT=8765 \
|
|
23
|
+
PRECOG_API_URL=http://localhost:8000 precog-mcp
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
The API must be reachable; see `precog-api` or the container/Helm deployment.
|
|
27
|
+
|
|
28
|
+
## License
|
|
29
|
+
|
|
30
|
+
MIT.
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "precog-mcp"
|
|
3
|
+
version = "1.0.0"
|
|
4
|
+
description = "MCP server exposing Precog forecasts as tools."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.12"
|
|
7
|
+
license = { text = "MIT" }
|
|
8
|
+
keywords = ["forecasting", "time-series", "mcp", "timesfm"]
|
|
9
|
+
classifiers = [
|
|
10
|
+
"Development Status :: 4 - Beta",
|
|
11
|
+
"Programming Language :: Python :: 3",
|
|
12
|
+
"Programming Language :: Python :: 3.12",
|
|
13
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
14
|
+
]
|
|
15
|
+
dependencies = [
|
|
16
|
+
"mcp>=2.2",
|
|
17
|
+
"httpx>=0.27",
|
|
18
|
+
"uvicorn[standard]>=0.30",
|
|
19
|
+
"prometheus-client>=0.20",
|
|
20
|
+
"pydantic>=2.7",
|
|
21
|
+
"pydantic-settings>=2.4",
|
|
22
|
+
"precog-schemas>=0.1.0,<0.2.0",
|
|
23
|
+
"precog-client>=0.1.0,<0.2.0",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[project.optional-dependencies]
|
|
27
|
+
otel = [
|
|
28
|
+
"opentelemetry-sdk>=1.27",
|
|
29
|
+
"opentelemetry-exporter-otlp-proto-http>=1.27",
|
|
30
|
+
"opentelemetry-instrumentation-httpx>=0.48b0",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Homepage = "https://github.com/Albe83/precog"
|
|
35
|
+
Repository = "https://github.com/Albe83/precog"
|
|
36
|
+
Issues = "https://github.com/Albe83/precog/issues"
|
|
37
|
+
Changelog = "https://github.com/Albe83/precog/blob/main/CHANGELOG.md"
|
|
38
|
+
|
|
39
|
+
[project.scripts]
|
|
40
|
+
precog-mcp = "precog_mcp.__main__:main"
|
|
41
|
+
|
|
42
|
+
[build-system]
|
|
43
|
+
requires = ["hatchling"]
|
|
44
|
+
build-backend = "hatchling.build"
|
|
45
|
+
|
|
46
|
+
[tool.hatch.build.targets.wheel]
|
|
47
|
+
packages = ["src/precog_mcp"]
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Entry point for the Precog MCP server."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from precog_mcp.config import Settings
|
|
6
|
+
from precog_mcp.server import create_server
|
|
7
|
+
|
|
8
|
+
DEFAULT_ALLOWED_HOSTS = ["127.0.0.1:*", "localhost:*", "[::1]:*"]
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def transport_security(allowed_hosts: str):
|
|
12
|
+
"""Build the HTTP transport security settings from ``PRECOG_MCP_ALLOWED_HOSTS``.
|
|
13
|
+
|
|
14
|
+
- ``"*"`` disables DNS-rebinding protection (use only where the listener is
|
|
15
|
+
reachable exclusively through a trusted gateway).
|
|
16
|
+
- a comma-separated list adds hosts to the localhost defaults.
|
|
17
|
+
- empty keeps the secure localhost-only default.
|
|
18
|
+
"""
|
|
19
|
+
from mcp.server.transport_security import TransportSecuritySettings
|
|
20
|
+
|
|
21
|
+
entries = [host.strip() for host in allowed_hosts.split(",") if host.strip()]
|
|
22
|
+
if "*" in entries:
|
|
23
|
+
return TransportSecuritySettings(enable_dns_rebinding_protection=False)
|
|
24
|
+
return TransportSecuritySettings(allowed_hosts=[*DEFAULT_ALLOWED_HOSTS, *entries])
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def main() -> None:
|
|
28
|
+
settings = Settings()
|
|
29
|
+
server = create_server(settings)
|
|
30
|
+
if settings.mcp_transport == "http":
|
|
31
|
+
import uvicorn
|
|
32
|
+
|
|
33
|
+
app = server.streamable_http_app(
|
|
34
|
+
transport_security=transport_security(settings.mcp_allowed_hosts)
|
|
35
|
+
)
|
|
36
|
+
uvicorn.run(app, host=settings.mcp_host, port=settings.mcp_port)
|
|
37
|
+
else:
|
|
38
|
+
server.run("stdio")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
if __name__ == "__main__":
|
|
42
|
+
main()
|
|
@@ -0,0 +1,391 @@
|
|
|
1
|
+
"""Anti-corruption adapter between the MCP contract and the Precog REST API.
|
|
2
|
+
|
|
3
|
+
The MCP server is an HTTP client of ``POST /v1/forecast`` through the official
|
|
4
|
+
:class:`~precog_client.AsyncPrecogClient`. This module owns the translation so
|
|
5
|
+
the public tool surface never leaks REST/backend DTOs or TimesFM-specific
|
|
6
|
+
execution controls, and it owns the sanitization of upstream failures.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import math
|
|
12
|
+
from collections.abc import Mapping
|
|
13
|
+
from enum import StrEnum
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from pydantic import ValidationError
|
|
17
|
+
|
|
18
|
+
from precog_client import (
|
|
19
|
+
AsyncPrecogClient,
|
|
20
|
+
PrecogAPIError,
|
|
21
|
+
PrecogConnectionError,
|
|
22
|
+
PrecogError,
|
|
23
|
+
PrecogTimeoutError,
|
|
24
|
+
)
|
|
25
|
+
from precog_mcp.models import (
|
|
26
|
+
BacktestMetrics,
|
|
27
|
+
BacktestResult,
|
|
28
|
+
BacktestToolRequest,
|
|
29
|
+
ForecastResult,
|
|
30
|
+
ForecastToolRequest,
|
|
31
|
+
HistoricalSeries,
|
|
32
|
+
IntervalCoverage,
|
|
33
|
+
KnownFutureSeries,
|
|
34
|
+
ModelProvenance,
|
|
35
|
+
TargetBacktest,
|
|
36
|
+
TargetForecast,
|
|
37
|
+
quantile_key,
|
|
38
|
+
)
|
|
39
|
+
from precog_schemas import (
|
|
40
|
+
ForecastRequest,
|
|
41
|
+
ForecastResponse,
|
|
42
|
+
)
|
|
43
|
+
from precog_schemas import (
|
|
44
|
+
HistoricalSeries as RestHistoricalSeries,
|
|
45
|
+
)
|
|
46
|
+
from precog_schemas import (
|
|
47
|
+
KnownFutureSeries as RestKnownFutureSeries,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
PROBLEM_MEDIA_TYPE = "application/problem+json"
|
|
51
|
+
UNAVAILABLE_MESSAGE = "Precog API is unavailable"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class ErrorCode(StrEnum):
|
|
55
|
+
"""Stable error codes for the MCP boundary.
|
|
56
|
+
|
|
57
|
+
The first five describe the forecast pipeline; ``INTERNAL_ERROR`` covers
|
|
58
|
+
unexpected MCP server defects that cannot be attributed to the caller.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
INVALID_REQUEST = "INVALID_REQUEST"
|
|
62
|
+
FORECAST_REJECTED = "FORECAST_REJECTED"
|
|
63
|
+
API_UNAVAILABLE = "API_UNAVAILABLE"
|
|
64
|
+
UPSTREAM_CONTRACT_ERROR = "UPSTREAM_CONTRACT_ERROR"
|
|
65
|
+
INFERENCE_FAILED = "INFERENCE_FAILED"
|
|
66
|
+
INTERNAL_ERROR = "INTERNAL_ERROR"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class ForecastAdapterError(RuntimeError):
|
|
70
|
+
"""A machine-readable failure crossing the MCP boundary."""
|
|
71
|
+
|
|
72
|
+
def __init__(
|
|
73
|
+
self,
|
|
74
|
+
code: ErrorCode,
|
|
75
|
+
message: str,
|
|
76
|
+
*,
|
|
77
|
+
details: dict[str, Any] | None = None,
|
|
78
|
+
status: int | None = None,
|
|
79
|
+
) -> None:
|
|
80
|
+
super().__init__(message)
|
|
81
|
+
self.code = code
|
|
82
|
+
self.message = message
|
|
83
|
+
self.details = details
|
|
84
|
+
self.status = status
|
|
85
|
+
|
|
86
|
+
def to_payload(self) -> dict[str, Any]:
|
|
87
|
+
"""Return the deterministic error envelope exposed to MCP consumers."""
|
|
88
|
+
payload: dict[str, Any] = {"code": self.code.value, "message": self.message}
|
|
89
|
+
if self.details:
|
|
90
|
+
payload["details"] = self.details
|
|
91
|
+
return payload
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def to_rest_request(request: ForecastToolRequest) -> ForecastRequest:
|
|
95
|
+
"""Map the semantic request to the canonical execution contract.
|
|
96
|
+
|
|
97
|
+
This is now a structural mapping: the semantic contract and the execution
|
|
98
|
+
contract share the same shape (targets, past covariates, known-future
|
|
99
|
+
history/future, explicit quantile levels).
|
|
100
|
+
"""
|
|
101
|
+
return ForecastRequest(
|
|
102
|
+
horizon=request.horizon,
|
|
103
|
+
targets=[
|
|
104
|
+
RestHistoricalSeries(id=target.id, values=list(target.values))
|
|
105
|
+
for target in request.targets
|
|
106
|
+
],
|
|
107
|
+
past_covariates=[
|
|
108
|
+
RestHistoricalSeries(id=covariate.id, values=list(covariate.values))
|
|
109
|
+
for covariate in request.past_covariates
|
|
110
|
+
],
|
|
111
|
+
known_future_covariates=[
|
|
112
|
+
RestKnownFutureSeries(
|
|
113
|
+
id=covariate.id,
|
|
114
|
+
history=list(covariate.history),
|
|
115
|
+
future=list(covariate.future),
|
|
116
|
+
)
|
|
117
|
+
for covariate in request.known_future_covariates
|
|
118
|
+
],
|
|
119
|
+
quantiles=list(request.quantiles),
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def from_rest_response(request: ForecastToolRequest, payload: Mapping[str, Any]) -> ForecastResult:
|
|
124
|
+
"""Validate an execution response and build the consumer-facing result."""
|
|
125
|
+
try:
|
|
126
|
+
response = ForecastResponse.model_validate(payload)
|
|
127
|
+
except ValidationError as exc:
|
|
128
|
+
raise ForecastAdapterError(
|
|
129
|
+
ErrorCode.UPSTREAM_CONTRACT_ERROR,
|
|
130
|
+
"Precog API returned a malformed forecast response",
|
|
131
|
+
details={"errors": exc.errors(include_url=False)},
|
|
132
|
+
) from exc
|
|
133
|
+
|
|
134
|
+
if response.horizon != request.horizon:
|
|
135
|
+
raise _contract_error(
|
|
136
|
+
"Precog API returned an unexpected horizon",
|
|
137
|
+
{"expected": request.horizon, "received": response.horizon},
|
|
138
|
+
)
|
|
139
|
+
if not response.model.id:
|
|
140
|
+
raise _contract_error("Precog API returned an empty model identifier")
|
|
141
|
+
|
|
142
|
+
requested_ids = [target.id for target in request.targets]
|
|
143
|
+
result_ids = [target.id for target in response.targets]
|
|
144
|
+
if result_ids != requested_ids:
|
|
145
|
+
raise _contract_error(
|
|
146
|
+
"Precog API returned unexpected target ids",
|
|
147
|
+
{"expected": requested_ids, "received": result_ids},
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
targets: list[TargetForecast] = []
|
|
151
|
+
for target, result in zip(request.targets, response.targets, strict=True):
|
|
152
|
+
forecast = _finite_vector(
|
|
153
|
+
result.forecast, request.horizon, label=f"forecast of '{target.id}'"
|
|
154
|
+
)
|
|
155
|
+
quantiles: dict[str, list[float]] = {}
|
|
156
|
+
if request.quantiles:
|
|
157
|
+
by_level = {round(quantile.level, 9): quantile.values for quantile in result.quantiles}
|
|
158
|
+
for level in request.quantiles:
|
|
159
|
+
values = by_level.get(round(level, 9))
|
|
160
|
+
if values is None:
|
|
161
|
+
raise _contract_error(
|
|
162
|
+
"Precog API omitted a requested quantile level",
|
|
163
|
+
{"target": target.id, "level": level},
|
|
164
|
+
)
|
|
165
|
+
quantiles[quantile_key(level)] = _finite_vector(
|
|
166
|
+
values,
|
|
167
|
+
request.horizon,
|
|
168
|
+
label=f"quantile {quantile_key(level)} of '{target.id}'",
|
|
169
|
+
)
|
|
170
|
+
targets.append(TargetForecast(id=target.id, forecast=forecast, quantiles=quantiles))
|
|
171
|
+
|
|
172
|
+
return ForecastResult(
|
|
173
|
+
horizon=request.horizon,
|
|
174
|
+
targets=targets,
|
|
175
|
+
model=ModelProvenance(id=response.model.id),
|
|
176
|
+
warnings=[],
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def map_client_error(exc: PrecogError) -> ForecastAdapterError:
|
|
181
|
+
"""Translate an SDK client failure into a typed, sanitized MCP-side error.
|
|
182
|
+
|
|
183
|
+
Error codes are stable: connectivity/timeouts and 401/403 map to
|
|
184
|
+
``API_UNAVAILABLE``; 5xx to ``INFERENCE_FAILED``; other 4xx rejections to
|
|
185
|
+
``FORECAST_REJECTED``; malformed/unexpected upstream contracts to
|
|
186
|
+
``UPSTREAM_CONTRACT_ERROR``. Messages never forward raw URLs, hosts, proxy
|
|
187
|
+
or TLS details.
|
|
188
|
+
"""
|
|
189
|
+
if isinstance(exc, (PrecogConnectionError, PrecogTimeoutError)):
|
|
190
|
+
return ForecastAdapterError(ErrorCode.API_UNAVAILABLE, UNAVAILABLE_MESSAGE)
|
|
191
|
+
if isinstance(exc, PrecogAPIError):
|
|
192
|
+
return _map_api_error(exc)
|
|
193
|
+
return ForecastAdapterError(
|
|
194
|
+
ErrorCode.UPSTREAM_CONTRACT_ERROR,
|
|
195
|
+
"Precog API returned a malformed forecast response",
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _map_api_error(exc: PrecogAPIError) -> ForecastAdapterError:
|
|
200
|
+
status = exc.status_code
|
|
201
|
+
message = _sanitized_message(exc)
|
|
202
|
+
if status in (401, 403):
|
|
203
|
+
return ForecastAdapterError(ErrorCode.API_UNAVAILABLE, message)
|
|
204
|
+
if status >= 500:
|
|
205
|
+
return ForecastAdapterError(ErrorCode.INFERENCE_FAILED, message, status=status)
|
|
206
|
+
return ForecastAdapterError(ErrorCode.FORECAST_REJECTED, message, status=status)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _sanitized_message(exc: PrecogAPIError) -> str:
|
|
210
|
+
"""Rebuild a problem message from trusted fields only, else stay generic."""
|
|
211
|
+
if (
|
|
212
|
+
exc.status_code >= 400
|
|
213
|
+
and exc.media_type == PROBLEM_MEDIA_TYPE
|
|
214
|
+
and isinstance(exc.payload, Mapping)
|
|
215
|
+
):
|
|
216
|
+
raw_title = exc.payload.get("title")
|
|
217
|
+
raw_detail = exc.payload.get("detail")
|
|
218
|
+
title = raw_title if isinstance(raw_title, str) and raw_title else None
|
|
219
|
+
detail = raw_detail if isinstance(raw_detail, str) and raw_detail else None
|
|
220
|
+
if detail is not None:
|
|
221
|
+
return f"{title or 'error'}: {detail}"
|
|
222
|
+
if title is not None:
|
|
223
|
+
return title
|
|
224
|
+
return f"Precog API error (HTTP {exc.status_code})"
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
async def execute_forecast(
|
|
228
|
+
client: AsyncPrecogClient, request: ForecastToolRequest
|
|
229
|
+
) -> ForecastResult:
|
|
230
|
+
"""Run one forecast through the official execution client."""
|
|
231
|
+
rest_request = to_rest_request(request)
|
|
232
|
+
try:
|
|
233
|
+
response = await client.forecast_request(rest_request)
|
|
234
|
+
except PrecogError as exc:
|
|
235
|
+
raise map_client_error(exc) from exc
|
|
236
|
+
return from_rest_response(request, response.model_dump(mode="json"))
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def backtest_to_forecast_request(request: BacktestToolRequest) -> ForecastToolRequest:
|
|
240
|
+
"""Split a backtest at the holdout cutoff and build the forecast problem.
|
|
241
|
+
|
|
242
|
+
The head of every series becomes the context; the tail is held out. A
|
|
243
|
+
known-future covariate's tail is forwarded as already-known future values,
|
|
244
|
+
which is the anti-leakage assumption the caller is responsible for.
|
|
245
|
+
"""
|
|
246
|
+
horizon = request.horizon
|
|
247
|
+
targets = [
|
|
248
|
+
HistoricalSeries(id=series.id, values=series.values[:-horizon])
|
|
249
|
+
for series in request.targets
|
|
250
|
+
]
|
|
251
|
+
past = [
|
|
252
|
+
HistoricalSeries(id=series.id, values=series.values[:-horizon])
|
|
253
|
+
for series in request.past_covariates
|
|
254
|
+
]
|
|
255
|
+
known = [
|
|
256
|
+
KnownFutureSeries(
|
|
257
|
+
id=series.id,
|
|
258
|
+
history=series.values[:-horizon],
|
|
259
|
+
future=series.values[-horizon:],
|
|
260
|
+
)
|
|
261
|
+
for series in request.known_future_covariates
|
|
262
|
+
]
|
|
263
|
+
return ForecastToolRequest(
|
|
264
|
+
targets=targets,
|
|
265
|
+
horizon=horizon,
|
|
266
|
+
past_covariates=past,
|
|
267
|
+
known_future_covariates=known,
|
|
268
|
+
quantiles=list(request.quantiles),
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def evaluate_backtest(request: BacktestToolRequest, result: ForecastResult) -> BacktestResult:
|
|
273
|
+
"""Compare a forecast against the held-out actuals and compute metrics."""
|
|
274
|
+
if result.horizon != request.horizon:
|
|
275
|
+
raise _contract_error(
|
|
276
|
+
"Precog API returned an unexpected horizon",
|
|
277
|
+
{"expected": request.horizon, "received": result.horizon},
|
|
278
|
+
)
|
|
279
|
+
expected_ids = [target.id for target in request.targets]
|
|
280
|
+
result_ids = [target.id for target in result.targets]
|
|
281
|
+
if result_ids != expected_ids:
|
|
282
|
+
raise _contract_error(
|
|
283
|
+
"Precog API returned unexpected target ids",
|
|
284
|
+
{"expected": expected_ids, "received": result_ids},
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
evaluated: list[TargetBacktest] = []
|
|
288
|
+
for source, predicted in zip(request.targets, result.targets, strict=True):
|
|
289
|
+
actual = list(source.values[-request.horizon :])
|
|
290
|
+
forecast = _finite_vector(
|
|
291
|
+
predicted.forecast, request.horizon, label=f"forecast of '{source.id}'"
|
|
292
|
+
)
|
|
293
|
+
evaluated.append(
|
|
294
|
+
TargetBacktest(
|
|
295
|
+
id=source.id,
|
|
296
|
+
actual=actual,
|
|
297
|
+
forecast=forecast,
|
|
298
|
+
metrics=_metrics(actual, forecast, predicted.quantiles),
|
|
299
|
+
)
|
|
300
|
+
)
|
|
301
|
+
return BacktestResult(
|
|
302
|
+
horizon=request.horizon,
|
|
303
|
+
targets=evaluated,
|
|
304
|
+
model=result.model,
|
|
305
|
+
warnings=list(result.warnings),
|
|
306
|
+
)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
async def execute_backtest(
|
|
310
|
+
client: AsyncPrecogClient, request: BacktestToolRequest
|
|
311
|
+
) -> BacktestResult:
|
|
312
|
+
"""Run one backtest through the same semantic forecast path as ``forecast``."""
|
|
313
|
+
result = await execute_forecast(client, backtest_to_forecast_request(request))
|
|
314
|
+
return evaluate_backtest(request, result)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _metrics(
|
|
318
|
+
actual: list[float],
|
|
319
|
+
forecast: list[float],
|
|
320
|
+
quantiles: Mapping[str, list[float]],
|
|
321
|
+
) -> BacktestMetrics:
|
|
322
|
+
errors = [a - f for a, f in zip(actual, forecast, strict=True)]
|
|
323
|
+
count = len(errors)
|
|
324
|
+
mae = sum(abs(error) for error in errors) / count
|
|
325
|
+
rmse = math.sqrt(sum(error * error for error in errors) / count)
|
|
326
|
+
return BacktestMetrics(
|
|
327
|
+
mae=mae,
|
|
328
|
+
rmse=rmse,
|
|
329
|
+
smape=_smape(actual, forecast),
|
|
330
|
+
coverage=_coverage(actual, quantiles),
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _smape(actual: list[float], forecast: list[float]) -> float:
|
|
335
|
+
"""Symmetric MAPE as a percentage.
|
|
336
|
+
|
|
337
|
+
``100 / n * sum(2 * |a - f| / (|a| + |f|))``. When ``|a| + |f| == 0`` the
|
|
338
|
+
term is defined as ``0`` (perfect agreement on a zero value).
|
|
339
|
+
"""
|
|
340
|
+
total = 0.0
|
|
341
|
+
for a, f in zip(actual, forecast, strict=True):
|
|
342
|
+
denominator = abs(a) + abs(f)
|
|
343
|
+
if denominator == 0.0:
|
|
344
|
+
continue
|
|
345
|
+
total += 2.0 * abs(a - f) / denominator
|
|
346
|
+
return 100.0 * total / len(actual)
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _coverage(actual: list[float], quantiles: Mapping[str, list[float]]) -> IntervalCoverage | None:
|
|
350
|
+
"""Coverage of the widest requested interval that brackets the median."""
|
|
351
|
+
levels = sorted(float(key) for key in quantiles)
|
|
352
|
+
lower_levels = [level for level in levels if level < 0.5]
|
|
353
|
+
upper_levels = [level for level in levels if level > 0.5]
|
|
354
|
+
if not lower_levels or not upper_levels:
|
|
355
|
+
return None
|
|
356
|
+
lower = min(lower_levels)
|
|
357
|
+
upper = max(upper_levels)
|
|
358
|
+
below = quantiles[quantile_key(lower)]
|
|
359
|
+
above = quantiles[quantile_key(upper)]
|
|
360
|
+
inside = sum(1 for a, low, high in zip(actual, below, above, strict=True) if low <= a <= high)
|
|
361
|
+
return IntervalCoverage(
|
|
362
|
+
lower_quantile=lower, upper_quantile=upper, percent=100.0 * inside / len(actual)
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _finite_vector(values: list[float], expected: int, *, label: str) -> list[float]:
|
|
367
|
+
if len(values) != expected:
|
|
368
|
+
raise _contract_error(
|
|
369
|
+
"Precog API returned a vector with the wrong length",
|
|
370
|
+
{"label": label, "expected": expected, "received": len(values)},
|
|
371
|
+
)
|
|
372
|
+
if not all(math.isfinite(value) for value in values):
|
|
373
|
+
raise _contract_error("Precog API returned a non-finite forecast", {"label": label})
|
|
374
|
+
return list(values)
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _contract_error(message: str, details: dict[str, Any] | None = None) -> ForecastAdapterError:
|
|
378
|
+
return ForecastAdapterError(ErrorCode.UPSTREAM_CONTRACT_ERROR, message, details=details)
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
__all__ = [
|
|
382
|
+
"ErrorCode",
|
|
383
|
+
"ForecastAdapterError",
|
|
384
|
+
"backtest_to_forecast_request",
|
|
385
|
+
"evaluate_backtest",
|
|
386
|
+
"execute_backtest",
|
|
387
|
+
"execute_forecast",
|
|
388
|
+
"from_rest_response",
|
|
389
|
+
"map_client_error",
|
|
390
|
+
"to_rest_request",
|
|
391
|
+
]
|