quantdb-sdk 0.2.4__tar.gz → 0.2.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/CHANGELOG.md +18 -0
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/MANIFEST.in +5 -5
- {quantdb_sdk-0.2.4/quantdb_sdk.egg-info → quantdb_sdk-0.2.6}/PKG-INFO +26 -1
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/README.md +26 -1
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/pyproject.toml +1 -1
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/quantdb_sdk/__init__.py +1 -1
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/quantdb_sdk/__main__.py +12 -5
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/quantdb_sdk/_utils.py +54 -2
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/quantdb_sdk/async_client.py +91 -44
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/quantdb_sdk/client.py +859 -814
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6/quantdb_sdk.egg-info}/PKG-INFO +26 -1
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/tests/test_async_client.py +6 -0
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/tests/test_client.py +24 -0
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/LICENSE +0 -0
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/quantdb_sdk/errors.py +0 -0
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/quantdb_sdk/py.typed +0 -0
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/quantdb_sdk.egg-info/SOURCES.txt +0 -0
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/quantdb_sdk.egg-info/dependency_links.txt +0 -0
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/quantdb_sdk.egg-info/entry_points.txt +0 -0
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/quantdb_sdk.egg-info/requires.txt +0 -0
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/quantdb_sdk.egg-info/top_level.txt +0 -0
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/setup.cfg +0 -0
- {quantdb_sdk-0.2.4 → quantdb_sdk-0.2.6}/tests/test_technical_indicators.py +0 -0
|
@@ -3,6 +3,24 @@
|
|
|
3
3
|
所有 notable 变更都会记录在此文件。格式基于 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.1.0/),
|
|
4
4
|
版本号遵循 [Semantic Versioning](https://semver.org/lang/zh-CN/)。
|
|
5
5
|
|
|
6
|
+
## [0.2.6] - 2026-07-27
|
|
7
|
+
|
|
8
|
+
### Fixed
|
|
9
|
+
- **适配服务端 CDN 直连下载(302 跳转)**:异步客户端(httpx 默认不跟随重定向)此前在服务端开启 302 直连后所有下载/同步接口失效,现已修复。
|
|
10
|
+
- **凭证保护**:同步客户端此前跟随 302 时会把 `X-API-Key` 透传给 CDN 域;现两个客户端统一改为「手动处理 302 + 无鉴权头裸请求直连 CDN」,凭证只发给 QuantDB 网关。
|
|
11
|
+
- 302 响应中的 COS ETag 回填至 CDN 响应,保证进程内 ETag 缓存与 If-None-Match 304 逻辑不受影响。
|
|
12
|
+
|
|
13
|
+
## [0.2.5] - 2026-07-26
|
|
14
|
+
|
|
15
|
+
### Security
|
|
16
|
+
- 下载响应文件名与 release/Manifest 相对路径均限制在指定下载目录内,防止路径穿越。
|
|
17
|
+
- 公网 API Host 强制 HTTPS;CLI 改从环境变量读取 API Key,避免密钥出现在命令历史和进程参数中。
|
|
18
|
+
- 下载增加单文件大小上限(默认 2 GiB,可通过 `QUANTDB_MAX_DOWNLOAD_BYTES` 配置)。
|
|
19
|
+
- `query_local()` / `a_query_local()` 只接受 WHERE 条件,拒绝完整 SQL、JOIN 和外部表函数。
|
|
20
|
+
|
|
21
|
+
### Documentation
|
|
22
|
+
- 更新官网 SDK 文档:并发下载建议、V1/V2 选择、安全配置与异常处理。
|
|
23
|
+
|
|
6
24
|
## [0.2.4] - 2026-07-26
|
|
7
25
|
|
|
8
26
|
### Fixed
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
include README.md
|
|
2
|
-
include LICENSE
|
|
3
|
-
include CHANGELOG.md
|
|
4
|
-
include pyproject.toml
|
|
5
|
-
recursive-include quantdb_sdk *.py *.typed
|
|
1
|
+
include README.md
|
|
2
|
+
include LICENSE
|
|
3
|
+
include CHANGELOG.md
|
|
4
|
+
include pyproject.toml
|
|
5
|
+
recursive-include quantdb_sdk *.py *.typed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: quantdb-sdk
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.6
|
|
4
4
|
Summary: QuantDB 量化数据平台官方 Python SDK
|
|
5
5
|
Author: QuantDB Team
|
|
6
6
|
License: MIT
|
|
@@ -111,6 +111,31 @@ result = client.sync_dataset("daily_forward", save_dir="D:/quantdb-data")
|
|
|
111
111
|
financial = client.sync_dataset("balance", save_dir="D:/quantdb-data")
|
|
112
112
|
```
|
|
113
113
|
|
|
114
|
+
### 加速批量下载:建议从 8 个工作线程开始
|
|
115
|
+
|
|
116
|
+
`sync_dataset()` 单次调用会按发布顺序串行下载,以保证本地 SQLite 同步状态和 release cursor 一致。
|
|
117
|
+
当需要下载多个独立标的或文件时,可由调用方并行调度;建议先使用 **8 个工作线程**,再结合网络带宽、磁盘写入能力和账户流量配额调整。
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
121
|
+
from quantdb_sdk import QuantDBClient
|
|
122
|
+
|
|
123
|
+
API_KEY = "qdb_xxx..."
|
|
124
|
+
symbols = ["600519.SH", "000001.SZ", "600036.SH"]
|
|
125
|
+
|
|
126
|
+
def download_one(symbol: str) -> str:
|
|
127
|
+
# 每个 worker 使用独立客户端,避免跨线程共享 HTTP Session。
|
|
128
|
+
with_client = QuantDBClient(api_key=API_KEY)
|
|
129
|
+
return with_client.download_file(
|
|
130
|
+
"1", "daily_forward", symbol=symbol, save_dir="D:/quantdb-data"
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
134
|
+
files = list(pool.map(download_one, symbols))
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
不要对相同 `save_dir` 并行调用多个 `sync_dataset()`:它们会共同写入 `quantdb_sync.sqlite`,可能产生锁竞争。批量同步本身仍建议一次一个数据集执行。
|
|
138
|
+
|
|
114
139
|
## 流量说明
|
|
115
140
|
|
|
116
141
|
免费注册用户获赠 100 MB 一次性体验流量;订阅用户每月含 30 GB 下载流量,超出部分按 ¥1/GB 从账户余额扣减。余额不足时下载会被拦截。
|
|
@@ -73,8 +73,33 @@ result = client.sync_dataset("daily_forward", save_dir="D:/quantdb-data")
|
|
|
73
73
|
# quantdb sync qdb_xxx balance --save-dir D:/quantdb-data
|
|
74
74
|
financial = client.sync_dataset("balance", save_dir="D:/quantdb-data")
|
|
75
75
|
```
|
|
76
|
+
|
|
77
|
+
### 加速批量下载:建议从 8 个工作线程开始
|
|
78
|
+
|
|
79
|
+
`sync_dataset()` 单次调用会按发布顺序串行下载,以保证本地 SQLite 同步状态和 release cursor 一致。
|
|
80
|
+
当需要下载多个独立标的或文件时,可由调用方并行调度;建议先使用 **8 个工作线程**,再结合网络带宽、磁盘写入能力和账户流量配额调整。
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
84
|
+
from quantdb_sdk import QuantDBClient
|
|
85
|
+
|
|
86
|
+
API_KEY = "qdb_xxx..."
|
|
87
|
+
symbols = ["600519.SH", "000001.SZ", "600036.SH"]
|
|
88
|
+
|
|
89
|
+
def download_one(symbol: str) -> str:
|
|
90
|
+
# 每个 worker 使用独立客户端,避免跨线程共享 HTTP Session。
|
|
91
|
+
with_client = QuantDBClient(api_key=API_KEY)
|
|
92
|
+
return with_client.download_file(
|
|
93
|
+
"1", "daily_forward", symbol=symbol, save_dir="D:/quantdb-data"
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
97
|
+
files = list(pool.map(download_one, symbols))
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
不要对相同 `save_dir` 并行调用多个 `sync_dataset()`:它们会共同写入 `quantdb_sync.sqlite`,可能产生锁竞争。批量同步本身仍建议一次一个数据集执行。
|
|
76
101
|
|
|
77
|
-
## 流量说明
|
|
102
|
+
## 流量说明
|
|
78
103
|
|
|
79
104
|
免费注册用户获赠 100 MB 一次性体验流量;订阅用户每月含 30 GB 下载流量,超出部分按 ¥1/GB 从账户余额扣减。余额不足时下载会被拦截。
|
|
80
105
|
|
|
@@ -3,10 +3,11 @@
|
|
|
3
3
|
提供快速验证和诊断功能:
|
|
4
4
|
python -m quantdb --version
|
|
5
5
|
python -m quantdb --help
|
|
6
|
-
|
|
6
|
+
QUANTDB_API_KEY=qdb_xxx quantdb check # 验证 API Key 是否有效
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
9
|
import argparse
|
|
10
|
+
import os
|
|
10
11
|
import sys
|
|
11
12
|
|
|
12
13
|
from . import __version__
|
|
@@ -28,19 +29,21 @@ def main() -> int:
|
|
|
28
29
|
default="https://quantdb.quantmind.cloud",
|
|
29
30
|
help="API 服务地址 (默认: https://quantdb.quantmind.cloud)",
|
|
30
31
|
)
|
|
32
|
+
parser.add_argument(
|
|
33
|
+
"--api-key-env",
|
|
34
|
+
default="QUANTDB_API_KEY",
|
|
35
|
+
help="保存 API Key 的环境变量名 (默认: QUANTDB_API_KEY)",
|
|
36
|
+
)
|
|
31
37
|
|
|
32
38
|
subparsers = parser.add_subparsers(dest="command", help="可用命令")
|
|
33
39
|
|
|
34
40
|
# check 命令:验证 API Key
|
|
35
41
|
check_parser = subparsers.add_parser("check", help="验证 API Key 是否有效")
|
|
36
|
-
check_parser.add_argument("api_key", help="要验证的 API Key")
|
|
37
42
|
|
|
38
43
|
# usage 命令:查询用量
|
|
39
44
|
usage_parser = subparsers.add_parser("usage", help="查询账户用量和订阅状态")
|
|
40
|
-
usage_parser.add_argument("api_key", help="API Key")
|
|
41
45
|
|
|
42
46
|
sync_parser = subparsers.add_parser("sync", help="以 release/Manifest 增量同步一个数据集")
|
|
43
|
-
sync_parser.add_argument("api_key", help="API Key")
|
|
44
47
|
sync_parser.add_argument("dataset", help="数据集,如 daily_forward、balance、etf_pcf")
|
|
45
48
|
sync_parser.add_argument("--save-dir", default=None, help="本地同步根目录")
|
|
46
49
|
sync_parser.add_argument("--after-release", default=None, help="覆盖本地 release cursor(一般无需传入)")
|
|
@@ -51,8 +54,12 @@ def main() -> int:
|
|
|
51
54
|
parser.print_help()
|
|
52
55
|
return 0
|
|
53
56
|
|
|
57
|
+
api_key = os.getenv(args.api_key_env)
|
|
58
|
+
if not api_key:
|
|
59
|
+
parser.error(f"请先设置 API Key 环境变量 {args.api_key_env}")
|
|
60
|
+
|
|
54
61
|
try:
|
|
55
|
-
client = QuantDBClient(api_host=args.api_host, api_key=
|
|
62
|
+
client = QuantDBClient(api_host=args.api_host, api_key=api_key)
|
|
56
63
|
|
|
57
64
|
if args.command == "check":
|
|
58
65
|
me = client.get_me()
|
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
|
|
6
6
|
import os
|
|
7
7
|
import re
|
|
8
|
-
from typing import Dict
|
|
9
|
-
from urllib.parse import unquote
|
|
8
|
+
from typing import Dict, Optional
|
|
9
|
+
from urllib.parse import unquote, urlparse
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
# ``sync_dataset`` 的统一数据集目录。V2 可用时走 release cursor;其余数据集
|
|
@@ -72,6 +72,58 @@ def parse_filename_from_content_disposition(value: str, fallback: str) -> str:
|
|
|
72
72
|
return fallback
|
|
73
73
|
|
|
74
74
|
|
|
75
|
+
def safe_filename(filename: str, fallback: str = "download.parquet") -> str:
|
|
76
|
+
"""Return one safe filename, using *fallback* for untrusted input."""
|
|
77
|
+
candidate = filename.strip()
|
|
78
|
+
if (not candidate or candidate in {".", ".."} or "/" in candidate
|
|
79
|
+
or "\\" in candidate or "\x00" in candidate
|
|
80
|
+
or re.match(r"^[A-Za-z]:", candidate)):
|
|
81
|
+
return fallback
|
|
82
|
+
return candidate
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def safe_join(root: str, relative_path: str) -> str:
|
|
86
|
+
"""Join an untrusted manifest path and prove it remains below *root*."""
|
|
87
|
+
if (not isinstance(relative_path, str) or not relative_path or "\\" in relative_path
|
|
88
|
+
or relative_path.startswith("/") or re.match(r"^[A-Za-z]:", relative_path)):
|
|
89
|
+
raise ValueError("发布清单包含非法路径")
|
|
90
|
+
parts = relative_path.split("/")
|
|
91
|
+
if any(part in {"", ".", ".."} for part in parts):
|
|
92
|
+
raise ValueError("发布清单包含路径穿越")
|
|
93
|
+
root_abs = os.path.abspath(root)
|
|
94
|
+
target = os.path.abspath(os.path.join(root_abs, *parts))
|
|
95
|
+
if os.path.commonpath([root_abs, target]) != root_abs:
|
|
96
|
+
raise ValueError("发布清单路径超出同步目录")
|
|
97
|
+
return target
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def validate_api_host(api_host: str) -> str:
|
|
101
|
+
"""Require HTTPS except for local development/test loopback endpoints."""
|
|
102
|
+
host = api_host.rstrip("/")
|
|
103
|
+
parsed = urlparse(host)
|
|
104
|
+
if parsed.scheme == "https" and parsed.netloc:
|
|
105
|
+
return host
|
|
106
|
+
if parsed.scheme == "http" and parsed.hostname in {"localhost", "127.0.0.1", "::1"}:
|
|
107
|
+
return host
|
|
108
|
+
raise ValueError("api_host 必须为 HTTPS 地址;仅 localhost 可使用 HTTP")
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def max_download_bytes() -> int:
|
|
112
|
+
"""Configured response cap; default 2 GiB protects memory and disk."""
|
|
113
|
+
try:
|
|
114
|
+
value = int(os.getenv("QUANTDB_MAX_DOWNLOAD_BYTES", str(2 * 1024 ** 3)))
|
|
115
|
+
except ValueError as exc:
|
|
116
|
+
raise ValueError("QUANTDB_MAX_DOWNLOAD_BYTES 必须是正整数") from exc
|
|
117
|
+
if value <= 0:
|
|
118
|
+
raise ValueError("QUANTDB_MAX_DOWNLOAD_BYTES 必须大于 0")
|
|
119
|
+
return value
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def check_download_size(content_length: Optional[str], maximum: int) -> None:
|
|
123
|
+
if content_length and content_length.isdigit() and int(content_length) > maximum:
|
|
124
|
+
raise ValueError(f"下载文件超过大小限制({maximum} 字节)")
|
|
125
|
+
|
|
126
|
+
|
|
75
127
|
def bytes_to_gb(n: int) -> float:
|
|
76
128
|
"""将字节数转换为 GB(1024 进制)。
|
|
77
129
|
|
|
@@ -5,12 +5,17 @@ import io
|
|
|
5
5
|
import os
|
|
6
6
|
import re
|
|
7
7
|
import sqlite3
|
|
8
|
+
from contextlib import asynccontextmanager
|
|
8
9
|
from typing import Any, Dict, List, Optional, Literal
|
|
9
10
|
|
|
10
11
|
import httpx
|
|
11
12
|
import pandas as pd
|
|
12
13
|
|
|
13
|
-
from ._utils import
|
|
14
|
+
from ._utils import (
|
|
15
|
+
SYNC_DATASET_CATEGORIES, bytes_to_gb, check_download_size, default_download_dir,
|
|
16
|
+
max_download_bytes, parse_filename_from_content_disposition, safe_filename, safe_join,
|
|
17
|
+
validate_api_host,
|
|
18
|
+
)
|
|
14
19
|
from .errors import (
|
|
15
20
|
AuthError,
|
|
16
21
|
InsufficientTrafficError,
|
|
@@ -21,6 +26,9 @@ from .errors import (
|
|
|
21
26
|
ValidationError,
|
|
22
27
|
)
|
|
23
28
|
|
|
29
|
+
# 维护提示:每次发版必须与 pyproject.toml 版本号同步
|
|
30
|
+
_USER_AGENT = "QuantDB-Python-SDK/0.2.6"
|
|
31
|
+
|
|
24
32
|
|
|
25
33
|
class AsyncQuantDBClient:
|
|
26
34
|
"""QuantDB 异步客户端。
|
|
@@ -35,9 +43,9 @@ class AsyncQuantDBClient:
|
|
|
35
43
|
token: Optional[str] = None,
|
|
36
44
|
timeout: float = 60.0,
|
|
37
45
|
):
|
|
38
|
-
self.api_host = api_host
|
|
46
|
+
self.api_host = validate_api_host(api_host)
|
|
39
47
|
self.timeout = timeout
|
|
40
|
-
headers = {"User-Agent":
|
|
48
|
+
headers = {"User-Agent": _USER_AGENT}
|
|
41
49
|
if api_key:
|
|
42
50
|
headers["X-API-Key"] = api_key
|
|
43
51
|
elif token:
|
|
@@ -49,17 +57,49 @@ class AsyncQuantDBClient:
|
|
|
49
57
|
headers=headers,
|
|
50
58
|
timeout=httpx.Timeout(timeout, connect=5.0),
|
|
51
59
|
)
|
|
60
|
+
# 裸客户端:跟随 302 直连 CDN 用,不携带任何鉴权头,避免凭证泄露给 CDN 域。
|
|
61
|
+
self._bare_client = httpx.AsyncClient(
|
|
62
|
+
headers={"User-Agent": _USER_AGENT},
|
|
63
|
+
timeout=httpx.Timeout(timeout, connect=5.0),
|
|
64
|
+
)
|
|
52
65
|
# 进程内 parquet ETag 缓存:同对象(ETag 未变)不重复下载,避免重复计费。
|
|
53
66
|
self._cache: Dict[str, Dict[str, Any]] = {}
|
|
54
67
|
|
|
55
68
|
async def close(self) -> None:
|
|
56
69
|
"""关闭底层 httpx 客户端。"""
|
|
57
70
|
await self.client.aclose()
|
|
71
|
+
await self._bare_client.aclose()
|
|
58
72
|
|
|
59
73
|
def clear_cache(self) -> None:
|
|
60
74
|
"""清空进程内 Parquet 缓存(强制下次重新下载最新数据)。"""
|
|
61
75
|
self._cache.clear()
|
|
62
76
|
|
|
77
|
+
@asynccontextmanager
|
|
78
|
+
async def _download_stream(self, params: Dict[str, Any], headers: Optional[Dict[str, str]] = None):
|
|
79
|
+
"""下载专用流式请求:手动处理服务端 302 CDN 直连跳转。
|
|
80
|
+
|
|
81
|
+
服务端开启 CDN 直连后,/api/v1/data/download 预扣流量成功时返回
|
|
82
|
+
302 -> CDN 签名 URL。httpx 默认不跟随重定向;这里用不带鉴权头的
|
|
83
|
+
裸客户端直连 Location,鉴权头不会泄露给 CDN 域。
|
|
84
|
+
"""
|
|
85
|
+
async with self.client.stream(
|
|
86
|
+
"GET", f"{self.api_host}/api/v1/data/download", params=params, headers=headers
|
|
87
|
+
) as resp:
|
|
88
|
+
if resp.status_code not in (301, 302, 303, 307, 308):
|
|
89
|
+
yield resp
|
|
90
|
+
return
|
|
91
|
+
location = resp.headers.get("Location", "")
|
|
92
|
+
etag = resp.headers.get("ETag", "")
|
|
93
|
+
if not location:
|
|
94
|
+
raise ServerError("下载重定向缺少 Location 头")
|
|
95
|
+
async with self._bare_client.stream("GET", location) as cdn_resp:
|
|
96
|
+
if cdn_resp.status_code != 200:
|
|
97
|
+
raise ServerError(f"CDN 直连下载失败:HTTP {cdn_resp.status_code}")
|
|
98
|
+
# 网关 302 响应携带 COS ETag;若 CDN 响应缺失则回填,保证缓存逻辑一致。
|
|
99
|
+
if etag and not cdn_resp.headers.get("ETag"):
|
|
100
|
+
cdn_resp.headers["ETag"] = etag
|
|
101
|
+
yield cdn_resp
|
|
102
|
+
|
|
63
103
|
@staticmethod
|
|
64
104
|
def _validate_layout(layout: str) -> Literal["auto", "v1", "v2"]:
|
|
65
105
|
if layout not in {"auto", "v1", "v2"}:
|
|
@@ -394,9 +434,7 @@ class AsyncQuantDBClient:
|
|
|
394
434
|
if object_key:
|
|
395
435
|
params["object_key"] = object_key
|
|
396
436
|
|
|
397
|
-
async with self.
|
|
398
|
-
"GET", f"{self.api_host}/api/v1/data/download", params=params
|
|
399
|
-
) as resp:
|
|
437
|
+
async with self._download_stream(params) as resp:
|
|
400
438
|
if resp.status_code != 200:
|
|
401
439
|
body = await resp.aread()
|
|
402
440
|
# 构造一个完整的 httpx.Response 用于错误解析,然后立即抛出
|
|
@@ -411,15 +449,26 @@ class AsyncQuantDBClient:
|
|
|
411
449
|
fallback = f"{sub_category}.parquet"
|
|
412
450
|
if symbol:
|
|
413
451
|
fallback = f"{sub_category}_{symbol}.parquet"
|
|
414
|
-
filename = parse_filename_from_content_disposition(cd, fallback)
|
|
452
|
+
filename = safe_filename(parse_filename_from_content_disposition(cd, fallback), safe_filename(fallback))
|
|
415
453
|
|
|
416
454
|
save_path = os.path.join(save_dir, filename)
|
|
417
455
|
tmp_path = save_path + ".part"
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
456
|
+
maximum = max_download_bytes()
|
|
457
|
+
check_download_size(resp.headers.get("Content-Length"), maximum)
|
|
458
|
+
written = 0
|
|
459
|
+
try:
|
|
460
|
+
with open(tmp_path, "wb") as f:
|
|
461
|
+
async for chunk in resp.aiter_bytes(chunk_size=8192):
|
|
462
|
+
if chunk:
|
|
463
|
+
written += len(chunk)
|
|
464
|
+
if written > maximum:
|
|
465
|
+
raise ServerError(f"下载文件超过大小限制({maximum} 字节)")
|
|
466
|
+
f.write(chunk)
|
|
467
|
+
os.replace(tmp_path, save_path)
|
|
468
|
+
except Exception:
|
|
469
|
+
if os.path.exists(tmp_path):
|
|
470
|
+
os.remove(tmp_path)
|
|
471
|
+
raise
|
|
423
472
|
return os.path.abspath(save_path)
|
|
424
473
|
|
|
425
474
|
async def a_load_as_df(
|
|
@@ -452,17 +501,25 @@ class AsyncQuantDBClient:
|
|
|
452
501
|
req_headers = {}
|
|
453
502
|
if cached and cached.get("etag"):
|
|
454
503
|
req_headers["If-None-Match"] = cached["etag"]
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
504
|
+
async with self._download_stream(params, headers=req_headers) as resp:
|
|
505
|
+
if resp.status_code == 304 and cached:
|
|
506
|
+
return cached["df"]
|
|
507
|
+
if resp.status_code != 200:
|
|
508
|
+
body = await resp.aread()
|
|
509
|
+
self._check_response(httpx.Response(resp.status_code, content=body))
|
|
510
|
+
etag = resp.headers.get("ETag") or ""
|
|
511
|
+
if cached and etag and cached.get("etag") == etag:
|
|
512
|
+
return cached["df"]
|
|
513
|
+
maximum = max_download_bytes()
|
|
514
|
+
check_download_size(resp.headers.get("Content-Length"), maximum)
|
|
515
|
+
payload, written = io.BytesIO(), 0
|
|
516
|
+
async for chunk in resp.aiter_bytes(chunk_size=1024 * 1024):
|
|
517
|
+
if chunk:
|
|
518
|
+
written += len(chunk)
|
|
519
|
+
if written > maximum:
|
|
520
|
+
raise ServerError(f"下载文件超过大小限制({maximum} 字节)")
|
|
521
|
+
payload.write(chunk)
|
|
522
|
+
df = pd.read_parquet(payload)
|
|
466
523
|
if etag:
|
|
467
524
|
self._cache[cache_key] = {"etag": etag, "df": df}
|
|
468
525
|
return df
|
|
@@ -480,13 +537,12 @@ class AsyncQuantDBClient:
|
|
|
480
537
|
若文件不存在会先自动下载。
|
|
481
538
|
|
|
482
539
|
sql 支持两种写法:
|
|
483
|
-
1.
|
|
484
|
-
2. 纯 WHERE 条件字符串(不含 FROM 时自动拼装为 SELECT * FROM <path> WHERE <条件>)
|
|
540
|
+
1. 仅支持纯 WHERE 条件字符串,SDK 固定查询已下载的 Parquet 文件
|
|
485
541
|
|
|
486
542
|
安全限制:
|
|
487
543
|
- 不允许分号(;),防止多语句注入
|
|
488
544
|
- 不允许注释(-- 或 /* */),防止注释注入
|
|
489
|
-
-
|
|
545
|
+
- 不允许子查询、JOIN、外部表函数或任何数据修改语句
|
|
490
546
|
"""
|
|
491
547
|
file_path = await self.a_download_file(
|
|
492
548
|
category_id=category_id,
|
|
@@ -503,27 +559,18 @@ class AsyncQuantDBClient:
|
|
|
503
559
|
|
|
504
560
|
# 安全检查:拒绝危险字符和关键字
|
|
505
561
|
dangerous_keywords = [
|
|
506
|
-
";", "--", "/*", "*/", "
|
|
507
|
-
"insert", "update", "alter", "create", "exec", "execute",
|
|
562
|
+
";", "--", "/*", "*/", "select", "from", "join", "union", "drop",
|
|
563
|
+
"delete", "insert", "update", "alter", "create", "exec", "execute",
|
|
564
|
+
"attach", "copy", "pragma", "install", "load", "read_", "http", "glob",
|
|
508
565
|
]
|
|
509
566
|
sql_upper = sql.upper()
|
|
510
567
|
for kw in dangerous_keywords:
|
|
511
568
|
if kw in sql_upper:
|
|
512
569
|
raise ValidationError(
|
|
513
570
|
f"SQL 包含危险关键字 '{kw}',已被拒绝。"
|
|
514
|
-
"a_query_local
|
|
571
|
+
"a_query_local 仅支持针对已下载文件的 WHERE 条件。"
|
|
515
572
|
)
|
|
516
|
-
|
|
517
|
-
if "FROM" in sql_upper:
|
|
518
|
-
sql = re.sub(
|
|
519
|
-
r"FROM\s+[`\"\']?\w+[`\"\']?",
|
|
520
|
-
f"FROM '{clean_path}'",
|
|
521
|
-
sql,
|
|
522
|
-
count=1,
|
|
523
|
-
flags=re.IGNORECASE,
|
|
524
|
-
)
|
|
525
|
-
else:
|
|
526
|
-
sql = f"SELECT * FROM '{clean_path}' WHERE {sql}"
|
|
573
|
+
sql = f"SELECT * FROM '{clean_path}' WHERE {sql}"
|
|
527
574
|
return duckdb.query(sql).df()
|
|
528
575
|
|
|
529
576
|
async def a_sync_dataset(self, dataset: str, save_dir: Optional[str] = None, after_release: Optional[str] = None) -> Dict[str, Any]:
|
|
@@ -546,13 +593,13 @@ class AsyncQuantDBClient:
|
|
|
546
593
|
for obj in release.get("objects", []):
|
|
547
594
|
key = self._normalise_release_key(obj["key"])
|
|
548
595
|
relative_path = obj.get("relative_path") or key
|
|
549
|
-
target =
|
|
596
|
+
target = safe_join(root, relative_path)
|
|
550
597
|
expected_size = obj.get("size")
|
|
551
598
|
old = state.execute("SELECT etag,sha256,size,path FROM objects WHERE key=?", (key,)).fetchone()
|
|
552
599
|
if old and old[0] == obj.get("etag") and old[1] == obj.get("sha256") and os.path.exists(old[3]) and (expected_size is None or os.path.getsize(old[3]) == expected_size): continue
|
|
553
600
|
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
554
601
|
tmp, digest = target + ".part", hashlib.sha256()
|
|
555
|
-
async with self.
|
|
602
|
+
async with self._download_stream({"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v2", "object_key": key}) as resp:
|
|
556
603
|
if resp.status_code != 200:
|
|
557
604
|
body = await resp.aread(); self._check_response(httpx.Response(resp.status_code, content=body)); raise QuantDBError("下载失败")
|
|
558
605
|
try:
|
|
@@ -573,14 +620,14 @@ class AsyncQuantDBClient:
|
|
|
573
620
|
if persisted: return {"dataset": dataset, "layout": "v2_daily_partition", "downloaded": [], "after_release": cursor, "release_id": persisted[0]}
|
|
574
621
|
files = (await self._get("/api/v1/data/download/manifest", {"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1"})).get("files", [])
|
|
575
622
|
for obj in files:
|
|
576
|
-
key, target = obj["key"],
|
|
623
|
+
key, target = obj["key"], safe_join(root, obj.get("relative_path") or obj["key"])
|
|
577
624
|
expected_size = obj.get("size")
|
|
578
625
|
old = state.execute("SELECT etag,size,path FROM objects WHERE key=?", (key,)).fetchone()
|
|
579
626
|
if old and old[0] == obj.get("etag") and os.path.exists(old[2]) and (expected_size is None or os.path.getsize(old[2]) == expected_size): continue
|
|
580
627
|
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
581
628
|
tmp, written_size = target + ".part", 0
|
|
582
629
|
try:
|
|
583
|
-
async with self.
|
|
630
|
+
async with self._download_stream({"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1", "symbol": obj.get("symbol", "")}) as resp:
|
|
584
631
|
if resp.status_code != 200:
|
|
585
632
|
body = await resp.aread(); self._check_response(httpx.Response(resp.status_code, content=body)); raise QuantDBError("下载失败")
|
|
586
633
|
with open(tmp, "wb") as fh:
|