quantdb-sdk 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/CHANGELOG.md +16 -0
- {quantdb_sdk-0.2.0/quantdb_sdk.egg-info → quantdb_sdk-0.2.2}/PKG-INFO +6 -1
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/README.md +5 -0
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/pyproject.toml +1 -1
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk/__init__.py +1 -1
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk/__main__.py +14 -0
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk/_utils.py +16 -0
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk/async_client.py +39 -22
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk/client.py +29 -20
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2/quantdb_sdk.egg-info}/PKG-INFO +6 -1
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/tests/test_async_client.py +14 -1
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/tests/test_client.py +31 -1
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/LICENSE +0 -0
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/MANIFEST.in +0 -0
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk/errors.py +0 -0
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk/py.typed +0 -0
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/SOURCES.txt +0 -0
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/dependency_links.txt +0 -0
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/entry_points.txt +0 -0
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/requires.txt +0 -0
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/top_level.txt +0 -0
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/setup.cfg +0 -0
- {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/tests/test_technical_indicators.py +0 -0
|
@@ -3,6 +3,22 @@
|
|
|
3
3
|
所有 notable 变更都会记录在此文件。格式基于 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.1.0/),
|
|
4
4
|
版本号遵循 [Semantic Versioning](https://semver.org/lang/zh-CN/)。
|
|
5
5
|
|
|
6
|
+
## [0.2.2] - 2026-07-26
|
|
7
|
+
|
|
8
|
+
### Added
|
|
9
|
+
- **全数据集增量同步**:`sync_dataset()` / `a_sync_dataset()` 覆盖基础、财务、ETF/可转债、行情、技术与 ML 子类;没有 V2 release 的数据集自动走 V1 Manifest。
|
|
10
|
+
- **CLI 同步**:新增 `quantdb sync <api_key> <dataset> --save-dir <目录>`。
|
|
11
|
+
|
|
12
|
+
### Fixed
|
|
13
|
+
- **V1 强校验**:V1 Manifest 同步以 ETag + size 判断变更,并在原子落盘前验证实际字节数。
|
|
14
|
+
- **同步/异步一致性**:异步 release 同步采用 `relative_path`,SQLite 同时记录对象大小,与同步客户端保持同构目录和恢复语义。
|
|
15
|
+
|
|
16
|
+
## [0.2.1] - 2026-07-26
|
|
17
|
+
|
|
18
|
+
### Fixed
|
|
19
|
+
- **历史发布清单兼容**:同步客户端自动归一化旧 release manifest 遗留的 `v2/` 前缀,并可下载 release patch 对象。
|
|
20
|
+
- **布局回退**:服务端以 COS Manifest 为可用性权威,V2 元数据缺失时可正确回退 V1,而非将数据误判为不可下载。
|
|
21
|
+
|
|
6
22
|
## [0.2.0] - 2026-07-26
|
|
7
23
|
|
|
8
24
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: quantdb-sdk
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: QuantDB 量化数据平台官方 Python SDK
|
|
5
5
|
Author: QuantDB Team
|
|
6
6
|
License: MIT
|
|
@@ -104,6 +104,11 @@ history = client.download_file("1", "daily_forward", symbol="600519.SH", layout=
|
|
|
104
104
|
|
|
105
105
|
# 以发布清单为 cursor 做原子化增量同步(含 V2 patch)
|
|
106
106
|
result = client.sync_dataset("daily_forward", save_dir="D:/quantdb-data")
|
|
107
|
+
|
|
108
|
+
# 财务、ETF/可转债等尚无 V2 release 的数据集自动按 V1 Manifest 增量同步,
|
|
109
|
+
# 使用 ETag + size 校验;也可从命令行执行:
|
|
110
|
+
# quantdb sync qdb_xxx balance --save-dir D:/quantdb-data
|
|
111
|
+
financial = client.sync_dataset("balance", save_dir="D:/quantdb-data")
|
|
107
112
|
```
|
|
108
113
|
|
|
109
114
|
## 流量说明
|
|
@@ -67,6 +67,11 @@ history = client.download_file("1", "daily_forward", symbol="600519.SH", layout=
|
|
|
67
67
|
|
|
68
68
|
# 以发布清单为 cursor 做原子化增量同步(含 V2 patch)
|
|
69
69
|
result = client.sync_dataset("daily_forward", save_dir="D:/quantdb-data")
|
|
70
|
+
|
|
71
|
+
# 财务、ETF/可转债等尚无 V2 release 的数据集自动按 V1 Manifest 增量同步,
|
|
72
|
+
# 使用 ETag + size 校验;也可从命令行执行:
|
|
73
|
+
# quantdb sync qdb_xxx balance --save-dir D:/quantdb-data
|
|
74
|
+
financial = client.sync_dataset("balance", save_dir="D:/quantdb-data")
|
|
70
75
|
```
|
|
71
76
|
|
|
72
77
|
## 流量说明
|
|
@@ -39,6 +39,12 @@ def main() -> int:
|
|
|
39
39
|
usage_parser = subparsers.add_parser("usage", help="查询账户用量和订阅状态")
|
|
40
40
|
usage_parser.add_argument("api_key", help="API Key")
|
|
41
41
|
|
|
42
|
+
sync_parser = subparsers.add_parser("sync", help="以 release/Manifest 增量同步一个数据集")
|
|
43
|
+
sync_parser.add_argument("api_key", help="API Key")
|
|
44
|
+
sync_parser.add_argument("dataset", help="数据集,如 daily_forward、balance、etf_pcf")
|
|
45
|
+
sync_parser.add_argument("--save-dir", default=None, help="本地同步根目录")
|
|
46
|
+
sync_parser.add_argument("--after-release", default=None, help="覆盖本地 release cursor(一般无需传入)")
|
|
47
|
+
|
|
42
48
|
args = parser.parse_args()
|
|
43
49
|
|
|
44
50
|
if not args.command:
|
|
@@ -66,6 +72,14 @@ def main() -> int:
|
|
|
66
72
|
print(f" 订阅: {sub.get('status', 'none')}")
|
|
67
73
|
return 0
|
|
68
74
|
|
|
75
|
+
if args.command == "sync":
|
|
76
|
+
result = client.sync_dataset(args.dataset, save_dir=args.save_dir, after_release=args.after_release)
|
|
77
|
+
print(f"同步完成: {result['dataset']} ({result['layout']})")
|
|
78
|
+
print(f" 新增/更新对象: {len(result['downloaded'])}")
|
|
79
|
+
if result.get("release_id"):
|
|
80
|
+
print(f" Release: {result['release_id']}")
|
|
81
|
+
return 0
|
|
82
|
+
|
|
69
83
|
except Exception as e:
|
|
70
84
|
print(f"✗ 错误: {e}", file=sys.stderr)
|
|
71
85
|
return 1
|
|
@@ -5,9 +5,25 @@
|
|
|
5
5
|
|
|
6
6
|
import os
|
|
7
7
|
import re
|
|
8
|
+
from typing import Dict
|
|
8
9
|
from urllib.parse import unquote
|
|
9
10
|
|
|
10
11
|
|
|
12
|
+
# ``sync_dataset`` 的统一数据集目录。V2 可用时走 release cursor;其余数据集
|
|
13
|
+
# 自动使用 V1 Manifest 的 ETag + size 增量同步。
|
|
14
|
+
SYNC_DATASET_CATEGORIES: Dict[str, str] = {
|
|
15
|
+
"daily_unadjusted": "1", "daily_forward": "1", "daily_backward": "1",
|
|
16
|
+
"index_daily": "1", "min1_kline": "1", "min5_kline": "1", "tick_data": "1",
|
|
17
|
+
"instrument_detail": "2", "sector_concept": "2", "index_weights": "2",
|
|
18
|
+
"trading_calendar": "2", "margin_trading": "2",
|
|
19
|
+
"balance": "3", "income": "3", "cashflow": "3", "capital": "3",
|
|
20
|
+
"holder_num": "3", "pershare_index": "3", "dividend_factors": "3",
|
|
21
|
+
"etf_pcf": "4", "convertible_bond": "4",
|
|
22
|
+
"valuation": "5", "technical_indicators": "5", "market_sentiment": "5",
|
|
23
|
+
"features_daily": "6", "l1_l2_factors": "6", "l2_factors": "6",
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
|
|
11
27
|
def default_download_dir() -> str:
|
|
12
28
|
"""返回默认下载目录。
|
|
13
29
|
|
|
@@ -10,7 +10,7 @@ from typing import Any, Dict, List, Optional, Literal
|
|
|
10
10
|
import httpx
|
|
11
11
|
import pandas as pd
|
|
12
12
|
|
|
13
|
-
from ._utils import bytes_to_gb, default_download_dir, parse_filename_from_content_disposition
|
|
13
|
+
from ._utils import SYNC_DATASET_CATEGORIES, bytes_to_gb, default_download_dir, parse_filename_from_content_disposition
|
|
14
14
|
from .errors import (
|
|
15
15
|
AuthError,
|
|
16
16
|
InsufficientTrafficError,
|
|
@@ -37,7 +37,7 @@ class AsyncQuantDBClient:
|
|
|
37
37
|
):
|
|
38
38
|
self.api_host = api_host.rstrip("/")
|
|
39
39
|
self.timeout = timeout
|
|
40
|
-
headers = {"User-Agent": "QuantDB-Python-SDK/0.2.
|
|
40
|
+
headers = {"User-Agent": "QuantDB-Python-SDK/0.2.2"}
|
|
41
41
|
if api_key:
|
|
42
42
|
headers["X-API-Key"] = api_key
|
|
43
43
|
elif token:
|
|
@@ -66,6 +66,10 @@ class AsyncQuantDBClient:
|
|
|
66
66
|
raise ValidationError("layout 仅支持 auto、v1 或 v2")
|
|
67
67
|
return layout # type: ignore[return-value]
|
|
68
68
|
|
|
69
|
+
@staticmethod
|
|
70
|
+
def _normalise_release_key(key: str) -> str:
|
|
71
|
+
return key.strip().lstrip("/").removeprefix("v2/")
|
|
72
|
+
|
|
69
73
|
@staticmethod
|
|
70
74
|
def _normalise_kline(df: pd.DataFrame, start_date: Optional[str], end_date: Optional[str], fields: str, limit: Optional[int]) -> pd.DataFrame:
|
|
71
75
|
if "time" in df.columns:
|
|
@@ -524,17 +528,13 @@ class AsyncQuantDBClient:
|
|
|
524
528
|
|
|
525
529
|
async def a_sync_dataset(self, dataset: str, save_dir: Optional[str] = None, after_release: Optional[str] = None) -> Dict[str, Any]:
|
|
526
530
|
"""异步版 release 增量同步;状态格式与 ``sync_dataset`` 兼容。"""
|
|
527
|
-
|
|
528
|
-
"daily_unadjusted": "1", "daily_forward": "1", "daily_backward": "1", "index_daily": "1",
|
|
529
|
-
"min1_kline": "1", "min5_kline": "1", "margin_trading": "2", "valuation": "5",
|
|
530
|
-
"technical_indicators": "5", "market_sentiment": "5", "features_daily": "6",
|
|
531
|
-
}
|
|
532
|
-
if dataset not in category_map: raise ValidationError(f"不支持同步的数据集: {dataset}")
|
|
531
|
+
if dataset not in SYNC_DATASET_CATEGORIES: raise ValidationError(f"不支持同步的数据集: {dataset}")
|
|
533
532
|
root = os.path.abspath(save_dir or default_download_dir()); os.makedirs(root, exist_ok=True)
|
|
534
533
|
state = sqlite3.connect(os.path.join(root, "quantdb_sync.sqlite"))
|
|
535
|
-
state.execute("CREATE TABLE IF NOT EXISTS objects (key TEXT PRIMARY KEY, etag TEXT, sha256 TEXT, path TEXT, layout TEXT, dataset TEXT)")
|
|
534
|
+
state.execute("CREATE TABLE IF NOT EXISTS objects (key TEXT PRIMARY KEY, etag TEXT, sha256 TEXT, size INTEGER, path TEXT, layout TEXT, dataset TEXT)")
|
|
536
535
|
cols = {row[1] for row in state.execute("PRAGMA table_info(objects)")}
|
|
537
536
|
if "dataset" not in cols: state.execute("ALTER TABLE objects ADD COLUMN dataset TEXT")
|
|
537
|
+
if "size" not in cols: state.execute("ALTER TABLE objects ADD COLUMN size INTEGER")
|
|
538
538
|
state.execute("CREATE TABLE IF NOT EXISTS releases (dataset TEXT PRIMARY KEY, release_id TEXT NOT NULL)")
|
|
539
539
|
downloaded: List[str] = []
|
|
540
540
|
try:
|
|
@@ -544,12 +544,15 @@ class AsyncQuantDBClient:
|
|
|
544
544
|
if releases:
|
|
545
545
|
for release in releases:
|
|
546
546
|
for obj in release.get("objects", []):
|
|
547
|
-
key
|
|
548
|
-
|
|
549
|
-
|
|
547
|
+
key = self._normalise_release_key(obj["key"])
|
|
548
|
+
relative_path = obj.get("relative_path") or key
|
|
549
|
+
target = os.path.join(root, *relative_path.split("/"))
|
|
550
|
+
expected_size = obj.get("size")
|
|
551
|
+
old = state.execute("SELECT etag,sha256,size,path FROM objects WHERE key=?", (key,)).fetchone()
|
|
552
|
+
if old and old[0] == obj.get("etag") and old[1] == obj.get("sha256") and os.path.exists(old[3]) and (expected_size is None or os.path.getsize(old[3]) == expected_size): continue
|
|
550
553
|
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
551
554
|
tmp, digest = target + ".part", hashlib.sha256()
|
|
552
|
-
async with self.client.stream("GET", f"{self.api_host}/api/v1/data/download", params={"category_id":
|
|
555
|
+
async with self.client.stream("GET", f"{self.api_host}/api/v1/data/download", params={"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v2", "object_key": key}) as resp:
|
|
553
556
|
if resp.status_code != 200:
|
|
554
557
|
body = await resp.aread(); self._check_response(httpx.Response(resp.status_code, content=body)); raise QuantDBError("下载失败")
|
|
555
558
|
try:
|
|
@@ -558,24 +561,38 @@ class AsyncQuantDBClient:
|
|
|
558
561
|
if chunk: fh.write(chunk); digest.update(chunk)
|
|
559
562
|
actual = digest.hexdigest()
|
|
560
563
|
if obj.get("sha256") and actual.lower() != obj["sha256"].lower(): raise ServerError("对象 SHA-256 校验失败")
|
|
564
|
+
written_size = os.path.getsize(tmp)
|
|
565
|
+
if expected_size is not None and written_size != expected_size: raise ServerError(f"对象大小校验失败:期望 {expected_size},实际 {written_size}")
|
|
561
566
|
os.replace(tmp, target)
|
|
562
567
|
except Exception:
|
|
563
568
|
if os.path.exists(tmp): os.remove(tmp)
|
|
564
569
|
raise
|
|
565
|
-
state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,path,layout,dataset) VALUES(
|
|
570
|
+
state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,size,path,layout,dataset) VALUES(?,?,?,?,?,?,?)", (key, obj.get("etag"), actual, written_size, target, "v2_daily_partition", dataset)); downloaded.append(key)
|
|
566
571
|
state.execute("INSERT OR REPLACE INTO releases(dataset,release_id) VALUES(?,?)", (dataset, release["release_id"])); state.commit()
|
|
567
572
|
return {"dataset": dataset, "layout": "v2_daily_partition", "downloaded": downloaded, "after_release": cursor, "release_id": releases[-1]["release_id"]}
|
|
568
573
|
if persisted: return {"dataset": dataset, "layout": "v2_daily_partition", "downloaded": [], "after_release": cursor, "release_id": persisted[0]}
|
|
569
|
-
files = (await self._get("/api/v1/data/download/manifest", {"category_id":
|
|
574
|
+
files = (await self._get("/api/v1/data/download/manifest", {"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1"})).get("files", [])
|
|
570
575
|
for obj in files:
|
|
571
576
|
key, target = obj["key"], os.path.join(root, *(obj.get("relative_path") or obj["key"]).split("/"))
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
577
|
+
expected_size = obj.get("size")
|
|
578
|
+
old = state.execute("SELECT etag,size,path FROM objects WHERE key=?", (key,)).fetchone()
|
|
579
|
+
if old and old[0] == obj.get("etag") and os.path.exists(old[2]) and (expected_size is None or os.path.getsize(old[2]) == expected_size): continue
|
|
580
|
+
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
581
|
+
tmp, written_size = target + ".part", 0
|
|
582
|
+
try:
|
|
583
|
+
async with self.client.stream("GET", f"{self.api_host}/api/v1/data/download", params={"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1", "symbol": obj.get("symbol", "")}) as resp:
|
|
584
|
+
if resp.status_code != 200:
|
|
585
|
+
body = await resp.aread(); self._check_response(httpx.Response(resp.status_code, content=body)); raise QuantDBError("下载失败")
|
|
586
|
+
with open(tmp, "wb") as fh:
|
|
587
|
+
async for chunk in resp.aiter_bytes(1024 * 1024):
|
|
588
|
+
if chunk: fh.write(chunk)
|
|
589
|
+
written_size = os.path.getsize(tmp)
|
|
590
|
+
if expected_size is not None and written_size != expected_size: raise ServerError(f"对象大小校验失败:期望 {expected_size},实际 {written_size}")
|
|
591
|
+
os.replace(tmp, target)
|
|
592
|
+
except Exception:
|
|
593
|
+
if os.path.exists(tmp): os.remove(tmp)
|
|
594
|
+
raise
|
|
595
|
+
state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,size,path,layout,dataset) VALUES(?,?,?,?,?,?,?)", (key, obj.get("etag"), "", written_size, target, "v1_symbol", dataset)); downloaded.append(key)
|
|
579
596
|
state.commit(); return {"dataset": dataset, "layout": "v1_symbol", "downloaded": downloaded, "after_release": cursor}
|
|
580
597
|
finally:
|
|
581
598
|
state.close()
|
|
@@ -13,7 +13,7 @@ import requests
|
|
|
13
13
|
from requests.adapters import HTTPAdapter
|
|
14
14
|
from urllib3.util.retry import Retry
|
|
15
15
|
|
|
16
|
-
from ._utils import bytes_to_gb, default_download_dir, parse_filename_from_content_disposition
|
|
16
|
+
from ._utils import SYNC_DATASET_CATEGORIES, bytes_to_gb, default_download_dir, parse_filename_from_content_disposition
|
|
17
17
|
from .errors import (
|
|
18
18
|
AuthError,
|
|
19
19
|
InsufficientTrafficError,
|
|
@@ -48,7 +48,7 @@ class QuantDBClient:
|
|
|
48
48
|
self.session = requests.Session()
|
|
49
49
|
# User-Agent 中的版本与 pyproject.toml 同步,用于服务端日志归因
|
|
50
50
|
# 维护提示:每次版本号变化必须同步改这里(init 里的 __version__ 走 metadata 自动同步)
|
|
51
|
-
self.session.headers.update({"User-Agent": "QuantDB-Python-SDK/0.2.
|
|
51
|
+
self.session.headers.update({"User-Agent": "QuantDB-Python-SDK/0.2.2"})
|
|
52
52
|
|
|
53
53
|
if api_key:
|
|
54
54
|
self.headers = {"X-API-Key": api_key}
|
|
@@ -164,6 +164,11 @@ class QuantDBClient:
|
|
|
164
164
|
raise ValidationError("layout 仅支持 auto、v1 或 v2")
|
|
165
165
|
return layout # type: ignore[return-value]
|
|
166
166
|
|
|
167
|
+
@staticmethod
|
|
168
|
+
def _normalise_release_key(key: str) -> str:
|
|
169
|
+
"""兼容历史 release manifest 遗留的 ``v2/`` 对象前缀。"""
|
|
170
|
+
return key.strip().lstrip("/").removeprefix("v2/")
|
|
171
|
+
|
|
167
172
|
@staticmethod
|
|
168
173
|
def _normalise_kline(df: pd.DataFrame, start_date: Optional[str], end_date: Optional[str], fields: str, limit: Optional[int]) -> pd.DataFrame:
|
|
169
174
|
"""统一 V1/V2 日线结果并在客户端按标的、日期与字段过滤。"""
|
|
@@ -617,18 +622,14 @@ class QuantDBClient:
|
|
|
617
622
|
root = os.path.abspath(save_dir or default_download_dir())
|
|
618
623
|
os.makedirs(root, exist_ok=True)
|
|
619
624
|
state = sqlite3.connect(os.path.join(root, "quantdb_sync.sqlite"))
|
|
620
|
-
state.execute("CREATE TABLE IF NOT EXISTS objects (key TEXT PRIMARY KEY, etag TEXT, sha256 TEXT, path TEXT, layout TEXT, dataset TEXT)")
|
|
625
|
+
state.execute("CREATE TABLE IF NOT EXISTS objects (key TEXT PRIMARY KEY, etag TEXT, sha256 TEXT, size INTEGER, path TEXT, layout TEXT, dataset TEXT)")
|
|
621
626
|
cols = {row[1] for row in state.execute("PRAGMA table_info(objects)")}
|
|
622
627
|
if "dataset" not in cols:
|
|
623
628
|
state.execute("ALTER TABLE objects ADD COLUMN dataset TEXT")
|
|
629
|
+
if "size" not in cols:
|
|
630
|
+
state.execute("ALTER TABLE objects ADD COLUMN size INTEGER")
|
|
624
631
|
state.execute("CREATE TABLE IF NOT EXISTS releases (dataset TEXT PRIMARY KEY, release_id TEXT NOT NULL)")
|
|
625
|
-
|
|
626
|
-
"daily_unadjusted": "1", "daily_forward": "1", "daily_backward": "1",
|
|
627
|
-
"index_daily": "1", "min1_kline": "1", "min5_kline": "1",
|
|
628
|
-
"margin_trading": "2", "valuation": "5", "technical_indicators": "5",
|
|
629
|
-
"market_sentiment": "5", "features_daily": "6",
|
|
630
|
-
}
|
|
631
|
-
if dataset not in category_map:
|
|
632
|
+
if dataset not in SYNC_DATASET_CATEGORIES:
|
|
632
633
|
raise ValidationError(f"不支持同步的数据集: {dataset}")
|
|
633
634
|
|
|
634
635
|
downloaded: List[str] = []
|
|
@@ -641,13 +642,14 @@ class QuantDBClient:
|
|
|
641
642
|
for release in releases:
|
|
642
643
|
release_id = release["release_id"]
|
|
643
644
|
for obj in release.get("objects", []):
|
|
644
|
-
key = obj["key"]
|
|
645
|
+
key = self._normalise_release_key(obj["key"])
|
|
645
646
|
relative_path = obj.get("relative_path") or key
|
|
646
647
|
target = os.path.join(root, *relative_path.split("/"))
|
|
647
|
-
|
|
648
|
-
|
|
648
|
+
expected_size = obj.get("size")
|
|
649
|
+
old = state.execute("SELECT etag, sha256, size, path FROM objects WHERE key=?", (key,)).fetchone()
|
|
650
|
+
if old and old[0] == obj.get("etag") and old[1] == obj.get("sha256") and os.path.exists(old[3]) and (expected_size is None or os.path.getsize(old[3]) == expected_size):
|
|
649
651
|
continue
|
|
650
|
-
resp = self._request("GET", "/api/v1/data/download", params={"category_id":
|
|
652
|
+
resp = self._request("GET", "/api/v1/data/download", params={"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v2", "object_key": key}, stream=True)
|
|
651
653
|
if resp.status_code != 200:
|
|
652
654
|
self._check_response(resp)
|
|
653
655
|
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
@@ -658,13 +660,16 @@ class QuantDBClient:
|
|
|
658
660
|
if chunk:
|
|
659
661
|
fh.write(chunk); digest.update(chunk)
|
|
660
662
|
actual = digest.hexdigest()
|
|
663
|
+
written_size = os.path.getsize(tmp)
|
|
661
664
|
if obj.get("sha256") and actual.lower() != obj["sha256"].lower():
|
|
662
665
|
raise ServerError("对象 SHA-256 校验失败")
|
|
666
|
+
if expected_size is not None and written_size != expected_size:
|
|
667
|
+
raise ServerError(f"对象大小校验失败:期望 {expected_size},实际 {written_size}")
|
|
663
668
|
os.replace(tmp, target)
|
|
664
669
|
except Exception:
|
|
665
670
|
if os.path.exists(tmp): os.remove(tmp)
|
|
666
671
|
raise
|
|
667
|
-
state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,path,layout,dataset) VALUES(
|
|
672
|
+
state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,size,path,layout,dataset) VALUES(?,?,?,?,?,?,?)", (key, obj.get("etag"), actual, written_size, target, "v2_daily_partition", dataset))
|
|
668
673
|
downloaded.append(key)
|
|
669
674
|
# 仅在该 release 的所有对象成功落盘和校验后提交 cursor。
|
|
670
675
|
state.execute("INSERT OR REPLACE INTO releases(dataset,release_id) VALUES(?,?)", (dataset, release_id))
|
|
@@ -674,14 +679,15 @@ class QuantDBClient:
|
|
|
674
679
|
if persisted:
|
|
675
680
|
return {"dataset": dataset, "layout": "v2_daily_partition", "downloaded": [], "after_release": cursor, "release_id": persisted[0]}
|
|
676
681
|
|
|
677
|
-
manifest = self._get("/api/v1/data/download/manifest", {"category_id":
|
|
682
|
+
manifest = self._get("/api/v1/data/download/manifest", {"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1"})
|
|
678
683
|
for obj in manifest.get("files", []):
|
|
679
684
|
key, relative_path = obj["key"], obj.get("relative_path") or obj["key"]
|
|
680
685
|
target = os.path.join(root, *relative_path.split("/"))
|
|
681
|
-
|
|
682
|
-
|
|
686
|
+
expected_size = obj.get("size")
|
|
687
|
+
old = state.execute("SELECT etag, size, path FROM objects WHERE key=?", (key,)).fetchone()
|
|
688
|
+
if old and old[0] == obj.get("etag") and os.path.exists(old[2]) and (expected_size is None or os.path.getsize(old[2]) == expected_size):
|
|
683
689
|
continue
|
|
684
|
-
resp = self._request("GET", "/api/v1/data/download", params={"category_id":
|
|
690
|
+
resp = self._request("GET", "/api/v1/data/download", params={"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1", "symbol": obj.get("symbol", "")}, stream=True)
|
|
685
691
|
if resp.status_code != 200: self._check_response(resp)
|
|
686
692
|
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
687
693
|
tmp = target + ".part"
|
|
@@ -689,11 +695,14 @@ class QuantDBClient:
|
|
|
689
695
|
with open(tmp, "wb") as fh:
|
|
690
696
|
for chunk in resp.iter_content(1024 * 1024):
|
|
691
697
|
if chunk: fh.write(chunk)
|
|
698
|
+
written_size = os.path.getsize(tmp)
|
|
699
|
+
if expected_size is not None and written_size != expected_size:
|
|
700
|
+
raise ServerError(f"对象大小校验失败:期望 {expected_size},实际 {written_size}")
|
|
692
701
|
os.replace(tmp, target)
|
|
693
702
|
except Exception:
|
|
694
703
|
if os.path.exists(tmp): os.remove(tmp)
|
|
695
704
|
raise
|
|
696
|
-
state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,path,layout,dataset) VALUES(
|
|
705
|
+
state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,size,path,layout,dataset) VALUES(?,?,?,?,?,?,?)", (key, obj.get("etag"), "", written_size, target, "v1_symbol", dataset))
|
|
697
706
|
downloaded.append(key)
|
|
698
707
|
state.commit()
|
|
699
708
|
return {"dataset": dataset, "layout": "v1_symbol", "downloaded": downloaded, "after_release": cursor}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: quantdb-sdk
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: QuantDB 量化数据平台官方 Python SDK
|
|
5
5
|
Author: QuantDB Team
|
|
6
6
|
License: MIT
|
|
@@ -104,6 +104,11 @@ history = client.download_file("1", "daily_forward", symbol="600519.SH", layout=
|
|
|
104
104
|
|
|
105
105
|
# 以发布清单为 cursor 做原子化增量同步(含 V2 patch)
|
|
106
106
|
result = client.sync_dataset("daily_forward", save_dir="D:/quantdb-data")
|
|
107
|
+
|
|
108
|
+
# 财务、ETF/可转债等尚无 V2 release 的数据集自动按 V1 Manifest 增量同步,
|
|
109
|
+
# 使用 ETag + size 校验;也可从命令行执行:
|
|
110
|
+
# quantdb sync qdb_xxx balance --save-dir D:/quantdb-data
|
|
111
|
+
financial = client.sync_dataset("balance", save_dir="D:/quantdb-data")
|
|
107
112
|
```
|
|
108
113
|
|
|
109
114
|
## 流量说明
|
|
@@ -55,7 +55,7 @@ async def test_a_query_kline_without_range_forces_v1_layout():
|
|
|
55
55
|
async def test_a_sync_dataset_uses_release_cursor(tmp_path):
|
|
56
56
|
payload = b"release-parquet"
|
|
57
57
|
digest = __import__("hashlib").sha256(payload).hexdigest()
|
|
58
|
-
respx.get(f"{API_HOST}/api/v1/data/releases").mock(return_value=httpx.Response(200, json={"releases": [{"release_id": "20260726_180000", "objects": [{"key": "1_kline_data/daily_forward/dt=20260726/data.parquet", "etag": "etag-1", "sha256": digest}]}]}))
|
|
58
|
+
respx.get(f"{API_HOST}/api/v1/data/releases").mock(return_value=httpx.Response(200, json={"releases": [{"release_id": "20260726_180000", "objects": [{"key": "v2/1_kline_data/daily_forward/dt=20260726/data.parquet", "etag": "etag-1", "sha256": digest}]}]}))
|
|
59
59
|
route = respx.get(f"{API_HOST}/api/v1/data/download").mock(return_value=httpx.Response(200, content=payload))
|
|
60
60
|
async with AsyncQuantDBClient(api_host=API_HOST, api_key="test-key") as client:
|
|
61
61
|
result = await client.a_sync_dataset("daily_forward", str(tmp_path))
|
|
@@ -64,6 +64,19 @@ async def test_a_sync_dataset_uses_release_cursor(tmp_path):
|
|
|
64
64
|
assert parse_qs(urlparse(str(route.calls[0].request.url)).query)["object_key"] == ["1_kline_data/daily_forward/dt=20260726/data.parquet"]
|
|
65
65
|
|
|
66
66
|
|
|
67
|
+
@pytest.mark.asyncio
|
|
68
|
+
@respx.mock
|
|
69
|
+
async def test_a_sync_financial_v1_validates_manifest_size(tmp_path):
|
|
70
|
+
payload = b"financial-parquet"
|
|
71
|
+
respx.get(f"{API_HOST}/api/v1/data/releases").mock(return_value=httpx.Response(200, json={"releases": []}))
|
|
72
|
+
respx.get(f"{API_HOST}/api/v1/data/download/manifest").mock(return_value=httpx.Response(200, json={"files": [{"key": "3_financial_data/balance/600000.SH.parquet", "relative_path": "3_financial_data/balance/600000.SH.parquet", "symbol": "600000.SH", "etag": "v1-etag", "size": len(payload)}]}))
|
|
73
|
+
respx.get(f"{API_HOST}/api/v1/data/download").mock(return_value=httpx.Response(200, content=payload))
|
|
74
|
+
async with AsyncQuantDBClient(api_host=API_HOST, api_key="test-key") as client:
|
|
75
|
+
result = await client.a_sync_dataset("balance", str(tmp_path))
|
|
76
|
+
assert result["layout"] == "v1_symbol"
|
|
77
|
+
assert (tmp_path / "3_financial_data" / "balance" / "600000.SH.parquet").read_bytes() == payload
|
|
78
|
+
|
|
79
|
+
|
|
67
80
|
@pytest.mark.asyncio
|
|
68
81
|
@respx.mock
|
|
69
82
|
async def test_a_auth_error_raises_auth_error():
|
|
@@ -8,6 +8,7 @@ import pytest
|
|
|
8
8
|
import responses
|
|
9
9
|
|
|
10
10
|
from quantdb_sdk import AuthError, InsufficientTrafficError, QuantDBClient
|
|
11
|
+
from quantdb_sdk.errors import ServerError
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
API_HOST = "http://localhost:5000"
|
|
@@ -79,7 +80,7 @@ def test_sync_dataset_uses_release_cursor_and_object_key(tmp_path):
|
|
|
79
80
|
digest = __import__("hashlib").sha256(payload).hexdigest()
|
|
80
81
|
responses.get(
|
|
81
82
|
f"{API_HOST}/api/v1/data/releases",
|
|
82
|
-
json={"releases": [{"release_id": "20260726_180000", "objects": [{"key": "1_kline_data/daily_forward/dt=20260726/data.parquet", "etag": "etag-1", "sha256": digest}]}]}, status=200,
|
|
83
|
+
json={"releases": [{"release_id": "20260726_180000", "objects": [{"key": "v2/1_kline_data/daily_forward/dt=20260726/data.parquet", "etag": "etag-1", "sha256": digest}]}]}, status=200,
|
|
83
84
|
)
|
|
84
85
|
responses.get(f"{API_HOST}/api/v1/data/download", body=payload, status=200)
|
|
85
86
|
client = QuantDBClient(api_host=API_HOST, api_key="test-key")
|
|
@@ -90,6 +91,35 @@ def test_sync_dataset_uses_release_cursor_and_object_key(tmp_path):
|
|
|
90
91
|
assert params["object_key"] == ["1_kline_data/daily_forward/dt=20260726/data.parquet"]
|
|
91
92
|
|
|
92
93
|
|
|
94
|
+
@responses.activate
|
|
95
|
+
def test_sync_financial_v1_validates_manifest_size(tmp_path):
|
|
96
|
+
payload = b"financial-parquet"
|
|
97
|
+
responses.get(f"{API_HOST}/api/v1/data/releases", json={"releases": []}, status=200)
|
|
98
|
+
responses.get(
|
|
99
|
+
f"{API_HOST}/api/v1/data/download/manifest",
|
|
100
|
+
json={"files": [{"key": "3_financial_data/balance/600000.SH.parquet", "relative_path": "3_financial_data/balance/600000.SH.parquet", "symbol": "600000.SH", "etag": "v1-etag", "size": len(payload)}]},
|
|
101
|
+
status=200,
|
|
102
|
+
)
|
|
103
|
+
responses.get(f"{API_HOST}/api/v1/data/download", body=payload, status=200)
|
|
104
|
+
result = QuantDBClient(api_host=API_HOST, api_key="test-key").sync_dataset("balance", str(tmp_path))
|
|
105
|
+
assert result["layout"] == "v1_symbol"
|
|
106
|
+
assert (tmp_path / "3_financial_data" / "balance" / "600000.SH.parquet").read_bytes() == payload
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@responses.activate
|
|
110
|
+
def test_sync_v1_size_failure_does_not_keep_partial_file(tmp_path):
|
|
111
|
+
responses.get(f"{API_HOST}/api/v1/data/releases", json={"releases": []}, status=200)
|
|
112
|
+
responses.get(
|
|
113
|
+
f"{API_HOST}/api/v1/data/download/manifest",
|
|
114
|
+
json={"files": [{"key": "3_financial_data/balance/600000.SH.parquet", "symbol": "600000.SH", "etag": "v1-etag", "size": 999}]},
|
|
115
|
+
status=200,
|
|
116
|
+
)
|
|
117
|
+
responses.get(f"{API_HOST}/api/v1/data/download", body=b"short", status=200)
|
|
118
|
+
with pytest.raises(ServerError):
|
|
119
|
+
QuantDBClient(api_host=API_HOST, api_key="test-key").sync_dataset("balance", str(tmp_path))
|
|
120
|
+
assert not list(tmp_path.rglob("*.part"))
|
|
121
|
+
|
|
122
|
+
|
|
93
123
|
@responses.activate
|
|
94
124
|
def test_auth_error_raises_auth_error():
|
|
95
125
|
responses.get(
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|