quantdb-sdk 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/CHANGELOG.md +10 -0
- {quantdb_sdk-0.2.1/quantdb_sdk.egg-info → quantdb_sdk-0.2.2}/PKG-INFO +6 -1
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/README.md +5 -0
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/pyproject.toml +1 -1
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk/__init__.py +1 -1
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk/__main__.py +14 -0
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk/_utils.py +16 -0
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk/async_client.py +34 -22
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk/client.py +23 -19
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2/quantdb_sdk.egg-info}/PKG-INFO +6 -1
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/tests/test_async_client.py +13 -0
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/tests/test_client.py +30 -0
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/LICENSE +0 -0
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/MANIFEST.in +0 -0
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk/errors.py +0 -0
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk/py.typed +0 -0
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/SOURCES.txt +0 -0
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/dependency_links.txt +0 -0
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/entry_points.txt +0 -0
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/requires.txt +0 -0
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/top_level.txt +0 -0
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/setup.cfg +0 -0
- {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/tests/test_technical_indicators.py +0 -0
|
@@ -3,6 +3,16 @@
|
|
|
3
3
|
所有 notable 变更都会记录在此文件。格式基于 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.1.0/),
|
|
4
4
|
版本号遵循 [Semantic Versioning](https://semver.org/lang/zh-CN/)。
|
|
5
5
|
|
|
6
|
+
## [0.2.2] - 2026-07-26
|
|
7
|
+
|
|
8
|
+
### Added
|
|
9
|
+
- **全数据集增量同步**:`sync_dataset()` / `a_sync_dataset()` 覆盖基础、财务、ETF/可转债、行情、技术与 ML 子类;没有 V2 release 的数据集自动走 V1 Manifest。
|
|
10
|
+
- **CLI 同步**:新增 `quantdb sync <api_key> <dataset> --save-dir <目录>`。
|
|
11
|
+
|
|
12
|
+
### Fixed
|
|
13
|
+
- **V1 强校验**:V1 Manifest 同步以 ETag + size 判断变更,并在原子落盘前验证实际字节数。
|
|
14
|
+
- **同步/异步一致性**:异步 release 同步采用 `relative_path`,SQLite 同时记录对象大小,与同步客户端保持同构目录和恢复语义。
|
|
15
|
+
|
|
6
16
|
## [0.2.1] - 2026-07-26
|
|
7
17
|
|
|
8
18
|
### Fixed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: quantdb-sdk
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: QuantDB 量化数据平台官方 Python SDK
|
|
5
5
|
Author: QuantDB Team
|
|
6
6
|
License: MIT
|
|
@@ -104,6 +104,11 @@ history = client.download_file("1", "daily_forward", symbol="600519.SH", layout=
|
|
|
104
104
|
|
|
105
105
|
# 以发布清单为 cursor 做原子化增量同步(含 V2 patch)
|
|
106
106
|
result = client.sync_dataset("daily_forward", save_dir="D:/quantdb-data")
|
|
107
|
+
|
|
108
|
+
# 财务、ETF/可转债等尚无 V2 release 的数据集自动按 V1 Manifest 增量同步,
|
|
109
|
+
# 使用 ETag + size 校验;也可从命令行执行:
|
|
110
|
+
# quantdb sync qdb_xxx balance --save-dir D:/quantdb-data
|
|
111
|
+
financial = client.sync_dataset("balance", save_dir="D:/quantdb-data")
|
|
107
112
|
```
|
|
108
113
|
|
|
109
114
|
## 流量说明
|
|
@@ -67,6 +67,11 @@ history = client.download_file("1", "daily_forward", symbol="600519.SH", layout=
|
|
|
67
67
|
|
|
68
68
|
# 以发布清单为 cursor 做原子化增量同步(含 V2 patch)
|
|
69
69
|
result = client.sync_dataset("daily_forward", save_dir="D:/quantdb-data")
|
|
70
|
+
|
|
71
|
+
# 财务、ETF/可转债等尚无 V2 release 的数据集自动按 V1 Manifest 增量同步,
|
|
72
|
+
# 使用 ETag + size 校验;也可从命令行执行:
|
|
73
|
+
# quantdb sync qdb_xxx balance --save-dir D:/quantdb-data
|
|
74
|
+
financial = client.sync_dataset("balance", save_dir="D:/quantdb-data")
|
|
70
75
|
```
|
|
71
76
|
|
|
72
77
|
## 流量说明
|
|
@@ -39,6 +39,12 @@ def main() -> int:
|
|
|
39
39
|
usage_parser = subparsers.add_parser("usage", help="查询账户用量和订阅状态")
|
|
40
40
|
usage_parser.add_argument("api_key", help="API Key")
|
|
41
41
|
|
|
42
|
+
sync_parser = subparsers.add_parser("sync", help="以 release/Manifest 增量同步一个数据集")
|
|
43
|
+
sync_parser.add_argument("api_key", help="API Key")
|
|
44
|
+
sync_parser.add_argument("dataset", help="数据集,如 daily_forward、balance、etf_pcf")
|
|
45
|
+
sync_parser.add_argument("--save-dir", default=None, help="本地同步根目录")
|
|
46
|
+
sync_parser.add_argument("--after-release", default=None, help="覆盖本地 release cursor(一般无需传入)")
|
|
47
|
+
|
|
42
48
|
args = parser.parse_args()
|
|
43
49
|
|
|
44
50
|
if not args.command:
|
|
@@ -66,6 +72,14 @@ def main() -> int:
|
|
|
66
72
|
print(f" 订阅: {sub.get('status', 'none')}")
|
|
67
73
|
return 0
|
|
68
74
|
|
|
75
|
+
if args.command == "sync":
|
|
76
|
+
result = client.sync_dataset(args.dataset, save_dir=args.save_dir, after_release=args.after_release)
|
|
77
|
+
print(f"同步完成: {result['dataset']} ({result['layout']})")
|
|
78
|
+
print(f" 新增/更新对象: {len(result['downloaded'])}")
|
|
79
|
+
if result.get("release_id"):
|
|
80
|
+
print(f" Release: {result['release_id']}")
|
|
81
|
+
return 0
|
|
82
|
+
|
|
69
83
|
except Exception as e:
|
|
70
84
|
print(f"✗ 错误: {e}", file=sys.stderr)
|
|
71
85
|
return 1
|
|
@@ -5,9 +5,25 @@
|
|
|
5
5
|
|
|
6
6
|
import os
|
|
7
7
|
import re
|
|
8
|
+
from typing import Dict
|
|
8
9
|
from urllib.parse import unquote
|
|
9
10
|
|
|
10
11
|
|
|
12
|
+
# ``sync_dataset`` 的统一数据集目录。V2 可用时走 release cursor;其余数据集
|
|
13
|
+
# 自动使用 V1 Manifest 的 ETag + size 增量同步。
|
|
14
|
+
SYNC_DATASET_CATEGORIES: Dict[str, str] = {
|
|
15
|
+
"daily_unadjusted": "1", "daily_forward": "1", "daily_backward": "1",
|
|
16
|
+
"index_daily": "1", "min1_kline": "1", "min5_kline": "1", "tick_data": "1",
|
|
17
|
+
"instrument_detail": "2", "sector_concept": "2", "index_weights": "2",
|
|
18
|
+
"trading_calendar": "2", "margin_trading": "2",
|
|
19
|
+
"balance": "3", "income": "3", "cashflow": "3", "capital": "3",
|
|
20
|
+
"holder_num": "3", "pershare_index": "3", "dividend_factors": "3",
|
|
21
|
+
"etf_pcf": "4", "convertible_bond": "4",
|
|
22
|
+
"valuation": "5", "technical_indicators": "5", "market_sentiment": "5",
|
|
23
|
+
"features_daily": "6", "l1_l2_factors": "6", "l2_factors": "6",
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
|
|
11
27
|
def default_download_dir() -> str:
|
|
12
28
|
"""返回默认下载目录。
|
|
13
29
|
|
|
@@ -10,7 +10,7 @@ from typing import Any, Dict, List, Optional, Literal
|
|
|
10
10
|
import httpx
|
|
11
11
|
import pandas as pd
|
|
12
12
|
|
|
13
|
-
from ._utils import bytes_to_gb, default_download_dir, parse_filename_from_content_disposition
|
|
13
|
+
from ._utils import SYNC_DATASET_CATEGORIES, bytes_to_gb, default_download_dir, parse_filename_from_content_disposition
|
|
14
14
|
from .errors import (
|
|
15
15
|
AuthError,
|
|
16
16
|
InsufficientTrafficError,
|
|
@@ -37,7 +37,7 @@ class AsyncQuantDBClient:
|
|
|
37
37
|
):
|
|
38
38
|
self.api_host = api_host.rstrip("/")
|
|
39
39
|
self.timeout = timeout
|
|
40
|
-
headers = {"User-Agent": "QuantDB-Python-SDK/0.2.
|
|
40
|
+
headers = {"User-Agent": "QuantDB-Python-SDK/0.2.2"}
|
|
41
41
|
if api_key:
|
|
42
42
|
headers["X-API-Key"] = api_key
|
|
43
43
|
elif token:
|
|
@@ -528,17 +528,13 @@ class AsyncQuantDBClient:
|
|
|
528
528
|
|
|
529
529
|
async def a_sync_dataset(self, dataset: str, save_dir: Optional[str] = None, after_release: Optional[str] = None) -> Dict[str, Any]:
|
|
530
530
|
"""异步版 release 增量同步;状态格式与 ``sync_dataset`` 兼容。"""
|
|
531
|
-
|
|
532
|
-
"daily_unadjusted": "1", "daily_forward": "1", "daily_backward": "1", "index_daily": "1",
|
|
533
|
-
"min1_kline": "1", "min5_kline": "1", "margin_trading": "2", "valuation": "5",
|
|
534
|
-
"technical_indicators": "5", "market_sentiment": "5", "features_daily": "6",
|
|
535
|
-
}
|
|
536
|
-
if dataset not in category_map: raise ValidationError(f"不支持同步的数据集: {dataset}")
|
|
531
|
+
if dataset not in SYNC_DATASET_CATEGORIES: raise ValidationError(f"不支持同步的数据集: {dataset}")
|
|
537
532
|
root = os.path.abspath(save_dir or default_download_dir()); os.makedirs(root, exist_ok=True)
|
|
538
533
|
state = sqlite3.connect(os.path.join(root, "quantdb_sync.sqlite"))
|
|
539
|
-
state.execute("CREATE TABLE IF NOT EXISTS objects (key TEXT PRIMARY KEY, etag TEXT, sha256 TEXT, path TEXT, layout TEXT, dataset TEXT)")
|
|
534
|
+
state.execute("CREATE TABLE IF NOT EXISTS objects (key TEXT PRIMARY KEY, etag TEXT, sha256 TEXT, size INTEGER, path TEXT, layout TEXT, dataset TEXT)")
|
|
540
535
|
cols = {row[1] for row in state.execute("PRAGMA table_info(objects)")}
|
|
541
536
|
if "dataset" not in cols: state.execute("ALTER TABLE objects ADD COLUMN dataset TEXT")
|
|
537
|
+
if "size" not in cols: state.execute("ALTER TABLE objects ADD COLUMN size INTEGER")
|
|
542
538
|
state.execute("CREATE TABLE IF NOT EXISTS releases (dataset TEXT PRIMARY KEY, release_id TEXT NOT NULL)")
|
|
543
539
|
downloaded: List[str] = []
|
|
544
540
|
try:
|
|
@@ -549,12 +545,14 @@ class AsyncQuantDBClient:
|
|
|
549
545
|
for release in releases:
|
|
550
546
|
for obj in release.get("objects", []):
|
|
551
547
|
key = self._normalise_release_key(obj["key"])
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
548
|
+
relative_path = obj.get("relative_path") or key
|
|
549
|
+
target = os.path.join(root, *relative_path.split("/"))
|
|
550
|
+
expected_size = obj.get("size")
|
|
551
|
+
old = state.execute("SELECT etag,sha256,size,path FROM objects WHERE key=?", (key,)).fetchone()
|
|
552
|
+
if old and old[0] == obj.get("etag") and old[1] == obj.get("sha256") and os.path.exists(old[3]) and (expected_size is None or os.path.getsize(old[3]) == expected_size): continue
|
|
555
553
|
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
556
554
|
tmp, digest = target + ".part", hashlib.sha256()
|
|
557
|
-
async with self.client.stream("GET", f"{self.api_host}/api/v1/data/download", params={"category_id":
|
|
555
|
+
async with self.client.stream("GET", f"{self.api_host}/api/v1/data/download", params={"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v2", "object_key": key}) as resp:
|
|
558
556
|
if resp.status_code != 200:
|
|
559
557
|
body = await resp.aread(); self._check_response(httpx.Response(resp.status_code, content=body)); raise QuantDBError("下载失败")
|
|
560
558
|
try:
|
|
@@ -563,24 +561,38 @@ class AsyncQuantDBClient:
|
|
|
563
561
|
if chunk: fh.write(chunk); digest.update(chunk)
|
|
564
562
|
actual = digest.hexdigest()
|
|
565
563
|
if obj.get("sha256") and actual.lower() != obj["sha256"].lower(): raise ServerError("对象 SHA-256 校验失败")
|
|
564
|
+
written_size = os.path.getsize(tmp)
|
|
565
|
+
if expected_size is not None and written_size != expected_size: raise ServerError(f"对象大小校验失败:期望 {expected_size},实际 {written_size}")
|
|
566
566
|
os.replace(tmp, target)
|
|
567
567
|
except Exception:
|
|
568
568
|
if os.path.exists(tmp): os.remove(tmp)
|
|
569
569
|
raise
|
|
570
|
-
state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,path,layout,dataset) VALUES(
|
|
570
|
+
state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,size,path,layout,dataset) VALUES(?,?,?,?,?,?,?)", (key, obj.get("etag"), actual, written_size, target, "v2_daily_partition", dataset)); downloaded.append(key)
|
|
571
571
|
state.execute("INSERT OR REPLACE INTO releases(dataset,release_id) VALUES(?,?)", (dataset, release["release_id"])); state.commit()
|
|
572
572
|
return {"dataset": dataset, "layout": "v2_daily_partition", "downloaded": downloaded, "after_release": cursor, "release_id": releases[-1]["release_id"]}
|
|
573
573
|
if persisted: return {"dataset": dataset, "layout": "v2_daily_partition", "downloaded": [], "after_release": cursor, "release_id": persisted[0]}
|
|
574
|
-
files = (await self._get("/api/v1/data/download/manifest", {"category_id":
|
|
574
|
+
files = (await self._get("/api/v1/data/download/manifest", {"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1"})).get("files", [])
|
|
575
575
|
for obj in files:
|
|
576
576
|
key, target = obj["key"], os.path.join(root, *(obj.get("relative_path") or obj["key"]).split("/"))
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
577
|
+
expected_size = obj.get("size")
|
|
578
|
+
old = state.execute("SELECT etag,size,path FROM objects WHERE key=?", (key,)).fetchone()
|
|
579
|
+
if old and old[0] == obj.get("etag") and os.path.exists(old[2]) and (expected_size is None or os.path.getsize(old[2]) == expected_size): continue
|
|
580
|
+
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
581
|
+
tmp, written_size = target + ".part", 0
|
|
582
|
+
try:
|
|
583
|
+
async with self.client.stream("GET", f"{self.api_host}/api/v1/data/download", params={"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1", "symbol": obj.get("symbol", "")}) as resp:
|
|
584
|
+
if resp.status_code != 200:
|
|
585
|
+
body = await resp.aread(); self._check_response(httpx.Response(resp.status_code, content=body)); raise QuantDBError("下载失败")
|
|
586
|
+
with open(tmp, "wb") as fh:
|
|
587
|
+
async for chunk in resp.aiter_bytes(1024 * 1024):
|
|
588
|
+
if chunk: fh.write(chunk)
|
|
589
|
+
written_size = os.path.getsize(tmp)
|
|
590
|
+
if expected_size is not None and written_size != expected_size: raise ServerError(f"对象大小校验失败:期望 {expected_size},实际 {written_size}")
|
|
591
|
+
os.replace(tmp, target)
|
|
592
|
+
except Exception:
|
|
593
|
+
if os.path.exists(tmp): os.remove(tmp)
|
|
594
|
+
raise
|
|
595
|
+
state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,size,path,layout,dataset) VALUES(?,?,?,?,?,?,?)", (key, obj.get("etag"), "", written_size, target, "v1_symbol", dataset)); downloaded.append(key)
|
|
584
596
|
state.commit(); return {"dataset": dataset, "layout": "v1_symbol", "downloaded": downloaded, "after_release": cursor}
|
|
585
597
|
finally:
|
|
586
598
|
state.close()
|
|
@@ -13,7 +13,7 @@ import requests
|
|
|
13
13
|
from requests.adapters import HTTPAdapter
|
|
14
14
|
from urllib3.util.retry import Retry
|
|
15
15
|
|
|
16
|
-
from ._utils import bytes_to_gb, default_download_dir, parse_filename_from_content_disposition
|
|
16
|
+
from ._utils import SYNC_DATASET_CATEGORIES, bytes_to_gb, default_download_dir, parse_filename_from_content_disposition
|
|
17
17
|
from .errors import (
|
|
18
18
|
AuthError,
|
|
19
19
|
InsufficientTrafficError,
|
|
@@ -48,7 +48,7 @@ class QuantDBClient:
|
|
|
48
48
|
self.session = requests.Session()
|
|
49
49
|
# User-Agent 中的版本与 pyproject.toml 同步,用于服务端日志归因
|
|
50
50
|
# 维护提示:每次版本号变化必须同步改这里(init 里的 __version__ 走 metadata 自动同步)
|
|
51
|
-
self.session.headers.update({"User-Agent": "QuantDB-Python-SDK/0.2.
|
|
51
|
+
self.session.headers.update({"User-Agent": "QuantDB-Python-SDK/0.2.2"})
|
|
52
52
|
|
|
53
53
|
if api_key:
|
|
54
54
|
self.headers = {"X-API-Key": api_key}
|
|
@@ -622,18 +622,14 @@ class QuantDBClient:
|
|
|
622
622
|
root = os.path.abspath(save_dir or default_download_dir())
|
|
623
623
|
os.makedirs(root, exist_ok=True)
|
|
624
624
|
state = sqlite3.connect(os.path.join(root, "quantdb_sync.sqlite"))
|
|
625
|
-
state.execute("CREATE TABLE IF NOT EXISTS objects (key TEXT PRIMARY KEY, etag TEXT, sha256 TEXT, path TEXT, layout TEXT, dataset TEXT)")
|
|
625
|
+
state.execute("CREATE TABLE IF NOT EXISTS objects (key TEXT PRIMARY KEY, etag TEXT, sha256 TEXT, size INTEGER, path TEXT, layout TEXT, dataset TEXT)")
|
|
626
626
|
cols = {row[1] for row in state.execute("PRAGMA table_info(objects)")}
|
|
627
627
|
if "dataset" not in cols:
|
|
628
628
|
state.execute("ALTER TABLE objects ADD COLUMN dataset TEXT")
|
|
629
|
+
if "size" not in cols:
|
|
630
|
+
state.execute("ALTER TABLE objects ADD COLUMN size INTEGER")
|
|
629
631
|
state.execute("CREATE TABLE IF NOT EXISTS releases (dataset TEXT PRIMARY KEY, release_id TEXT NOT NULL)")
|
|
630
|
-
|
|
631
|
-
"daily_unadjusted": "1", "daily_forward": "1", "daily_backward": "1",
|
|
632
|
-
"index_daily": "1", "min1_kline": "1", "min5_kline": "1",
|
|
633
|
-
"margin_trading": "2", "valuation": "5", "technical_indicators": "5",
|
|
634
|
-
"market_sentiment": "5", "features_daily": "6",
|
|
635
|
-
}
|
|
636
|
-
if dataset not in category_map:
|
|
632
|
+
if dataset not in SYNC_DATASET_CATEGORIES:
|
|
637
633
|
raise ValidationError(f"不支持同步的数据集: {dataset}")
|
|
638
634
|
|
|
639
635
|
downloaded: List[str] = []
|
|
@@ -649,10 +645,11 @@ class QuantDBClient:
|
|
|
649
645
|
key = self._normalise_release_key(obj["key"])
|
|
650
646
|
relative_path = obj.get("relative_path") or key
|
|
651
647
|
target = os.path.join(root, *relative_path.split("/"))
|
|
652
|
-
|
|
653
|
-
|
|
648
|
+
expected_size = obj.get("size")
|
|
649
|
+
old = state.execute("SELECT etag, sha256, size, path FROM objects WHERE key=?", (key,)).fetchone()
|
|
650
|
+
if old and old[0] == obj.get("etag") and old[1] == obj.get("sha256") and os.path.exists(old[3]) and (expected_size is None or os.path.getsize(old[3]) == expected_size):
|
|
654
651
|
continue
|
|
655
|
-
resp = self._request("GET", "/api/v1/data/download", params={"category_id":
|
|
652
|
+
resp = self._request("GET", "/api/v1/data/download", params={"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v2", "object_key": key}, stream=True)
|
|
656
653
|
if resp.status_code != 200:
|
|
657
654
|
self._check_response(resp)
|
|
658
655
|
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
@@ -663,13 +660,16 @@ class QuantDBClient:
|
|
|
663
660
|
if chunk:
|
|
664
661
|
fh.write(chunk); digest.update(chunk)
|
|
665
662
|
actual = digest.hexdigest()
|
|
663
|
+
written_size = os.path.getsize(tmp)
|
|
666
664
|
if obj.get("sha256") and actual.lower() != obj["sha256"].lower():
|
|
667
665
|
raise ServerError("对象 SHA-256 校验失败")
|
|
666
|
+
if expected_size is not None and written_size != expected_size:
|
|
667
|
+
raise ServerError(f"对象大小校验失败:期望 {expected_size},实际 {written_size}")
|
|
668
668
|
os.replace(tmp, target)
|
|
669
669
|
except Exception:
|
|
670
670
|
if os.path.exists(tmp): os.remove(tmp)
|
|
671
671
|
raise
|
|
672
|
-
state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,path,layout,dataset) VALUES(
|
|
672
|
+
state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,size,path,layout,dataset) VALUES(?,?,?,?,?,?,?)", (key, obj.get("etag"), actual, written_size, target, "v2_daily_partition", dataset))
|
|
673
673
|
downloaded.append(key)
|
|
674
674
|
# 仅在该 release 的所有对象成功落盘和校验后提交 cursor。
|
|
675
675
|
state.execute("INSERT OR REPLACE INTO releases(dataset,release_id) VALUES(?,?)", (dataset, release_id))
|
|
@@ -679,14 +679,15 @@ class QuantDBClient:
|
|
|
679
679
|
if persisted:
|
|
680
680
|
return {"dataset": dataset, "layout": "v2_daily_partition", "downloaded": [], "after_release": cursor, "release_id": persisted[0]}
|
|
681
681
|
|
|
682
|
-
manifest = self._get("/api/v1/data/download/manifest", {"category_id":
|
|
682
|
+
manifest = self._get("/api/v1/data/download/manifest", {"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1"})
|
|
683
683
|
for obj in manifest.get("files", []):
|
|
684
684
|
key, relative_path = obj["key"], obj.get("relative_path") or obj["key"]
|
|
685
685
|
target = os.path.join(root, *relative_path.split("/"))
|
|
686
|
-
|
|
687
|
-
|
|
686
|
+
expected_size = obj.get("size")
|
|
687
|
+
old = state.execute("SELECT etag, size, path FROM objects WHERE key=?", (key,)).fetchone()
|
|
688
|
+
if old and old[0] == obj.get("etag") and os.path.exists(old[2]) and (expected_size is None or os.path.getsize(old[2]) == expected_size):
|
|
688
689
|
continue
|
|
689
|
-
resp = self._request("GET", "/api/v1/data/download", params={"category_id":
|
|
690
|
+
resp = self._request("GET", "/api/v1/data/download", params={"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1", "symbol": obj.get("symbol", "")}, stream=True)
|
|
690
691
|
if resp.status_code != 200: self._check_response(resp)
|
|
691
692
|
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
692
693
|
tmp = target + ".part"
|
|
@@ -694,11 +695,14 @@ class QuantDBClient:
|
|
|
694
695
|
with open(tmp, "wb") as fh:
|
|
695
696
|
for chunk in resp.iter_content(1024 * 1024):
|
|
696
697
|
if chunk: fh.write(chunk)
|
|
698
|
+
written_size = os.path.getsize(tmp)
|
|
699
|
+
if expected_size is not None and written_size != expected_size:
|
|
700
|
+
raise ServerError(f"对象大小校验失败:期望 {expected_size},实际 {written_size}")
|
|
697
701
|
os.replace(tmp, target)
|
|
698
702
|
except Exception:
|
|
699
703
|
if os.path.exists(tmp): os.remove(tmp)
|
|
700
704
|
raise
|
|
701
|
-
state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,path,layout,dataset) VALUES(
|
|
705
|
+
state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,size,path,layout,dataset) VALUES(?,?,?,?,?,?,?)", (key, obj.get("etag"), "", written_size, target, "v1_symbol", dataset))
|
|
702
706
|
downloaded.append(key)
|
|
703
707
|
state.commit()
|
|
704
708
|
return {"dataset": dataset, "layout": "v1_symbol", "downloaded": downloaded, "after_release": cursor}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: quantdb-sdk
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: QuantDB 量化数据平台官方 Python SDK
|
|
5
5
|
Author: QuantDB Team
|
|
6
6
|
License: MIT
|
|
@@ -104,6 +104,11 @@ history = client.download_file("1", "daily_forward", symbol="600519.SH", layout=
|
|
|
104
104
|
|
|
105
105
|
# 以发布清单为 cursor 做原子化增量同步(含 V2 patch)
|
|
106
106
|
result = client.sync_dataset("daily_forward", save_dir="D:/quantdb-data")
|
|
107
|
+
|
|
108
|
+
# 财务、ETF/可转债等尚无 V2 release 的数据集自动按 V1 Manifest 增量同步,
|
|
109
|
+
# 使用 ETag + size 校验;也可从命令行执行:
|
|
110
|
+
# quantdb sync qdb_xxx balance --save-dir D:/quantdb-data
|
|
111
|
+
financial = client.sync_dataset("balance", save_dir="D:/quantdb-data")
|
|
107
112
|
```
|
|
108
113
|
|
|
109
114
|
## 流量说明
|
|
@@ -64,6 +64,19 @@ async def test_a_sync_dataset_uses_release_cursor(tmp_path):
|
|
|
64
64
|
assert parse_qs(urlparse(str(route.calls[0].request.url)).query)["object_key"] == ["1_kline_data/daily_forward/dt=20260726/data.parquet"]
|
|
65
65
|
|
|
66
66
|
|
|
67
|
+
@pytest.mark.asyncio
|
|
68
|
+
@respx.mock
|
|
69
|
+
async def test_a_sync_financial_v1_validates_manifest_size(tmp_path):
|
|
70
|
+
payload = b"financial-parquet"
|
|
71
|
+
respx.get(f"{API_HOST}/api/v1/data/releases").mock(return_value=httpx.Response(200, json={"releases": []}))
|
|
72
|
+
respx.get(f"{API_HOST}/api/v1/data/download/manifest").mock(return_value=httpx.Response(200, json={"files": [{"key": "3_financial_data/balance/600000.SH.parquet", "relative_path": "3_financial_data/balance/600000.SH.parquet", "symbol": "600000.SH", "etag": "v1-etag", "size": len(payload)}]}))
|
|
73
|
+
respx.get(f"{API_HOST}/api/v1/data/download").mock(return_value=httpx.Response(200, content=payload))
|
|
74
|
+
async with AsyncQuantDBClient(api_host=API_HOST, api_key="test-key") as client:
|
|
75
|
+
result = await client.a_sync_dataset("balance", str(tmp_path))
|
|
76
|
+
assert result["layout"] == "v1_symbol"
|
|
77
|
+
assert (tmp_path / "3_financial_data" / "balance" / "600000.SH.parquet").read_bytes() == payload
|
|
78
|
+
|
|
79
|
+
|
|
67
80
|
@pytest.mark.asyncio
|
|
68
81
|
@respx.mock
|
|
69
82
|
async def test_a_auth_error_raises_auth_error():
|
|
@@ -8,6 +8,7 @@ import pytest
|
|
|
8
8
|
import responses
|
|
9
9
|
|
|
10
10
|
from quantdb_sdk import AuthError, InsufficientTrafficError, QuantDBClient
|
|
11
|
+
from quantdb_sdk.errors import ServerError
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
API_HOST = "http://localhost:5000"
|
|
@@ -90,6 +91,35 @@ def test_sync_dataset_uses_release_cursor_and_object_key(tmp_path):
|
|
|
90
91
|
assert params["object_key"] == ["1_kline_data/daily_forward/dt=20260726/data.parquet"]
|
|
91
92
|
|
|
92
93
|
|
|
94
|
+
@responses.activate
|
|
95
|
+
def test_sync_financial_v1_validates_manifest_size(tmp_path):
|
|
96
|
+
payload = b"financial-parquet"
|
|
97
|
+
responses.get(f"{API_HOST}/api/v1/data/releases", json={"releases": []}, status=200)
|
|
98
|
+
responses.get(
|
|
99
|
+
f"{API_HOST}/api/v1/data/download/manifest",
|
|
100
|
+
json={"files": [{"key": "3_financial_data/balance/600000.SH.parquet", "relative_path": "3_financial_data/balance/600000.SH.parquet", "symbol": "600000.SH", "etag": "v1-etag", "size": len(payload)}]},
|
|
101
|
+
status=200,
|
|
102
|
+
)
|
|
103
|
+
responses.get(f"{API_HOST}/api/v1/data/download", body=payload, status=200)
|
|
104
|
+
result = QuantDBClient(api_host=API_HOST, api_key="test-key").sync_dataset("balance", str(tmp_path))
|
|
105
|
+
assert result["layout"] == "v1_symbol"
|
|
106
|
+
assert (tmp_path / "3_financial_data" / "balance" / "600000.SH.parquet").read_bytes() == payload
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@responses.activate
|
|
110
|
+
def test_sync_v1_size_failure_does_not_keep_partial_file(tmp_path):
|
|
111
|
+
responses.get(f"{API_HOST}/api/v1/data/releases", json={"releases": []}, status=200)
|
|
112
|
+
responses.get(
|
|
113
|
+
f"{API_HOST}/api/v1/data/download/manifest",
|
|
114
|
+
json={"files": [{"key": "3_financial_data/balance/600000.SH.parquet", "symbol": "600000.SH", "etag": "v1-etag", "size": 999}]},
|
|
115
|
+
status=200,
|
|
116
|
+
)
|
|
117
|
+
responses.get(f"{API_HOST}/api/v1/data/download", body=b"short", status=200)
|
|
118
|
+
with pytest.raises(ServerError):
|
|
119
|
+
QuantDBClient(api_host=API_HOST, api_key="test-key").sync_dataset("balance", str(tmp_path))
|
|
120
|
+
assert not list(tmp_path.rglob("*.part"))
|
|
121
|
+
|
|
122
|
+
|
|
93
123
|
@responses.activate
|
|
94
124
|
def test_auth_error_raises_auth_error():
|
|
95
125
|
responses.get(
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|