quantdb-sdk 0.2.0__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/CHANGELOG.md +16 -0
  2. {quantdb_sdk-0.2.0/quantdb_sdk.egg-info → quantdb_sdk-0.2.2}/PKG-INFO +6 -1
  3. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/README.md +5 -0
  4. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/pyproject.toml +1 -1
  5. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk/__init__.py +1 -1
  6. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk/__main__.py +14 -0
  7. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk/_utils.py +16 -0
  8. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk/async_client.py +39 -22
  9. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk/client.py +29 -20
  10. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2/quantdb_sdk.egg-info}/PKG-INFO +6 -1
  11. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/tests/test_async_client.py +14 -1
  12. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/tests/test_client.py +31 -1
  13. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/LICENSE +0 -0
  14. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/MANIFEST.in +0 -0
  15. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk/errors.py +0 -0
  16. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk/py.typed +0 -0
  17. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/SOURCES.txt +0 -0
  18. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/dependency_links.txt +0 -0
  19. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/entry_points.txt +0 -0
  20. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/requires.txt +0 -0
  21. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/top_level.txt +0 -0
  22. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/setup.cfg +0 -0
  23. {quantdb_sdk-0.2.0 → quantdb_sdk-0.2.2}/tests/test_technical_indicators.py +0 -0
@@ -3,6 +3,22 @@
3
3
  所有 notable 变更都会记录在此文件。格式基于 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.1.0/),
4
4
  版本号遵循 [Semantic Versioning](https://semver.org/lang/zh-CN/)。
5
5
 
6
+ ## [0.2.2] - 2026-07-26
7
+
8
+ ### Added
9
+ - **全数据集增量同步**:`sync_dataset()` / `a_sync_dataset()` 覆盖基础、财务、ETF/可转债、行情、技术与 ML 子类;没有 V2 release 的数据集自动走 V1 Manifest。
10
+ - **CLI 同步**:新增 `quantdb sync <api_key> <dataset> --save-dir <目录>`。
11
+
12
+ ### Fixed
13
+ - **V1 强校验**:V1 Manifest 同步以 ETag + size 判断变更,并在原子落盘前验证实际字节数。
14
+ - **同步/异步一致性**:异步 release 同步采用 `relative_path`,SQLite 同时记录对象大小,与同步客户端保持同构目录和恢复语义。
15
+
16
+ ## [0.2.1] - 2026-07-26
17
+
18
+ ### Fixed
19
+ - **历史发布清单兼容**:同步客户端自动归一化旧 release manifest 遗留的 `v2/` 前缀,并可下载 release patch 对象。
20
+ - **布局回退**:服务端以 COS Manifest 为可用性权威,V2 元数据缺失时可正确回退 V1,而非将数据误判为不可下载。
21
+
6
22
  ## [0.2.0] - 2026-07-26
7
23
 
8
24
  ### Added
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: quantdb-sdk
3
- Version: 0.2.0
3
+ Version: 0.2.2
4
4
  Summary: QuantDB 量化数据平台官方 Python SDK
5
5
  Author: QuantDB Team
6
6
  License: MIT
@@ -104,6 +104,11 @@ history = client.download_file("1", "daily_forward", symbol="600519.SH", layout=
104
104
 
105
105
  # 以发布清单为 cursor 做原子化增量同步(含 V2 patch)
106
106
  result = client.sync_dataset("daily_forward", save_dir="D:/quantdb-data")
107
+
108
+ # 财务、ETF/可转债等尚无 V2 release 的数据集自动按 V1 Manifest 增量同步,
109
+ # 使用 ETag + size 校验;也可从命令行执行:
110
+ # quantdb sync qdb_xxx balance --save-dir D:/quantdb-data
111
+ financial = client.sync_dataset("balance", save_dir="D:/quantdb-data")
107
112
  ```
108
113
 
109
114
  ## 流量说明
@@ -67,6 +67,11 @@ history = client.download_file("1", "daily_forward", symbol="600519.SH", layout=
67
67
 
68
68
  # 以发布清单为 cursor 做原子化增量同步(含 V2 patch)
69
69
  result = client.sync_dataset("daily_forward", save_dir="D:/quantdb-data")
70
+
71
+ # 财务、ETF/可转债等尚无 V2 release 的数据集自动按 V1 Manifest 增量同步,
72
+ # 使用 ETag + size 校验;也可从命令行执行:
73
+ # quantdb sync qdb_xxx balance --save-dir D:/quantdb-data
74
+ financial = client.sync_dataset("balance", save_dir="D:/quantdb-data")
70
75
  ```
71
76
 
72
77
  ## 流量说明
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "quantdb-sdk"
7
- version = "0.2.0"
7
+ version = "0.2.2"
8
8
  description = "QuantDB 量化数据平台官方 Python SDK"
9
9
  readme = "README.md"
10
10
  license = {text = "MIT"}
@@ -5,7 +5,7 @@ try:
5
5
  # 注意:此处用 PyPI 包名(连字符)查找版本号,不是模块名(下划线)
6
6
  __version__ = version("quantdb-sdk")
7
7
  except ImportError:
8
- __version__ = "0.2.0"
8
+ __version__ = "0.2.2"
9
9
 
10
10
  from .async_client import AsyncQuantDBClient
11
11
  from .client import DuckDBWarehouse, QuantDBClient
@@ -39,6 +39,12 @@ def main() -> int:
39
39
  usage_parser = subparsers.add_parser("usage", help="查询账户用量和订阅状态")
40
40
  usage_parser.add_argument("api_key", help="API Key")
41
41
 
42
+ sync_parser = subparsers.add_parser("sync", help="以 release/Manifest 增量同步一个数据集")
43
+ sync_parser.add_argument("api_key", help="API Key")
44
+ sync_parser.add_argument("dataset", help="数据集,如 daily_forward、balance、etf_pcf")
45
+ sync_parser.add_argument("--save-dir", default=None, help="本地同步根目录")
46
+ sync_parser.add_argument("--after-release", default=None, help="覆盖本地 release cursor(一般无需传入)")
47
+
42
48
  args = parser.parse_args()
43
49
 
44
50
  if not args.command:
@@ -66,6 +72,14 @@ def main() -> int:
66
72
  print(f" 订阅: {sub.get('status', 'none')}")
67
73
  return 0
68
74
 
75
+ if args.command == "sync":
76
+ result = client.sync_dataset(args.dataset, save_dir=args.save_dir, after_release=args.after_release)
77
+ print(f"同步完成: {result['dataset']} ({result['layout']})")
78
+ print(f" 新增/更新对象: {len(result['downloaded'])}")
79
+ if result.get("release_id"):
80
+ print(f" Release: {result['release_id']}")
81
+ return 0
82
+
69
83
  except Exception as e:
70
84
  print(f"✗ 错误: {e}", file=sys.stderr)
71
85
  return 1
@@ -5,9 +5,25 @@
5
5
 
6
6
  import os
7
7
  import re
8
+ from typing import Dict
8
9
  from urllib.parse import unquote
9
10
 
10
11
 
12
+ # ``sync_dataset`` 的统一数据集目录。V2 可用时走 release cursor;其余数据集
13
+ # 自动使用 V1 Manifest 的 ETag + size 增量同步。
14
+ SYNC_DATASET_CATEGORIES: Dict[str, str] = {
15
+ "daily_unadjusted": "1", "daily_forward": "1", "daily_backward": "1",
16
+ "index_daily": "1", "min1_kline": "1", "min5_kline": "1", "tick_data": "1",
17
+ "instrument_detail": "2", "sector_concept": "2", "index_weights": "2",
18
+ "trading_calendar": "2", "margin_trading": "2",
19
+ "balance": "3", "income": "3", "cashflow": "3", "capital": "3",
20
+ "holder_num": "3", "pershare_index": "3", "dividend_factors": "3",
21
+ "etf_pcf": "4", "convertible_bond": "4",
22
+ "valuation": "5", "technical_indicators": "5", "market_sentiment": "5",
23
+ "features_daily": "6", "l1_l2_factors": "6", "l2_factors": "6",
24
+ }
25
+
26
+
11
27
  def default_download_dir() -> str:
12
28
  """返回默认下载目录。
13
29
 
@@ -10,7 +10,7 @@ from typing import Any, Dict, List, Optional, Literal
10
10
  import httpx
11
11
  import pandas as pd
12
12
 
13
- from ._utils import bytes_to_gb, default_download_dir, parse_filename_from_content_disposition
13
+ from ._utils import SYNC_DATASET_CATEGORIES, bytes_to_gb, default_download_dir, parse_filename_from_content_disposition
14
14
  from .errors import (
15
15
  AuthError,
16
16
  InsufficientTrafficError,
@@ -37,7 +37,7 @@ class AsyncQuantDBClient:
37
37
  ):
38
38
  self.api_host = api_host.rstrip("/")
39
39
  self.timeout = timeout
40
- headers = {"User-Agent": "QuantDB-Python-SDK/0.2.0"}
40
+ headers = {"User-Agent": "QuantDB-Python-SDK/0.2.2"}
41
41
  if api_key:
42
42
  headers["X-API-Key"] = api_key
43
43
  elif token:
@@ -66,6 +66,10 @@ class AsyncQuantDBClient:
66
66
  raise ValidationError("layout 仅支持 auto、v1 或 v2")
67
67
  return layout # type: ignore[return-value]
68
68
 
69
+ @staticmethod
70
+ def _normalise_release_key(key: str) -> str:
71
+ return key.strip().lstrip("/").removeprefix("v2/")
72
+
69
73
  @staticmethod
70
74
  def _normalise_kline(df: pd.DataFrame, start_date: Optional[str], end_date: Optional[str], fields: str, limit: Optional[int]) -> pd.DataFrame:
71
75
  if "time" in df.columns:
@@ -524,17 +528,13 @@ class AsyncQuantDBClient:
524
528
 
525
529
  async def a_sync_dataset(self, dataset: str, save_dir: Optional[str] = None, after_release: Optional[str] = None) -> Dict[str, Any]:
526
530
  """异步版 release 增量同步;状态格式与 ``sync_dataset`` 兼容。"""
527
- category_map = {
528
- "daily_unadjusted": "1", "daily_forward": "1", "daily_backward": "1", "index_daily": "1",
529
- "min1_kline": "1", "min5_kline": "1", "margin_trading": "2", "valuation": "5",
530
- "technical_indicators": "5", "market_sentiment": "5", "features_daily": "6",
531
- }
532
- if dataset not in category_map: raise ValidationError(f"不支持同步的数据集: {dataset}")
531
+ if dataset not in SYNC_DATASET_CATEGORIES: raise ValidationError(f"不支持同步的数据集: {dataset}")
533
532
  root = os.path.abspath(save_dir or default_download_dir()); os.makedirs(root, exist_ok=True)
534
533
  state = sqlite3.connect(os.path.join(root, "quantdb_sync.sqlite"))
535
- state.execute("CREATE TABLE IF NOT EXISTS objects (key TEXT PRIMARY KEY, etag TEXT, sha256 TEXT, path TEXT, layout TEXT, dataset TEXT)")
534
+ state.execute("CREATE TABLE IF NOT EXISTS objects (key TEXT PRIMARY KEY, etag TEXT, sha256 TEXT, size INTEGER, path TEXT, layout TEXT, dataset TEXT)")
536
535
  cols = {row[1] for row in state.execute("PRAGMA table_info(objects)")}
537
536
  if "dataset" not in cols: state.execute("ALTER TABLE objects ADD COLUMN dataset TEXT")
537
+ if "size" not in cols: state.execute("ALTER TABLE objects ADD COLUMN size INTEGER")
538
538
  state.execute("CREATE TABLE IF NOT EXISTS releases (dataset TEXT PRIMARY KEY, release_id TEXT NOT NULL)")
539
539
  downloaded: List[str] = []
540
540
  try:
@@ -544,12 +544,15 @@ class AsyncQuantDBClient:
544
544
  if releases:
545
545
  for release in releases:
546
546
  for obj in release.get("objects", []):
547
- key, target = obj["key"], os.path.join(root, *obj["key"].split("/"))
548
- old = state.execute("SELECT etag,sha256,path FROM objects WHERE key=?", (key,)).fetchone()
549
- if old and old[0] == obj.get("etag") and old[1] == obj.get("sha256") and os.path.exists(old[2]): continue
547
+ key = self._normalise_release_key(obj["key"])
548
+ relative_path = obj.get("relative_path") or key
549
+ target = os.path.join(root, *relative_path.split("/"))
550
+ expected_size = obj.get("size")
551
+ old = state.execute("SELECT etag,sha256,size,path FROM objects WHERE key=?", (key,)).fetchone()
552
+ if old and old[0] == obj.get("etag") and old[1] == obj.get("sha256") and os.path.exists(old[3]) and (expected_size is None or os.path.getsize(old[3]) == expected_size): continue
550
553
  os.makedirs(os.path.dirname(target), exist_ok=True)
551
554
  tmp, digest = target + ".part", hashlib.sha256()
552
- async with self.client.stream("GET", f"{self.api_host}/api/v1/data/download", params={"category_id": category_map[dataset], "sub_category": dataset, "layout": "v2", "object_key": key}) as resp:
555
+ async with self.client.stream("GET", f"{self.api_host}/api/v1/data/download", params={"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v2", "object_key": key}) as resp:
553
556
  if resp.status_code != 200:
554
557
  body = await resp.aread(); self._check_response(httpx.Response(resp.status_code, content=body)); raise QuantDBError("下载失败")
555
558
  try:
@@ -558,24 +561,38 @@ class AsyncQuantDBClient:
558
561
  if chunk: fh.write(chunk); digest.update(chunk)
559
562
  actual = digest.hexdigest()
560
563
  if obj.get("sha256") and actual.lower() != obj["sha256"].lower(): raise ServerError("对象 SHA-256 校验失败")
564
+ written_size = os.path.getsize(tmp)
565
+ if expected_size is not None and written_size != expected_size: raise ServerError(f"对象大小校验失败:期望 {expected_size},实际 {written_size}")
561
566
  os.replace(tmp, target)
562
567
  except Exception:
563
568
  if os.path.exists(tmp): os.remove(tmp)
564
569
  raise
565
- state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,path,layout,dataset) VALUES(?,?,?,?,?,?)", (key, obj.get("etag"), actual, target, "v2_daily_partition", dataset)); downloaded.append(key)
570
+ state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,size,path,layout,dataset) VALUES(?,?,?,?,?,?,?)", (key, obj.get("etag"), actual, written_size, target, "v2_daily_partition", dataset)); downloaded.append(key)
566
571
  state.execute("INSERT OR REPLACE INTO releases(dataset,release_id) VALUES(?,?)", (dataset, release["release_id"])); state.commit()
567
572
  return {"dataset": dataset, "layout": "v2_daily_partition", "downloaded": downloaded, "after_release": cursor, "release_id": releases[-1]["release_id"]}
568
573
  if persisted: return {"dataset": dataset, "layout": "v2_daily_partition", "downloaded": [], "after_release": cursor, "release_id": persisted[0]}
569
- files = (await self._get("/api/v1/data/download/manifest", {"category_id": category_map[dataset], "sub_category": dataset, "layout": "v1"})).get("files", [])
574
+ files = (await self._get("/api/v1/data/download/manifest", {"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1"})).get("files", [])
570
575
  for obj in files:
571
576
  key, target = obj["key"], os.path.join(root, *(obj.get("relative_path") or obj["key"]).split("/"))
572
- old = state.execute("SELECT etag,path FROM objects WHERE key=?", (key,)).fetchone()
573
- if old and old[0] == obj.get("etag") and os.path.exists(old[1]): continue
574
- await self.a_download_file(category_map[dataset], dataset, symbol=obj.get("symbol"), save_dir=os.path.dirname(target), layout="v1")
575
- # 下载文件名来自服务端,按清单目标路径归位以保证本地根目录同构。
576
- source = os.path.join(os.path.dirname(target), os.path.basename(target))
577
- if os.path.abspath(source) != os.path.abspath(target): os.replace(source, target)
578
- state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,path,layout,dataset) VALUES(?,?,?,?,?,?)", (key, obj.get("etag"), "", target, "v1_symbol", dataset)); downloaded.append(key)
577
+ expected_size = obj.get("size")
578
+ old = state.execute("SELECT etag,size,path FROM objects WHERE key=?", (key,)).fetchone()
579
+ if old and old[0] == obj.get("etag") and os.path.exists(old[2]) and (expected_size is None or os.path.getsize(old[2]) == expected_size): continue
580
+ os.makedirs(os.path.dirname(target), exist_ok=True)
581
+ tmp, written_size = target + ".part", 0
582
+ try:
583
+ async with self.client.stream("GET", f"{self.api_host}/api/v1/data/download", params={"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1", "symbol": obj.get("symbol", "")}) as resp:
584
+ if resp.status_code != 200:
585
+ body = await resp.aread(); self._check_response(httpx.Response(resp.status_code, content=body)); raise QuantDBError("下载失败")
586
+ with open(tmp, "wb") as fh:
587
+ async for chunk in resp.aiter_bytes(1024 * 1024):
588
+ if chunk: fh.write(chunk)
589
+ written_size = os.path.getsize(tmp)
590
+ if expected_size is not None and written_size != expected_size: raise ServerError(f"对象大小校验失败:期望 {expected_size},实际 {written_size}")
591
+ os.replace(tmp, target)
592
+ except Exception:
593
+ if os.path.exists(tmp): os.remove(tmp)
594
+ raise
595
+ state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,size,path,layout,dataset) VALUES(?,?,?,?,?,?,?)", (key, obj.get("etag"), "", written_size, target, "v1_symbol", dataset)); downloaded.append(key)
579
596
  state.commit(); return {"dataset": dataset, "layout": "v1_symbol", "downloaded": downloaded, "after_release": cursor}
580
597
  finally:
581
598
  state.close()
@@ -13,7 +13,7 @@ import requests
13
13
  from requests.adapters import HTTPAdapter
14
14
  from urllib3.util.retry import Retry
15
15
 
16
- from ._utils import bytes_to_gb, default_download_dir, parse_filename_from_content_disposition
16
+ from ._utils import SYNC_DATASET_CATEGORIES, bytes_to_gb, default_download_dir, parse_filename_from_content_disposition
17
17
  from .errors import (
18
18
  AuthError,
19
19
  InsufficientTrafficError,
@@ -48,7 +48,7 @@ class QuantDBClient:
48
48
  self.session = requests.Session()
49
49
  # User-Agent 中的版本与 pyproject.toml 同步,用于服务端日志归因
50
50
  # 维护提示:每次版本号变化必须同步改这里(init 里的 __version__ 走 metadata 自动同步)
51
- self.session.headers.update({"User-Agent": "QuantDB-Python-SDK/0.2.0"})
51
+ self.session.headers.update({"User-Agent": "QuantDB-Python-SDK/0.2.2"})
52
52
 
53
53
  if api_key:
54
54
  self.headers = {"X-API-Key": api_key}
@@ -164,6 +164,11 @@ class QuantDBClient:
164
164
  raise ValidationError("layout 仅支持 auto、v1 或 v2")
165
165
  return layout # type: ignore[return-value]
166
166
 
167
+ @staticmethod
168
+ def _normalise_release_key(key: str) -> str:
169
+ """兼容历史 release manifest 遗留的 ``v2/`` 对象前缀。"""
170
+ return key.strip().lstrip("/").removeprefix("v2/")
171
+
167
172
  @staticmethod
168
173
  def _normalise_kline(df: pd.DataFrame, start_date: Optional[str], end_date: Optional[str], fields: str, limit: Optional[int]) -> pd.DataFrame:
169
174
  """统一 V1/V2 日线结果并在客户端按标的、日期与字段过滤。"""
@@ -617,18 +622,14 @@ class QuantDBClient:
617
622
  root = os.path.abspath(save_dir or default_download_dir())
618
623
  os.makedirs(root, exist_ok=True)
619
624
  state = sqlite3.connect(os.path.join(root, "quantdb_sync.sqlite"))
620
- state.execute("CREATE TABLE IF NOT EXISTS objects (key TEXT PRIMARY KEY, etag TEXT, sha256 TEXT, path TEXT, layout TEXT, dataset TEXT)")
625
+ state.execute("CREATE TABLE IF NOT EXISTS objects (key TEXT PRIMARY KEY, etag TEXT, sha256 TEXT, size INTEGER, path TEXT, layout TEXT, dataset TEXT)")
621
626
  cols = {row[1] for row in state.execute("PRAGMA table_info(objects)")}
622
627
  if "dataset" not in cols:
623
628
  state.execute("ALTER TABLE objects ADD COLUMN dataset TEXT")
629
+ if "size" not in cols:
630
+ state.execute("ALTER TABLE objects ADD COLUMN size INTEGER")
624
631
  state.execute("CREATE TABLE IF NOT EXISTS releases (dataset TEXT PRIMARY KEY, release_id TEXT NOT NULL)")
625
- category_map = {
626
- "daily_unadjusted": "1", "daily_forward": "1", "daily_backward": "1",
627
- "index_daily": "1", "min1_kline": "1", "min5_kline": "1",
628
- "margin_trading": "2", "valuation": "5", "technical_indicators": "5",
629
- "market_sentiment": "5", "features_daily": "6",
630
- }
631
- if dataset not in category_map:
632
+ if dataset not in SYNC_DATASET_CATEGORIES:
632
633
  raise ValidationError(f"不支持同步的数据集: {dataset}")
633
634
 
634
635
  downloaded: List[str] = []
@@ -641,13 +642,14 @@ class QuantDBClient:
641
642
  for release in releases:
642
643
  release_id = release["release_id"]
643
644
  for obj in release.get("objects", []):
644
- key = obj["key"]
645
+ key = self._normalise_release_key(obj["key"])
645
646
  relative_path = obj.get("relative_path") or key
646
647
  target = os.path.join(root, *relative_path.split("/"))
647
- old = state.execute("SELECT etag, sha256, path FROM objects WHERE key=?", (key,)).fetchone()
648
- if old and old[0] == obj.get("etag") and old[1] == obj.get("sha256") and os.path.exists(old[2]):
648
+ expected_size = obj.get("size")
649
+ old = state.execute("SELECT etag, sha256, size, path FROM objects WHERE key=?", (key,)).fetchone()
650
+ if old and old[0] == obj.get("etag") and old[1] == obj.get("sha256") and os.path.exists(old[3]) and (expected_size is None or os.path.getsize(old[3]) == expected_size):
649
651
  continue
650
- resp = self._request("GET", "/api/v1/data/download", params={"category_id": category_map[dataset], "sub_category": dataset, "layout": "v2", "object_key": key}, stream=True)
652
+ resp = self._request("GET", "/api/v1/data/download", params={"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v2", "object_key": key}, stream=True)
651
653
  if resp.status_code != 200:
652
654
  self._check_response(resp)
653
655
  os.makedirs(os.path.dirname(target), exist_ok=True)
@@ -658,13 +660,16 @@ class QuantDBClient:
658
660
  if chunk:
659
661
  fh.write(chunk); digest.update(chunk)
660
662
  actual = digest.hexdigest()
663
+ written_size = os.path.getsize(tmp)
661
664
  if obj.get("sha256") and actual.lower() != obj["sha256"].lower():
662
665
  raise ServerError("对象 SHA-256 校验失败")
666
+ if expected_size is not None and written_size != expected_size:
667
+ raise ServerError(f"对象大小校验失败:期望 {expected_size},实际 {written_size}")
663
668
  os.replace(tmp, target)
664
669
  except Exception:
665
670
  if os.path.exists(tmp): os.remove(tmp)
666
671
  raise
667
- state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,path,layout,dataset) VALUES(?,?,?,?,?,?)", (key, obj.get("etag"), actual, target, "v2_daily_partition", dataset))
672
+ state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,size,path,layout,dataset) VALUES(?,?,?,?,?,?,?)", (key, obj.get("etag"), actual, written_size, target, "v2_daily_partition", dataset))
668
673
  downloaded.append(key)
669
674
  # 仅在该 release 的所有对象成功落盘和校验后提交 cursor。
670
675
  state.execute("INSERT OR REPLACE INTO releases(dataset,release_id) VALUES(?,?)", (dataset, release_id))
@@ -674,14 +679,15 @@ class QuantDBClient:
674
679
  if persisted:
675
680
  return {"dataset": dataset, "layout": "v2_daily_partition", "downloaded": [], "after_release": cursor, "release_id": persisted[0]}
676
681
 
677
- manifest = self._get("/api/v1/data/download/manifest", {"category_id": category_map[dataset], "sub_category": dataset, "layout": "v1"})
682
+ manifest = self._get("/api/v1/data/download/manifest", {"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1"})
678
683
  for obj in manifest.get("files", []):
679
684
  key, relative_path = obj["key"], obj.get("relative_path") or obj["key"]
680
685
  target = os.path.join(root, *relative_path.split("/"))
681
- old = state.execute("SELECT etag, path FROM objects WHERE key=?", (key,)).fetchone()
682
- if old and old[0] == obj.get("etag") and os.path.exists(old[1]):
686
+ expected_size = obj.get("size")
687
+ old = state.execute("SELECT etag, size, path FROM objects WHERE key=?", (key,)).fetchone()
688
+ if old and old[0] == obj.get("etag") and os.path.exists(old[2]) and (expected_size is None or os.path.getsize(old[2]) == expected_size):
683
689
  continue
684
- resp = self._request("GET", "/api/v1/data/download", params={"category_id": category_map[dataset], "sub_category": dataset, "layout": "v1", "symbol": obj.get("symbol", "")}, stream=True)
690
+ resp = self._request("GET", "/api/v1/data/download", params={"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1", "symbol": obj.get("symbol", "")}, stream=True)
685
691
  if resp.status_code != 200: self._check_response(resp)
686
692
  os.makedirs(os.path.dirname(target), exist_ok=True)
687
693
  tmp = target + ".part"
@@ -689,11 +695,14 @@ class QuantDBClient:
689
695
  with open(tmp, "wb") as fh:
690
696
  for chunk in resp.iter_content(1024 * 1024):
691
697
  if chunk: fh.write(chunk)
698
+ written_size = os.path.getsize(tmp)
699
+ if expected_size is not None and written_size != expected_size:
700
+ raise ServerError(f"对象大小校验失败:期望 {expected_size},实际 {written_size}")
692
701
  os.replace(tmp, target)
693
702
  except Exception:
694
703
  if os.path.exists(tmp): os.remove(tmp)
695
704
  raise
696
- state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,path,layout,dataset) VALUES(?,?,?,?,?,?)", (key, obj.get("etag"), "", target, "v1_symbol", dataset))
705
+ state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,size,path,layout,dataset) VALUES(?,?,?,?,?,?,?)", (key, obj.get("etag"), "", written_size, target, "v1_symbol", dataset))
697
706
  downloaded.append(key)
698
707
  state.commit()
699
708
  return {"dataset": dataset, "layout": "v1_symbol", "downloaded": downloaded, "after_release": cursor}
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: quantdb-sdk
3
- Version: 0.2.0
3
+ Version: 0.2.2
4
4
  Summary: QuantDB 量化数据平台官方 Python SDK
5
5
  Author: QuantDB Team
6
6
  License: MIT
@@ -104,6 +104,11 @@ history = client.download_file("1", "daily_forward", symbol="600519.SH", layout=
104
104
 
105
105
  # 以发布清单为 cursor 做原子化增量同步(含 V2 patch)
106
106
  result = client.sync_dataset("daily_forward", save_dir="D:/quantdb-data")
107
+
108
+ # 财务、ETF/可转债等尚无 V2 release 的数据集自动按 V1 Manifest 增量同步,
109
+ # 使用 ETag + size 校验;也可从命令行执行:
110
+ # quantdb sync qdb_xxx balance --save-dir D:/quantdb-data
111
+ financial = client.sync_dataset("balance", save_dir="D:/quantdb-data")
107
112
  ```
108
113
 
109
114
  ## 流量说明
@@ -55,7 +55,7 @@ async def test_a_query_kline_without_range_forces_v1_layout():
55
55
  async def test_a_sync_dataset_uses_release_cursor(tmp_path):
56
56
  payload = b"release-parquet"
57
57
  digest = __import__("hashlib").sha256(payload).hexdigest()
58
- respx.get(f"{API_HOST}/api/v1/data/releases").mock(return_value=httpx.Response(200, json={"releases": [{"release_id": "20260726_180000", "objects": [{"key": "1_kline_data/daily_forward/dt=20260726/data.parquet", "etag": "etag-1", "sha256": digest}]}]}))
58
+ respx.get(f"{API_HOST}/api/v1/data/releases").mock(return_value=httpx.Response(200, json={"releases": [{"release_id": "20260726_180000", "objects": [{"key": "v2/1_kline_data/daily_forward/dt=20260726/data.parquet", "etag": "etag-1", "sha256": digest}]}]}))
59
59
  route = respx.get(f"{API_HOST}/api/v1/data/download").mock(return_value=httpx.Response(200, content=payload))
60
60
  async with AsyncQuantDBClient(api_host=API_HOST, api_key="test-key") as client:
61
61
  result = await client.a_sync_dataset("daily_forward", str(tmp_path))
@@ -64,6 +64,19 @@ async def test_a_sync_dataset_uses_release_cursor(tmp_path):
64
64
  assert parse_qs(urlparse(str(route.calls[0].request.url)).query)["object_key"] == ["1_kline_data/daily_forward/dt=20260726/data.parquet"]
65
65
 
66
66
 
67
+ @pytest.mark.asyncio
68
+ @respx.mock
69
+ async def test_a_sync_financial_v1_validates_manifest_size(tmp_path):
70
+ payload = b"financial-parquet"
71
+ respx.get(f"{API_HOST}/api/v1/data/releases").mock(return_value=httpx.Response(200, json={"releases": []}))
72
+ respx.get(f"{API_HOST}/api/v1/data/download/manifest").mock(return_value=httpx.Response(200, json={"files": [{"key": "3_financial_data/balance/600000.SH.parquet", "relative_path": "3_financial_data/balance/600000.SH.parquet", "symbol": "600000.SH", "etag": "v1-etag", "size": len(payload)}]}))
73
+ respx.get(f"{API_HOST}/api/v1/data/download").mock(return_value=httpx.Response(200, content=payload))
74
+ async with AsyncQuantDBClient(api_host=API_HOST, api_key="test-key") as client:
75
+ result = await client.a_sync_dataset("balance", str(tmp_path))
76
+ assert result["layout"] == "v1_symbol"
77
+ assert (tmp_path / "3_financial_data" / "balance" / "600000.SH.parquet").read_bytes() == payload
78
+
79
+
67
80
  @pytest.mark.asyncio
68
81
  @respx.mock
69
82
  async def test_a_auth_error_raises_auth_error():
@@ -8,6 +8,7 @@ import pytest
8
8
  import responses
9
9
 
10
10
  from quantdb_sdk import AuthError, InsufficientTrafficError, QuantDBClient
11
+ from quantdb_sdk.errors import ServerError
11
12
 
12
13
 
13
14
  API_HOST = "http://localhost:5000"
@@ -79,7 +80,7 @@ def test_sync_dataset_uses_release_cursor_and_object_key(tmp_path):
79
80
  digest = __import__("hashlib").sha256(payload).hexdigest()
80
81
  responses.get(
81
82
  f"{API_HOST}/api/v1/data/releases",
82
- json={"releases": [{"release_id": "20260726_180000", "objects": [{"key": "1_kline_data/daily_forward/dt=20260726/data.parquet", "etag": "etag-1", "sha256": digest}]}]}, status=200,
83
+ json={"releases": [{"release_id": "20260726_180000", "objects": [{"key": "v2/1_kline_data/daily_forward/dt=20260726/data.parquet", "etag": "etag-1", "sha256": digest}]}]}, status=200,
83
84
  )
84
85
  responses.get(f"{API_HOST}/api/v1/data/download", body=payload, status=200)
85
86
  client = QuantDBClient(api_host=API_HOST, api_key="test-key")
@@ -90,6 +91,35 @@ def test_sync_dataset_uses_release_cursor_and_object_key(tmp_path):
90
91
  assert params["object_key"] == ["1_kline_data/daily_forward/dt=20260726/data.parquet"]
91
92
 
92
93
 
94
+ @responses.activate
95
+ def test_sync_financial_v1_validates_manifest_size(tmp_path):
96
+ payload = b"financial-parquet"
97
+ responses.get(f"{API_HOST}/api/v1/data/releases", json={"releases": []}, status=200)
98
+ responses.get(
99
+ f"{API_HOST}/api/v1/data/download/manifest",
100
+ json={"files": [{"key": "3_financial_data/balance/600000.SH.parquet", "relative_path": "3_financial_data/balance/600000.SH.parquet", "symbol": "600000.SH", "etag": "v1-etag", "size": len(payload)}]},
101
+ status=200,
102
+ )
103
+ responses.get(f"{API_HOST}/api/v1/data/download", body=payload, status=200)
104
+ result = QuantDBClient(api_host=API_HOST, api_key="test-key").sync_dataset("balance", str(tmp_path))
105
+ assert result["layout"] == "v1_symbol"
106
+ assert (tmp_path / "3_financial_data" / "balance" / "600000.SH.parquet").read_bytes() == payload
107
+
108
+
109
+ @responses.activate
110
+ def test_sync_v1_size_failure_does_not_keep_partial_file(tmp_path):
111
+ responses.get(f"{API_HOST}/api/v1/data/releases", json={"releases": []}, status=200)
112
+ responses.get(
113
+ f"{API_HOST}/api/v1/data/download/manifest",
114
+ json={"files": [{"key": "3_financial_data/balance/600000.SH.parquet", "symbol": "600000.SH", "etag": "v1-etag", "size": 999}]},
115
+ status=200,
116
+ )
117
+ responses.get(f"{API_HOST}/api/v1/data/download", body=b"short", status=200)
118
+ with pytest.raises(ServerError):
119
+ QuantDBClient(api_host=API_HOST, api_key="test-key").sync_dataset("balance", str(tmp_path))
120
+ assert not list(tmp_path.rglob("*.part"))
121
+
122
+
93
123
  @responses.activate
94
124
  def test_auth_error_raises_auth_error():
95
125
  responses.get(
File without changes
File without changes
File without changes