quantdb-sdk 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/CHANGELOG.md +10 -0
  2. {quantdb_sdk-0.2.1/quantdb_sdk.egg-info → quantdb_sdk-0.2.2}/PKG-INFO +6 -1
  3. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/README.md +5 -0
  4. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/pyproject.toml +1 -1
  5. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk/__init__.py +1 -1
  6. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk/__main__.py +14 -0
  7. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk/_utils.py +16 -0
  8. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk/async_client.py +34 -22
  9. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk/client.py +23 -19
  10. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2/quantdb_sdk.egg-info}/PKG-INFO +6 -1
  11. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/tests/test_async_client.py +13 -0
  12. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/tests/test_client.py +30 -0
  13. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/LICENSE +0 -0
  14. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/MANIFEST.in +0 -0
  15. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk/errors.py +0 -0
  16. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk/py.typed +0 -0
  17. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/SOURCES.txt +0 -0
  18. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/dependency_links.txt +0 -0
  19. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/entry_points.txt +0 -0
  20. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/requires.txt +0 -0
  21. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/quantdb_sdk.egg-info/top_level.txt +0 -0
  22. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/setup.cfg +0 -0
  23. {quantdb_sdk-0.2.1 → quantdb_sdk-0.2.2}/tests/test_technical_indicators.py +0 -0
@@ -3,6 +3,16 @@
3
3
  所有 notable 变更都会记录在此文件。格式基于 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.1.0/),
4
4
  版本号遵循 [Semantic Versioning](https://semver.org/lang/zh-CN/)。
5
5
 
6
+ ## [0.2.2] - 2026-07-26
7
+
8
+ ### Added
9
+ - **全数据集增量同步**:`sync_dataset()` / `a_sync_dataset()` 覆盖基础、财务、ETF/可转债、行情、技术与 ML 子类;没有 V2 release 的数据集自动走 V1 Manifest。
10
+ - **CLI 同步**:新增 `quantdb sync <api_key> <dataset> --save-dir <目录>`。
11
+
12
+ ### Fixed
13
+ - **V1 强校验**:V1 Manifest 同步以 ETag + size 判断变更,并在原子落盘前验证实际字节数。
14
+ - **同步/异步一致性**:异步 release 同步采用 `relative_path`,SQLite 同时记录对象大小,与同步客户端保持同构目录和恢复语义。
15
+
6
16
  ## [0.2.1] - 2026-07-26
7
17
 
8
18
  ### Fixed
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: quantdb-sdk
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: QuantDB 量化数据平台官方 Python SDK
5
5
  Author: QuantDB Team
6
6
  License: MIT
@@ -104,6 +104,11 @@ history = client.download_file("1", "daily_forward", symbol="600519.SH", layout=
104
104
 
105
105
  # 以发布清单为 cursor 做原子化增量同步(含 V2 patch)
106
106
  result = client.sync_dataset("daily_forward", save_dir="D:/quantdb-data")
107
+
108
+ # 财务、ETF/可转债等尚无 V2 release 的数据集自动按 V1 Manifest 增量同步,
109
+ # 使用 ETag + size 校验;也可从命令行执行:
110
+ # quantdb sync qdb_xxx balance --save-dir D:/quantdb-data
111
+ financial = client.sync_dataset("balance", save_dir="D:/quantdb-data")
107
112
  ```
108
113
 
109
114
  ## 流量说明
@@ -67,6 +67,11 @@ history = client.download_file("1", "daily_forward", symbol="600519.SH", layout=
67
67
 
68
68
  # 以发布清单为 cursor 做原子化增量同步(含 V2 patch)
69
69
  result = client.sync_dataset("daily_forward", save_dir="D:/quantdb-data")
70
+
71
+ # 财务、ETF/可转债等尚无 V2 release 的数据集自动按 V1 Manifest 增量同步,
72
+ # 使用 ETag + size 校验;也可从命令行执行:
73
+ # quantdb sync qdb_xxx balance --save-dir D:/quantdb-data
74
+ financial = client.sync_dataset("balance", save_dir="D:/quantdb-data")
70
75
  ```
71
76
 
72
77
  ## 流量说明
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "quantdb-sdk"
7
- version = "0.2.1"
7
+ version = "0.2.2"
8
8
  description = "QuantDB 量化数据平台官方 Python SDK"
9
9
  readme = "README.md"
10
10
  license = {text = "MIT"}
@@ -5,7 +5,7 @@ try:
5
5
  # 注意:此处用 PyPI 包名(连字符)查找版本号,不是模块名(下划线)
6
6
  __version__ = version("quantdb-sdk")
7
7
  except ImportError:
8
- __version__ = "0.2.1"
8
+ __version__ = "0.2.2"
9
9
 
10
10
  from .async_client import AsyncQuantDBClient
11
11
  from .client import DuckDBWarehouse, QuantDBClient
@@ -39,6 +39,12 @@ def main() -> int:
39
39
  usage_parser = subparsers.add_parser("usage", help="查询账户用量和订阅状态")
40
40
  usage_parser.add_argument("api_key", help="API Key")
41
41
 
42
+ sync_parser = subparsers.add_parser("sync", help="以 release/Manifest 增量同步一个数据集")
43
+ sync_parser.add_argument("api_key", help="API Key")
44
+ sync_parser.add_argument("dataset", help="数据集,如 daily_forward、balance、etf_pcf")
45
+ sync_parser.add_argument("--save-dir", default=None, help="本地同步根目录")
46
+ sync_parser.add_argument("--after-release", default=None, help="覆盖本地 release cursor(一般无需传入)")
47
+
42
48
  args = parser.parse_args()
43
49
 
44
50
  if not args.command:
@@ -66,6 +72,14 @@ def main() -> int:
66
72
  print(f" 订阅: {sub.get('status', 'none')}")
67
73
  return 0
68
74
 
75
+ if args.command == "sync":
76
+ result = client.sync_dataset(args.dataset, save_dir=args.save_dir, after_release=args.after_release)
77
+ print(f"同步完成: {result['dataset']} ({result['layout']})")
78
+ print(f" 新增/更新对象: {len(result['downloaded'])}")
79
+ if result.get("release_id"):
80
+ print(f" Release: {result['release_id']}")
81
+ return 0
82
+
69
83
  except Exception as e:
70
84
  print(f"✗ 错误: {e}", file=sys.stderr)
71
85
  return 1
@@ -5,9 +5,25 @@
5
5
 
6
6
  import os
7
7
  import re
8
+ from typing import Dict
8
9
  from urllib.parse import unquote
9
10
 
10
11
 
12
+ # ``sync_dataset`` 的统一数据集目录。V2 可用时走 release cursor;其余数据集
13
+ # 自动使用 V1 Manifest 的 ETag + size 增量同步。
14
+ SYNC_DATASET_CATEGORIES: Dict[str, str] = {
15
+ "daily_unadjusted": "1", "daily_forward": "1", "daily_backward": "1",
16
+ "index_daily": "1", "min1_kline": "1", "min5_kline": "1", "tick_data": "1",
17
+ "instrument_detail": "2", "sector_concept": "2", "index_weights": "2",
18
+ "trading_calendar": "2", "margin_trading": "2",
19
+ "balance": "3", "income": "3", "cashflow": "3", "capital": "3",
20
+ "holder_num": "3", "pershare_index": "3", "dividend_factors": "3",
21
+ "etf_pcf": "4", "convertible_bond": "4",
22
+ "valuation": "5", "technical_indicators": "5", "market_sentiment": "5",
23
+ "features_daily": "6", "l1_l2_factors": "6", "l2_factors": "6",
24
+ }
25
+
26
+
11
27
  def default_download_dir() -> str:
12
28
  """返回默认下载目录。
13
29
 
@@ -10,7 +10,7 @@ from typing import Any, Dict, List, Optional, Literal
10
10
  import httpx
11
11
  import pandas as pd
12
12
 
13
- from ._utils import bytes_to_gb, default_download_dir, parse_filename_from_content_disposition
13
+ from ._utils import SYNC_DATASET_CATEGORIES, bytes_to_gb, default_download_dir, parse_filename_from_content_disposition
14
14
  from .errors import (
15
15
  AuthError,
16
16
  InsufficientTrafficError,
@@ -37,7 +37,7 @@ class AsyncQuantDBClient:
37
37
  ):
38
38
  self.api_host = api_host.rstrip("/")
39
39
  self.timeout = timeout
40
- headers = {"User-Agent": "QuantDB-Python-SDK/0.2.1"}
40
+ headers = {"User-Agent": "QuantDB-Python-SDK/0.2.2"}
41
41
  if api_key:
42
42
  headers["X-API-Key"] = api_key
43
43
  elif token:
@@ -528,17 +528,13 @@ class AsyncQuantDBClient:
528
528
 
529
529
  async def a_sync_dataset(self, dataset: str, save_dir: Optional[str] = None, after_release: Optional[str] = None) -> Dict[str, Any]:
530
530
  """异步版 release 增量同步;状态格式与 ``sync_dataset`` 兼容。"""
531
- category_map = {
532
- "daily_unadjusted": "1", "daily_forward": "1", "daily_backward": "1", "index_daily": "1",
533
- "min1_kline": "1", "min5_kline": "1", "margin_trading": "2", "valuation": "5",
534
- "technical_indicators": "5", "market_sentiment": "5", "features_daily": "6",
535
- }
536
- if dataset not in category_map: raise ValidationError(f"不支持同步的数据集: {dataset}")
531
+ if dataset not in SYNC_DATASET_CATEGORIES: raise ValidationError(f"不支持同步的数据集: {dataset}")
537
532
  root = os.path.abspath(save_dir or default_download_dir()); os.makedirs(root, exist_ok=True)
538
533
  state = sqlite3.connect(os.path.join(root, "quantdb_sync.sqlite"))
539
- state.execute("CREATE TABLE IF NOT EXISTS objects (key TEXT PRIMARY KEY, etag TEXT, sha256 TEXT, path TEXT, layout TEXT, dataset TEXT)")
534
+ state.execute("CREATE TABLE IF NOT EXISTS objects (key TEXT PRIMARY KEY, etag TEXT, sha256 TEXT, size INTEGER, path TEXT, layout TEXT, dataset TEXT)")
540
535
  cols = {row[1] for row in state.execute("PRAGMA table_info(objects)")}
541
536
  if "dataset" not in cols: state.execute("ALTER TABLE objects ADD COLUMN dataset TEXT")
537
+ if "size" not in cols: state.execute("ALTER TABLE objects ADD COLUMN size INTEGER")
542
538
  state.execute("CREATE TABLE IF NOT EXISTS releases (dataset TEXT PRIMARY KEY, release_id TEXT NOT NULL)")
543
539
  downloaded: List[str] = []
544
540
  try:
@@ -549,12 +545,14 @@ class AsyncQuantDBClient:
549
545
  for release in releases:
550
546
  for obj in release.get("objects", []):
551
547
  key = self._normalise_release_key(obj["key"])
552
- target = os.path.join(root, *key.split("/"))
553
- old = state.execute("SELECT etag,sha256,path FROM objects WHERE key=?", (key,)).fetchone()
554
- if old and old[0] == obj.get("etag") and old[1] == obj.get("sha256") and os.path.exists(old[2]): continue
548
+ relative_path = obj.get("relative_path") or key
549
+ target = os.path.join(root, *relative_path.split("/"))
550
+ expected_size = obj.get("size")
551
+ old = state.execute("SELECT etag,sha256,size,path FROM objects WHERE key=?", (key,)).fetchone()
552
+ if old and old[0] == obj.get("etag") and old[1] == obj.get("sha256") and os.path.exists(old[3]) and (expected_size is None or os.path.getsize(old[3]) == expected_size): continue
555
553
  os.makedirs(os.path.dirname(target), exist_ok=True)
556
554
  tmp, digest = target + ".part", hashlib.sha256()
557
- async with self.client.stream("GET", f"{self.api_host}/api/v1/data/download", params={"category_id": category_map[dataset], "sub_category": dataset, "layout": "v2", "object_key": key}) as resp:
555
+ async with self.client.stream("GET", f"{self.api_host}/api/v1/data/download", params={"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v2", "object_key": key}) as resp:
558
556
  if resp.status_code != 200:
559
557
  body = await resp.aread(); self._check_response(httpx.Response(resp.status_code, content=body)); raise QuantDBError("下载失败")
560
558
  try:
@@ -563,24 +561,38 @@ class AsyncQuantDBClient:
563
561
  if chunk: fh.write(chunk); digest.update(chunk)
564
562
  actual = digest.hexdigest()
565
563
  if obj.get("sha256") and actual.lower() != obj["sha256"].lower(): raise ServerError("对象 SHA-256 校验失败")
564
+ written_size = os.path.getsize(tmp)
565
+ if expected_size is not None and written_size != expected_size: raise ServerError(f"对象大小校验失败:期望 {expected_size},实际 {written_size}")
566
566
  os.replace(tmp, target)
567
567
  except Exception:
568
568
  if os.path.exists(tmp): os.remove(tmp)
569
569
  raise
570
- state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,path,layout,dataset) VALUES(?,?,?,?,?,?)", (key, obj.get("etag"), actual, target, "v2_daily_partition", dataset)); downloaded.append(key)
570
+ state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,size,path,layout,dataset) VALUES(?,?,?,?,?,?,?)", (key, obj.get("etag"), actual, written_size, target, "v2_daily_partition", dataset)); downloaded.append(key)
571
571
  state.execute("INSERT OR REPLACE INTO releases(dataset,release_id) VALUES(?,?)", (dataset, release["release_id"])); state.commit()
572
572
  return {"dataset": dataset, "layout": "v2_daily_partition", "downloaded": downloaded, "after_release": cursor, "release_id": releases[-1]["release_id"]}
573
573
  if persisted: return {"dataset": dataset, "layout": "v2_daily_partition", "downloaded": [], "after_release": cursor, "release_id": persisted[0]}
574
- files = (await self._get("/api/v1/data/download/manifest", {"category_id": category_map[dataset], "sub_category": dataset, "layout": "v1"})).get("files", [])
574
+ files = (await self._get("/api/v1/data/download/manifest", {"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1"})).get("files", [])
575
575
  for obj in files:
576
576
  key, target = obj["key"], os.path.join(root, *(obj.get("relative_path") or obj["key"]).split("/"))
577
- old = state.execute("SELECT etag,path FROM objects WHERE key=?", (key,)).fetchone()
578
- if old and old[0] == obj.get("etag") and os.path.exists(old[1]): continue
579
- await self.a_download_file(category_map[dataset], dataset, symbol=obj.get("symbol"), save_dir=os.path.dirname(target), layout="v1")
580
- # 下载文件名来自服务端,按清单目标路径归位以保证本地根目录同构。
581
- source = os.path.join(os.path.dirname(target), os.path.basename(target))
582
- if os.path.abspath(source) != os.path.abspath(target): os.replace(source, target)
583
- state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,path,layout,dataset) VALUES(?,?,?,?,?,?)", (key, obj.get("etag"), "", target, "v1_symbol", dataset)); downloaded.append(key)
577
+ expected_size = obj.get("size")
578
+ old = state.execute("SELECT etag,size,path FROM objects WHERE key=?", (key,)).fetchone()
579
+ if old and old[0] == obj.get("etag") and os.path.exists(old[2]) and (expected_size is None or os.path.getsize(old[2]) == expected_size): continue
580
+ os.makedirs(os.path.dirname(target), exist_ok=True)
581
+ tmp, written_size = target + ".part", 0
582
+ try:
583
+ async with self.client.stream("GET", f"{self.api_host}/api/v1/data/download", params={"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1", "symbol": obj.get("symbol", "")}) as resp:
584
+ if resp.status_code != 200:
585
+ body = await resp.aread(); self._check_response(httpx.Response(resp.status_code, content=body)); raise QuantDBError("下载失败")
586
+ with open(tmp, "wb") as fh:
587
+ async for chunk in resp.aiter_bytes(1024 * 1024):
588
+ if chunk: fh.write(chunk)
589
+ written_size = os.path.getsize(tmp)
590
+ if expected_size is not None and written_size != expected_size: raise ServerError(f"对象大小校验失败:期望 {expected_size},实际 {written_size}")
591
+ os.replace(tmp, target)
592
+ except Exception:
593
+ if os.path.exists(tmp): os.remove(tmp)
594
+ raise
595
+ state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,size,path,layout,dataset) VALUES(?,?,?,?,?,?,?)", (key, obj.get("etag"), "", written_size, target, "v1_symbol", dataset)); downloaded.append(key)
584
596
  state.commit(); return {"dataset": dataset, "layout": "v1_symbol", "downloaded": downloaded, "after_release": cursor}
585
597
  finally:
586
598
  state.close()
@@ -13,7 +13,7 @@ import requests
13
13
  from requests.adapters import HTTPAdapter
14
14
  from urllib3.util.retry import Retry
15
15
 
16
- from ._utils import bytes_to_gb, default_download_dir, parse_filename_from_content_disposition
16
+ from ._utils import SYNC_DATASET_CATEGORIES, bytes_to_gb, default_download_dir, parse_filename_from_content_disposition
17
17
  from .errors import (
18
18
  AuthError,
19
19
  InsufficientTrafficError,
@@ -48,7 +48,7 @@ class QuantDBClient:
48
48
  self.session = requests.Session()
49
49
  # User-Agent 中的版本与 pyproject.toml 同步,用于服务端日志归因
50
50
  # 维护提示:每次版本号变化必须同步改这里(init 里的 __version__ 走 metadata 自动同步)
51
- self.session.headers.update({"User-Agent": "QuantDB-Python-SDK/0.2.1"})
51
+ self.session.headers.update({"User-Agent": "QuantDB-Python-SDK/0.2.2"})
52
52
 
53
53
  if api_key:
54
54
  self.headers = {"X-API-Key": api_key}
@@ -622,18 +622,14 @@ class QuantDBClient:
622
622
  root = os.path.abspath(save_dir or default_download_dir())
623
623
  os.makedirs(root, exist_ok=True)
624
624
  state = sqlite3.connect(os.path.join(root, "quantdb_sync.sqlite"))
625
- state.execute("CREATE TABLE IF NOT EXISTS objects (key TEXT PRIMARY KEY, etag TEXT, sha256 TEXT, path TEXT, layout TEXT, dataset TEXT)")
625
+ state.execute("CREATE TABLE IF NOT EXISTS objects (key TEXT PRIMARY KEY, etag TEXT, sha256 TEXT, size INTEGER, path TEXT, layout TEXT, dataset TEXT)")
626
626
  cols = {row[1] for row in state.execute("PRAGMA table_info(objects)")}
627
627
  if "dataset" not in cols:
628
628
  state.execute("ALTER TABLE objects ADD COLUMN dataset TEXT")
629
+ if "size" not in cols:
630
+ state.execute("ALTER TABLE objects ADD COLUMN size INTEGER")
629
631
  state.execute("CREATE TABLE IF NOT EXISTS releases (dataset TEXT PRIMARY KEY, release_id TEXT NOT NULL)")
630
- category_map = {
631
- "daily_unadjusted": "1", "daily_forward": "1", "daily_backward": "1",
632
- "index_daily": "1", "min1_kline": "1", "min5_kline": "1",
633
- "margin_trading": "2", "valuation": "5", "technical_indicators": "5",
634
- "market_sentiment": "5", "features_daily": "6",
635
- }
636
- if dataset not in category_map:
632
+ if dataset not in SYNC_DATASET_CATEGORIES:
637
633
  raise ValidationError(f"不支持同步的数据集: {dataset}")
638
634
 
639
635
  downloaded: List[str] = []
@@ -649,10 +645,11 @@ class QuantDBClient:
649
645
  key = self._normalise_release_key(obj["key"])
650
646
  relative_path = obj.get("relative_path") or key
651
647
  target = os.path.join(root, *relative_path.split("/"))
652
- old = state.execute("SELECT etag, sha256, path FROM objects WHERE key=?", (key,)).fetchone()
653
- if old and old[0] == obj.get("etag") and old[1] == obj.get("sha256") and os.path.exists(old[2]):
648
+ expected_size = obj.get("size")
649
+ old = state.execute("SELECT etag, sha256, size, path FROM objects WHERE key=?", (key,)).fetchone()
650
+ if old and old[0] == obj.get("etag") and old[1] == obj.get("sha256") and os.path.exists(old[3]) and (expected_size is None or os.path.getsize(old[3]) == expected_size):
654
651
  continue
655
- resp = self._request("GET", "/api/v1/data/download", params={"category_id": category_map[dataset], "sub_category": dataset, "layout": "v2", "object_key": key}, stream=True)
652
+ resp = self._request("GET", "/api/v1/data/download", params={"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v2", "object_key": key}, stream=True)
656
653
  if resp.status_code != 200:
657
654
  self._check_response(resp)
658
655
  os.makedirs(os.path.dirname(target), exist_ok=True)
@@ -663,13 +660,16 @@ class QuantDBClient:
663
660
  if chunk:
664
661
  fh.write(chunk); digest.update(chunk)
665
662
  actual = digest.hexdigest()
663
+ written_size = os.path.getsize(tmp)
666
664
  if obj.get("sha256") and actual.lower() != obj["sha256"].lower():
667
665
  raise ServerError("对象 SHA-256 校验失败")
666
+ if expected_size is not None and written_size != expected_size:
667
+ raise ServerError(f"对象大小校验失败:期望 {expected_size},实际 {written_size}")
668
668
  os.replace(tmp, target)
669
669
  except Exception:
670
670
  if os.path.exists(tmp): os.remove(tmp)
671
671
  raise
672
- state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,path,layout,dataset) VALUES(?,?,?,?,?,?)", (key, obj.get("etag"), actual, target, "v2_daily_partition", dataset))
672
+ state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,size,path,layout,dataset) VALUES(?,?,?,?,?,?,?)", (key, obj.get("etag"), actual, written_size, target, "v2_daily_partition", dataset))
673
673
  downloaded.append(key)
674
674
  # 仅在该 release 的所有对象成功落盘和校验后提交 cursor。
675
675
  state.execute("INSERT OR REPLACE INTO releases(dataset,release_id) VALUES(?,?)", (dataset, release_id))
@@ -679,14 +679,15 @@ class QuantDBClient:
679
679
  if persisted:
680
680
  return {"dataset": dataset, "layout": "v2_daily_partition", "downloaded": [], "after_release": cursor, "release_id": persisted[0]}
681
681
 
682
- manifest = self._get("/api/v1/data/download/manifest", {"category_id": category_map[dataset], "sub_category": dataset, "layout": "v1"})
682
+ manifest = self._get("/api/v1/data/download/manifest", {"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1"})
683
683
  for obj in manifest.get("files", []):
684
684
  key, relative_path = obj["key"], obj.get("relative_path") or obj["key"]
685
685
  target = os.path.join(root, *relative_path.split("/"))
686
- old = state.execute("SELECT etag, path FROM objects WHERE key=?", (key,)).fetchone()
687
- if old and old[0] == obj.get("etag") and os.path.exists(old[1]):
686
+ expected_size = obj.get("size")
687
+ old = state.execute("SELECT etag, size, path FROM objects WHERE key=?", (key,)).fetchone()
688
+ if old and old[0] == obj.get("etag") and os.path.exists(old[2]) and (expected_size is None or os.path.getsize(old[2]) == expected_size):
688
689
  continue
689
- resp = self._request("GET", "/api/v1/data/download", params={"category_id": category_map[dataset], "sub_category": dataset, "layout": "v1", "symbol": obj.get("symbol", "")}, stream=True)
690
+ resp = self._request("GET", "/api/v1/data/download", params={"category_id": SYNC_DATASET_CATEGORIES[dataset], "sub_category": dataset, "layout": "v1", "symbol": obj.get("symbol", "")}, stream=True)
690
691
  if resp.status_code != 200: self._check_response(resp)
691
692
  os.makedirs(os.path.dirname(target), exist_ok=True)
692
693
  tmp = target + ".part"
@@ -694,11 +695,14 @@ class QuantDBClient:
694
695
  with open(tmp, "wb") as fh:
695
696
  for chunk in resp.iter_content(1024 * 1024):
696
697
  if chunk: fh.write(chunk)
698
+ written_size = os.path.getsize(tmp)
699
+ if expected_size is not None and written_size != expected_size:
700
+ raise ServerError(f"对象大小校验失败:期望 {expected_size},实际 {written_size}")
697
701
  os.replace(tmp, target)
698
702
  except Exception:
699
703
  if os.path.exists(tmp): os.remove(tmp)
700
704
  raise
701
- state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,path,layout,dataset) VALUES(?,?,?,?,?,?)", (key, obj.get("etag"), "", target, "v1_symbol", dataset))
705
+ state.execute("INSERT OR REPLACE INTO objects(key,etag,sha256,size,path,layout,dataset) VALUES(?,?,?,?,?,?,?)", (key, obj.get("etag"), "", written_size, target, "v1_symbol", dataset))
702
706
  downloaded.append(key)
703
707
  state.commit()
704
708
  return {"dataset": dataset, "layout": "v1_symbol", "downloaded": downloaded, "after_release": cursor}
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: quantdb-sdk
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: QuantDB 量化数据平台官方 Python SDK
5
5
  Author: QuantDB Team
6
6
  License: MIT
@@ -104,6 +104,11 @@ history = client.download_file("1", "daily_forward", symbol="600519.SH", layout=
104
104
 
105
105
  # 以发布清单为 cursor 做原子化增量同步(含 V2 patch)
106
106
  result = client.sync_dataset("daily_forward", save_dir="D:/quantdb-data")
107
+
108
+ # 财务、ETF/可转债等尚无 V2 release 的数据集自动按 V1 Manifest 增量同步,
109
+ # 使用 ETag + size 校验;也可从命令行执行:
110
+ # quantdb sync qdb_xxx balance --save-dir D:/quantdb-data
111
+ financial = client.sync_dataset("balance", save_dir="D:/quantdb-data")
107
112
  ```
108
113
 
109
114
  ## 流量说明
@@ -64,6 +64,19 @@ async def test_a_sync_dataset_uses_release_cursor(tmp_path):
64
64
  assert parse_qs(urlparse(str(route.calls[0].request.url)).query)["object_key"] == ["1_kline_data/daily_forward/dt=20260726/data.parquet"]
65
65
 
66
66
 
67
+ @pytest.mark.asyncio
68
+ @respx.mock
69
+ async def test_a_sync_financial_v1_validates_manifest_size(tmp_path):
70
+ payload = b"financial-parquet"
71
+ respx.get(f"{API_HOST}/api/v1/data/releases").mock(return_value=httpx.Response(200, json={"releases": []}))
72
+ respx.get(f"{API_HOST}/api/v1/data/download/manifest").mock(return_value=httpx.Response(200, json={"files": [{"key": "3_financial_data/balance/600000.SH.parquet", "relative_path": "3_financial_data/balance/600000.SH.parquet", "symbol": "600000.SH", "etag": "v1-etag", "size": len(payload)}]}))
73
+ respx.get(f"{API_HOST}/api/v1/data/download").mock(return_value=httpx.Response(200, content=payload))
74
+ async with AsyncQuantDBClient(api_host=API_HOST, api_key="test-key") as client:
75
+ result = await client.a_sync_dataset("balance", str(tmp_path))
76
+ assert result["layout"] == "v1_symbol"
77
+ assert (tmp_path / "3_financial_data" / "balance" / "600000.SH.parquet").read_bytes() == payload
78
+
79
+
67
80
  @pytest.mark.asyncio
68
81
  @respx.mock
69
82
  async def test_a_auth_error_raises_auth_error():
@@ -8,6 +8,7 @@ import pytest
8
8
  import responses
9
9
 
10
10
  from quantdb_sdk import AuthError, InsufficientTrafficError, QuantDBClient
11
+ from quantdb_sdk.errors import ServerError
11
12
 
12
13
 
13
14
  API_HOST = "http://localhost:5000"
@@ -90,6 +91,35 @@ def test_sync_dataset_uses_release_cursor_and_object_key(tmp_path):
90
91
  assert params["object_key"] == ["1_kline_data/daily_forward/dt=20260726/data.parquet"]
91
92
 
92
93
 
94
+ @responses.activate
95
+ def test_sync_financial_v1_validates_manifest_size(tmp_path):
96
+ payload = b"financial-parquet"
97
+ responses.get(f"{API_HOST}/api/v1/data/releases", json={"releases": []}, status=200)
98
+ responses.get(
99
+ f"{API_HOST}/api/v1/data/download/manifest",
100
+ json={"files": [{"key": "3_financial_data/balance/600000.SH.parquet", "relative_path": "3_financial_data/balance/600000.SH.parquet", "symbol": "600000.SH", "etag": "v1-etag", "size": len(payload)}]},
101
+ status=200,
102
+ )
103
+ responses.get(f"{API_HOST}/api/v1/data/download", body=payload, status=200)
104
+ result = QuantDBClient(api_host=API_HOST, api_key="test-key").sync_dataset("balance", str(tmp_path))
105
+ assert result["layout"] == "v1_symbol"
106
+ assert (tmp_path / "3_financial_data" / "balance" / "600000.SH.parquet").read_bytes() == payload
107
+
108
+
109
+ @responses.activate
110
+ def test_sync_v1_size_failure_does_not_keep_partial_file(tmp_path):
111
+ responses.get(f"{API_HOST}/api/v1/data/releases", json={"releases": []}, status=200)
112
+ responses.get(
113
+ f"{API_HOST}/api/v1/data/download/manifest",
114
+ json={"files": [{"key": "3_financial_data/balance/600000.SH.parquet", "symbol": "600000.SH", "etag": "v1-etag", "size": 999}]},
115
+ status=200,
116
+ )
117
+ responses.get(f"{API_HOST}/api/v1/data/download", body=b"short", status=200)
118
+ with pytest.raises(ServerError):
119
+ QuantDBClient(api_host=API_HOST, api_key="test-key").sync_dataset("balance", str(tmp_path))
120
+ assert not list(tmp_path.rglob("*.part"))
121
+
122
+
93
123
  @responses.activate
94
124
  def test_auth_error_raises_auth_error():
95
125
  responses.get(
File without changes
File without changes
File without changes