quantdb-sdk 0.4.0__tar.gz → 0.4.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/CHANGELOG.md +25 -0
  2. {quantdb_sdk-0.4.0/quantdb_sdk.egg-info → quantdb_sdk-0.4.2}/PKG-INFO +2 -2
  3. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/pyproject.toml +69 -67
  4. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk/__init__.py +1 -1
  5. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk/_utils.py +1 -1
  6. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk/async_client.py +1 -1
  7. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk/client.py +1 -1
  8. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk/hub.py +122 -36
  9. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2/quantdb_sdk.egg-info}/PKG-INFO +2 -2
  10. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk.egg-info/requires.txt +1 -1
  11. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/tests/test_data_hub.py +36 -2
  12. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/LICENSE +0 -0
  13. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/MANIFEST.in +0 -0
  14. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/README.md +0 -0
  15. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk/__main__.py +0 -0
  16. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk/errors.py +0 -0
  17. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk/examples/__init__.py +0 -0
  18. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk/examples/demo_01_connect_basics.py +0 -0
  19. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk/examples/demo_02_kline_v2_daily.py +0 -0
  20. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk/examples/demo_03_sync_full_market.py +0 -0
  21. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk/examples/demo_04_duckdb_warehouse.py +0 -0
  22. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk/examples/demo_05_read_local_data.py +0 -0
  23. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk/examples/demo_06_tick_and_factors.py +0 -0
  24. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk/examples/demo_07_data_hub.py +0 -0
  25. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk/layout.py +0 -0
  26. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk/py.typed +0 -0
  27. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk.egg-info/SOURCES.txt +0 -0
  28. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk.egg-info/dependency_links.txt +0 -0
  29. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk.egg-info/entry_points.txt +0 -0
  30. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/quantdb_sdk.egg-info/top_level.txt +0 -0
  31. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/setup.cfg +0 -0
  32. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/tests/test_async_client.py +0 -0
  33. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/tests/test_client.py +0 -0
  34. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/tests/test_hub_models.py +0 -0
  35. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/tests/test_layout.py +0 -0
  36. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/tests/test_technical_indicators.py +0 -0
  37. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/tests/test_update_pipeline.py +0 -0
  38. {quantdb_sdk-0.4.0 → quantdb_sdk-0.4.2}/tests/test_utils.py +0 -0
@@ -3,6 +3,31 @@
3
3
  所有 notable 变更都会记录在此文件。格式基于 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.1.0/),
4
4
  版本号遵循 [Semantic Versioning](https://semver.org/lang/zh-CN/)。
5
5
 
6
+ ## \[0.4.2] - 2026-10-05
7
+
8
+ ### Added
9
+
10
+ - **新增数据集 `per_share`(每股指标补充表,分类 3)**:补登记为第 30 个数据集。
11
+
12
+ - `SYNC_DATASET_CATEGORIES` 与 `DATASET_REGISTRY` 由 29 项增至 30 项;`client.data.datasets()` / `info()` / `local_files()` / `read_local()` / `sync_dataset()` / `mount()` 均可直接按 `per_share` 使用。
13
+ - 物理形态为 V1 逐股票文件 `3_financial_data/per_share/{Symbol}.parquet`(由 `03_financial.py` 采集,部分股票为空);`layout=v1_symbol`,读取/查询需提供 `symbol`。
14
+ - 与 MCP catalog、后端 `subCategoryMapping`、前端分类表同步对齐(均为 30 项)。
15
+
16
+ ## \[0.4.1] - 2026-10-05
17
+
18
+ ### Changed
19
+
20
+ - **本地零流量读取提速**(`client.data.read_local` / `read`):改用 DuckDB 对**精确分区文件列表**做一次 `read_parquet`,把 `symbol` / 日期区间 / 列裁剪全部下推到 SQL,替代原先逐文件 `pd.read_parquet` 再 `concat`。实测(120 分区 × 1000 标的)单标的全历史 `0.62s → 0.22s`,带 `columns` 列裁剪 `→ 0.07s`。
21
+
22
+ - V2 按日分区与 tick 分片的 `trade_date` 由分区目录名 / 文件名推导,**不依赖** hive 注入的 `dt` 列(该列仅在较新 DuckDB 存在)。
23
+ - V1 逐股票表日期过滤用 `substr(replace(CAST(col AS VARCHAR),'-',''),1,8)` 归一化后下推,同时兼容整型报告期 `20251231` 与字符串 `2025-12-31` / 带时分秒。
24
+
25
+ - **依赖下界调整**:`duckdb` 由 `>=0.8.0` 改为 `>=1.0.0`。`read_local` 用到 `read_parquet(filename=true)` + `* EXCLUDE (filename)`;0.8.x 在部分环境(Windows / Python 3.10)导入即崩溃,1.0.0 ~ 1.5.5 已验证通过。
26
+
27
+ ### Fixed
28
+
29
+ - `read_local(..., columns=[...])` 在 V2 / tick 上未把 `symbol` 过滤下推:当 `columns` 不含 `symbol` 时,旧实现会在读取后被静默跳过过滤、返回全市场行。现 `symbol` 始终在 SQL 中下推。
30
+
6
31
  ## \[0.4.0] - 2026-10-05
7
32
 
8
33
  ### Added
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: quantdb-sdk
3
- Version: 0.4.0
3
+ Version: 0.4.2
4
4
  Summary: QuantDB 量化数据平台官方 Python SDK
5
5
  Author: QuantDB Team
6
6
  License: MIT
@@ -26,7 +26,7 @@ License-File: LICENSE
26
26
  Requires-Dist: requests>=2.25.0
27
27
  Requires-Dist: pandas>=1.3.0
28
28
  Requires-Dist: httpx>=0.24.0
29
- Requires-Dist: duckdb>=0.8.0
29
+ Requires-Dist: duckdb>=1.0.0
30
30
  Requires-Dist: pyarrow>=10.0.0
31
31
  Provides-Extra: dev
32
32
  Requires-Dist: pytest>=7.0.0; extra == "dev"
@@ -1,67 +1,69 @@
1
- [build-system]
2
- requires = ["setuptools>=61.0", "wheel"]
3
- build-backend = "setuptools.build_meta"
4
-
5
- [project]
6
- name = "quantdb-sdk"
7
- version = "0.4.0"
8
- description = "QuantDB 量化数据平台官方 Python SDK"
9
- readme = "README.md"
10
- license = {text = "MIT"}
11
- requires-python = ">=3.8"
12
- authors = [
13
- {name = "QuantDB Team"},
14
- ]
15
- keywords = ["quant", "finance", "data", "a-share", "parquet"]
16
- classifiers = [
17
- "Development Status :: 3 - Alpha",
18
- "Intended Audience :: Developers",
19
- "Intended Audience :: Financial and Insurance Industry",
20
- "License :: OSI Approved :: MIT License",
21
- "Programming Language :: Python :: 3",
22
- "Programming Language :: Python :: 3.8",
23
- "Programming Language :: Python :: 3.9",
24
- "Programming Language :: Python :: 3.10",
25
- "Programming Language :: Python :: 3.11",
26
- "Programming Language :: Python :: 3.12",
27
- "Topic :: Office/Business :: Financial",
28
- ]
29
- dependencies = [
30
- "requests>=2.25.0",
31
- "pandas>=1.3.0",
32
- "httpx>=0.24.0",
33
- "duckdb>=0.8.0",
34
- # 0.1.1 修复:pyarrow 是 pd.read_parquet 的硬依赖(client.py:482, async_client.py:413),
35
- # 但 0.1.0 漏声明,导致 pip install 后首次 K线/Tick 查询会抛 ImportError。
36
- # 锁定到 >=10.0.0 以获得 Python 3.10+ 完整支持(pyarrow 14+ 更好)。
37
- "pyarrow>=10.0.0",
38
- ]
39
-
40
- # 0.1.2 新增:装包后提供 `quantdb` shell 命令,等价于 `python -m quantdb_sdk`。
41
- # 0.1.0/0.1.1 README 写的是 `python -m quantdb`(用了 PyPI 名带连字符),
42
- # 但 Python 模块名是下划线,安装后 `python -m quantdb` 会报 No module named quantdb。
43
- # 加这个 entry point 后两个名字都能用,README 不需要再改。
44
- [project.scripts]
45
- quantdb = "quantdb_sdk.__main__:main"
46
-
47
- [project.optional-dependencies]
48
- dev = [
49
- "pytest>=7.0.0",
50
- "pytest-asyncio>=0.21.0",
51
- "responses>=0.23.0",
52
- "respx>=0.20.0",
53
- ]
54
-
55
- [project.urls]
56
- Homepage = "https://quantdb.quantmind.cloud"
57
- Documentation = "https://quantdb.quantmind.cloud/docs/sdk.html"
58
- Repository = "https://github.com/quantdb/quantdb"
59
- Changelog = "https://github.com/quantdb/quantdb/blob/main/CHANGELOG.md"
60
-
61
- [tool.setuptools.packages.find]
62
- where = ["."]
63
- include = ["quantdb_sdk*"]
64
-
65
- # 0.1.2 新增:让 wheel 包含 LICENSE 和 CHANGELOG.md
66
- [tool.setuptools.package-data]
67
- quantdb_sdk = ["py.typed"]
1
+ [build-system]
2
+ requires = ["setuptools>=61.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "quantdb-sdk"
7
+ version = "0.4.2"
8
+ description = "QuantDB 量化数据平台官方 Python SDK"
9
+ readme = "README.md"
10
+ license = {text = "MIT"}
11
+ requires-python = ">=3.8"
12
+ authors = [
13
+ {name = "QuantDB Team"},
14
+ ]
15
+ keywords = ["quant", "finance", "data", "a-share", "parquet"]
16
+ classifiers = [
17
+ "Development Status :: 3 - Alpha",
18
+ "Intended Audience :: Developers",
19
+ "Intended Audience :: Financial and Insurance Industry",
20
+ "License :: OSI Approved :: MIT License",
21
+ "Programming Language :: Python :: 3",
22
+ "Programming Language :: Python :: 3.8",
23
+ "Programming Language :: Python :: 3.9",
24
+ "Programming Language :: Python :: 3.10",
25
+ "Programming Language :: Python :: 3.11",
26
+ "Programming Language :: Python :: 3.12",
27
+ "Topic :: Office/Business :: Financial",
28
+ ]
29
+ dependencies = [
30
+ "requests>=2.25.0",
31
+ "pandas>=1.3.0",
32
+ "httpx>=0.24.0",
33
+ # 0.4.1 起 client.data.read_local 用 read_parquet(filename=true) + `* EXCLUDE (filename)`
34
+ # 做精确分区读取;0.8.x 在部分环境(Windows/Python 3.10)导入即崩溃,1.0.0~1.5.5 已验证。
35
+ "duckdb>=1.0.0",
36
+ # 0.1.1 修复:pyarrow 是 pd.read_parquet 的硬依赖(client.py:482, async_client.py:413),
37
+ # 但 0.1.0 漏声明,导致 pip install 后首次 K线/Tick 查询会抛 ImportError。
38
+ # 锁定到 >=10.0.0 以获得 Python 3.10+ 完整支持(pyarrow 14+ 更好)。
39
+ "pyarrow>=10.0.0",
40
+ ]
41
+
42
+ # 0.1.2 新增:装包后提供 `quantdb` shell 命令,等价于 `python -m quantdb_sdk`。
43
+ # 0.1.0/0.1.1 README 写的是 `python -m quantdb`(用了 PyPI 名带连字符),
44
+ # 但 Python 模块名是下划线,安装后 `python -m quantdb` 会报 No module named quantdb。
45
+ # 加这个 entry point 后两个名字都能用,README 不需要再改。
46
+ [project.scripts]
47
+ quantdb = "quantdb_sdk.__main__:main"
48
+
49
+ [project.optional-dependencies]
50
+ dev = [
51
+ "pytest>=7.0.0",
52
+ "pytest-asyncio>=0.21.0",
53
+ "responses>=0.23.0",
54
+ "respx>=0.20.0",
55
+ ]
56
+
57
+ [project.urls]
58
+ Homepage = "https://quantdb.quantmind.cloud"
59
+ Documentation = "https://quantdb.quantmind.cloud/docs/sdk.html"
60
+ Repository = "https://github.com/quantdb/quantdb"
61
+ Changelog = "https://github.com/quantdb/quantdb/blob/main/CHANGELOG.md"
62
+
63
+ [tool.setuptools.packages.find]
64
+ where = ["."]
65
+ include = ["quantdb_sdk*"]
66
+
67
+ # 0.1.2 新增:让 wheel 包含 LICENSE 和 CHANGELOG.md
68
+ [tool.setuptools.package-data]
69
+ quantdb_sdk = ["py.typed"]
@@ -5,7 +5,7 @@ try:
5
5
  # 注意:此处用 PyPI 包名(连字符)查找版本号,不是模块名(下划线)
6
6
  __version__ = version("quantdb-sdk")
7
7
  except ImportError:
8
- __version__ = "0.4.0"
8
+ __version__ = "0.4.2"
9
9
 
10
10
  from . import layout
11
11
  from ._utils import scan_local_directory
@@ -19,7 +19,7 @@ SYNC_DATASET_CATEGORIES: Dict[str, str] = {
19
19
  "instrument_detail": "2", "sector_concept": "2", "index_weights": "2",
20
20
  "trading_calendar": "2", "margin_trading": "2",
21
21
  "balance": "3", "income": "3", "cashflow": "3", "capital": "3",
22
- "holder_num": "3", "pershare_index": "3", "dividend_factors": "3",
22
+ "holder_num": "3", "pershare_index": "3", "per_share": "3", "dividend_factors": "3",
23
23
  "etf_pcf": "4", "convertible_bond": "4", "etf_kline": "4",
24
24
  "valuation": "5", "technical_indicators": "5", "market_sentiment": "5",
25
25
  "features_daily": "6", "l1_factors": "6", "l1_l2_factors": "6", "l2_factors": "6",
@@ -28,7 +28,7 @@ from .errors import (
28
28
  )
29
29
 
30
30
  # 维护提示:每次发版必须与 pyproject.toml 版本号同步
31
- _USER_AGENT = "QuantDB-Python-SDK/0.4.0"
31
+ _USER_AGENT = "QuantDB-Python-SDK/0.4.2"
32
32
 
33
33
 
34
34
  def _etag_hex(value: Optional[str]) -> Optional[str]:
@@ -32,7 +32,7 @@ from .errors import (
32
32
  )
33
33
 
34
34
  # 维护提示:每次发版必须与 pyproject.toml 版本号同步
35
- _USER_AGENT = "QuantDB-Python-SDK/0.4.0"
35
+ _USER_AGENT = "QuantDB-Python-SDK/0.4.2"
36
36
 
37
37
 
38
38
  def _etag_hex(value: Optional[str]) -> Optional[str]:
@@ -23,12 +23,14 @@
23
23
  ``tick_shard``),用于 ``query()`` 路由;不用 :func:`quantdb_sdk.layout.dataset_layout`
24
24
  (其 V2 集合已过期,缺 min1/min5/etf/tick)。
25
25
  - 本地读取用**多形态 glob** 与目录布局集合解耦,支持显式 ``root``(数据根目录可配置,
26
- 非固定目录;缺省读 :func:`quantdb_sdk.layout.data_root`)。
26
+ 非固定目录;缺省读 :func:`quantdb_sdk.layout.data_root`);读取本身交给 DuckDB 对
27
+ 精确文件列表做一次 ``read_parquet``(列裁剪 / symbol / 日期下推),不逐文件拼接。
27
28
  """
28
29
 
29
30
  import glob
30
31
  import os
31
32
  import re
33
+ import threading
32
34
  from dataclasses import dataclass, field
33
35
  from typing import Any, Dict, List, Optional, Tuple
34
36
 
@@ -52,6 +54,16 @@ _KLINE_ADJ_TYPES = {
52
54
  # V1 逐股票表的日期列候选(财务表多为 m_timetag;holder_num 为 endDate;l1_l2_factors 为 date)
53
55
  _V1_DATE_COLUMNS = ("time", "trade_date", "date", "_filename_date", "m_timetag", "endDate")
54
56
 
57
+ # 从路径推导 trade_date 的 DuckDB 表达式(V2 用 dt= 目录名;tick 用文件名 _YYYYMMDD)。
58
+ # 统一从 filename 提取,不依赖 hive 注入的 dt 列——dt 列仅在较新 DuckDB 自动出现,
59
+ # 低版本(如 1.0.x)不存在,会使 `EXCLUDE (dt)` / `CAST(dt ...)` 直接报错。
60
+ _V2_DATE_EXPR = (
61
+ "strftime(strptime(regexp_extract(filename, 'dt=(\\d{8})', 1), '%Y%m%d'), '%Y-%m-%d')"
62
+ )
63
+ _TICK_DATE_EXPR = (
64
+ "strftime(strptime(regexp_extract(filename, '_(\\d{8})\\.parquet$', 1), '%Y%m%d'), '%Y-%m-%d')"
65
+ )
66
+
55
67
 
56
68
  @dataclass(frozen=True)
57
69
  class DatasetInfo:
@@ -121,6 +133,8 @@ _ENTRIES: Tuple[Tuple[str, str, str, str, Tuple[str, ...], str, Tuple[str, ...]]
121
133
  ("股东", "户数", "holder"), "v1_symbol", ("symbol",)),
122
134
  ("pershare_index", "每股指标", "每股收益、净资产等指标。", "free",
123
135
  ("每股", "eps", "财务"), "v1_symbol", ("symbol",)),
136
+ ("per_share", "每股指标(补充)", "每股指标补充表;部分股票为空。", "free",
137
+ ("每股", "eps", "补充", "per_share"), "v1_symbol", ("symbol",)),
124
138
  ("dividend_factors", "除权复权因子", "分红送配及复权因子。", "free",
125
139
  ("分红", "除权", "复权", "dividend"), "v1_symbol", ("symbol",)),
126
140
  ("etf_kline", "ETF 日线", "ETF 日线 OHLCV。", "free",
@@ -315,6 +329,41 @@ def _local_path(
315
329
  return files[-1]
316
330
 
317
331
 
332
+ # DuckDB 连接(线程内复用):精确文件列表查询用裸连接即可,无需挂载 glob 视图。
333
+ # 参考实现 quantmindoss QuantDBDataHub._exact_conn 的结论:复用视图连接会为每个
334
+ # 新线程重建视图,首查开销极大;精确路径查询不需要视图。
335
+ _DUCK_LOCAL = threading.local()
336
+
337
+
338
+ def _duck_conn():
339
+ """返回当前线程复用的 DuckDB 连接(懒建)。"""
340
+ conn = getattr(_DUCK_LOCAL, "conn", None)
341
+ if conn is None:
342
+ try:
343
+ import duckdb # type: ignore
344
+ except ImportError as exc:
345
+ raise RuntimeError("本地零流量读取需要 duckdb:pip install duckdb") from exc
346
+ conn = duckdb.connect()
347
+ _DUCK_LOCAL.conn = conn
348
+ return conn
349
+
350
+
351
+ def _sql_file_list(files: List[str]) -> str:
352
+ """把文件列表转成 ``read_parquet([...])`` 的 SQL 字面量(/ 分隔、转义单引号)。"""
353
+ return ", ".join("'" + p.replace("\\", "/").replace("'", "''") + "'" for p in files)
354
+
355
+
356
+ def _exec_df(sql: str, params: Optional[List[Any]] = None) -> pd.DataFrame:
357
+ return _duck_conn().execute(sql, params or []).fetchdf()
358
+
359
+
360
+ def _detect_local_date_column(files: List[str]) -> Optional[str]:
361
+ """用 DuckDB 元数据(``LIMIT 0``,不读行数据)探测 V1 文件的日期列。"""
362
+ sql = f"SELECT * FROM read_parquet([{_sql_file_list(files)}], union_by_name=true) LIMIT 0"
363
+ names = [col[0] for col in _duck_conn().execute(sql).description]
364
+ return next((c for c in _V1_DATE_COLUMNS if c in names), None)
365
+
366
+
318
367
  def _read_local(
319
368
  info: DatasetInfo,
320
369
  symbol: Optional[str] = None,
@@ -325,52 +374,86 @@ def _read_local(
325
374
  root: Optional[str] = None,
326
375
  latest: bool = False,
327
376
  ) -> pd.DataFrame:
328
- """读取本地 Parquet(零流量)。V2/tick 无日期列时从路径注入 ``trade_date``。"""
377
+ """读取本地 Parquet(零流量)。
378
+
379
+ 先用多形态 glob 定位**所需分区**(只枚举目录名 / 文件名,不读文件内容),再交给
380
+ DuckDB 对精确文件列表做**一次** ``read_parquet``,并把 symbol / 日期 / 列裁剪
381
+ 下推到 SQL——不再逐文件 ``pd.read_parquet`` 再 ``concat``。
382
+
383
+ V2 按日分区与 tick 分片文件内无日期列,``trade_date`` 由分区目录名 / 文件名推导。
384
+ """
329
385
  files = _local_files(info, symbol, trade_date, root, start_date, end_date)
330
386
  if not files:
331
387
  raise NotFoundError(f"本地未找到数据集 {info.dataset} 的文件(root={_local_root(root)})")
332
388
  if latest:
333
389
  files = [files[-1]]
334
390
 
335
- need_date_filter = info.layout == "v1_symbol" and bool(start_date or end_date)
336
- frames: List[pd.DataFrame] = []
337
- for path in files:
338
- # V1 需要日期列做区间过滤,先整表读;其余按 columns 下推
339
- df = pd.read_parquet(path) if need_date_filter else pd.read_parquet(path, columns=columns)
340
- if info.layout == "v2_daily_partition":
341
- td = _v2_trade_date(path)
342
- if td and "trade_date" not in df.columns:
343
- df = df.copy()
344
- df["trade_date"] = td
345
- elif info.layout == "tick_shard":
346
- td = _tick_trade_date(path)
347
- if td and "trade_date" not in df.columns:
348
- df = df.copy()
349
- df["trade_date"] = td
350
- frames.append(df)
391
+ if info.layout == "v1_symbol":
392
+ return _read_local_v1(files, start_date, end_date, columns)
393
+ return _read_local_partitioned(info, files, symbol, columns)
351
394
 
352
- df = frames[0] if len(frames) == 1 else pd.concat(frames, ignore_index=True, sort=False)
353
395
 
354
- # tick / V2 全市场分区需按 symbol 后置过滤
355
- if symbol and "symbol" in df.columns:
356
- df = df[df["symbol"].astype(str).str.upper() == str(symbol).strip().upper()].copy()
396
+ def _read_local_partitioned(
397
+ info: DatasetInfo,
398
+ files: List[str],
399
+ symbol: Optional[str],
400
+ columns: Optional[List[str]],
401
+ ) -> pd.DataFrame:
402
+ """V2 按日分区 / tick 分片:``trade_date`` 由路径推导,读取与过滤下推到 DuckDB。"""
403
+ if info.layout == "v2_daily_partition":
404
+ date_expr, order = _V2_DATE_EXPR, " ORDER BY trade_date"
405
+ else:
406
+ date_expr, order = _TICK_DATE_EXPR, ""
407
+
408
+ if columns:
409
+ wanted = [c for c in columns if c != "trade_date"]
410
+ projected = ", ".join(f'"{c}"' for c in wanted)
411
+ select = f"{projected}, {date_expr} AS trade_date" if projected else f"{date_expr} AS trade_date"
412
+ else:
413
+ select = f"* EXCLUDE (filename), {date_expr} AS trade_date"
414
+
415
+ where: List[str] = []
416
+ params: List[Any] = []
417
+ if symbol:
418
+ where.append("UPPER(symbol) = ?")
419
+ params.append(str(symbol).strip().upper())
420
+
421
+ sql = f"SELECT {select} FROM read_parquet([{_sql_file_list(files)}], union_by_name=true, filename=true)"
422
+ if where:
423
+ sql += " WHERE " + " AND ".join(where)
424
+ sql += order
425
+ return _exec_df(sql, params).reset_index(drop=True)
426
+
427
+
428
+ def _read_local_v1(
429
+ files: List[str],
430
+ start_date: Optional[str],
431
+ end_date: Optional[str],
432
+ columns: Optional[List[str]],
433
+ ) -> pd.DataFrame:
434
+ """V1 逐股票 / 单文件:日期列在文件内,用归一化 YYYYMMDD 字符串比较下推过滤。
357
435
 
358
- if need_date_filter:
359
- date_col = _detect_date_column(df)
436
+ ``substr(replace(CAST(col AS VARCHAR),'-',''),1,8)`` 同时兼容 ``20260331``(整型
437
+ 报告期)与 ``2025-12-31`` / ``2025-12-31 09:30:00``(字符串),无需先探测列类型。
438
+ """
439
+ where: List[str] = []
440
+ params: List[Any] = []
441
+ if start_date or end_date:
442
+ date_col = _detect_local_date_column(files)
360
443
  if date_col:
361
- dt = pd.to_datetime(df[date_col], errors="coerce")
362
- mask = pd.Series(True, index=df.index)
444
+ norm = f"substr(replace(CAST(\"{date_col}\" AS VARCHAR), '-', ''), 1, 8)"
363
445
  if start_date:
364
- mask &= dt >= pd.to_datetime(_norm_date(start_date))
446
+ where.append(f"{norm} >= ?")
447
+ params.append(_yyyymmdd(start_date))
365
448
  if end_date:
366
- mask &= dt <= pd.to_datetime(_norm_date(end_date))
367
- df = df[mask].copy()
368
- if columns:
369
- df = df.loc[:, [c for c in columns if c in df.columns]]
449
+ where.append(f"{norm} <= ?")
450
+ params.append(_yyyymmdd(end_date))
370
451
 
371
- if info.layout == "v2_daily_partition" and "trade_date" in df.columns:
372
- df = df.sort_values("trade_date", kind="stable")
373
- return df.reset_index(drop=True)
452
+ select = ", ".join(f'"{c}"' for c in columns) if columns else "*"
453
+ sql = f"SELECT {select} FROM read_parquet([{_sql_file_list(files)}], union_by_name=true)"
454
+ if where:
455
+ sql += " WHERE " + " AND ".join(where)
456
+ return _exec_df(sql, params).reset_index(drop=True)
374
457
 
375
458
 
376
459
  # ---------------------------------------------------------------------------
@@ -478,8 +561,11 @@ class DataHub:
478
561
  ) -> pd.DataFrame:
479
562
  """读取本地已落地的 Parquet(零流量)。
480
563
 
481
- - ``start_date/end_date`` 先按 ``dt=`` 目录名 / tick 文件名裁剪文件,再读取;
482
- - V2 与 tick 文件无日期列时,``trade_date`` 从路径注入;
564
+ - ``start_date/end_date`` 先按 ``dt=`` 目录名 / tick 文件名裁剪文件(只枚举目录,
565
+ 不读内容),再交给 DuckDB 对精确文件列表做一次 ``read_parquet``;
566
+ - ``symbol`` / ``columns`` / 日期过滤全部下推到 SQL(宽表只读需要的列),
567
+ 避免逐文件 ``pd.read_parquet`` 再 ``concat``;
568
+ - V2 与 tick 文件无日期列时,``trade_date`` 由分区目录名 / 文件名推导;
483
569
  - ``latest=True`` 只读最新一个文件;
484
570
  - 未命中本地文件时抛 ``NotFoundError``(不自动走在线,避免静默消耗流量)。
485
571
  """
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: quantdb-sdk
3
- Version: 0.4.0
3
+ Version: 0.4.2
4
4
  Summary: QuantDB 量化数据平台官方 Python SDK
5
5
  Author: QuantDB Team
6
6
  License: MIT
@@ -26,7 +26,7 @@ License-File: LICENSE
26
26
  Requires-Dist: requests>=2.25.0
27
27
  Requires-Dist: pandas>=1.3.0
28
28
  Requires-Dist: httpx>=0.24.0
29
- Requires-Dist: duckdb>=0.8.0
29
+ Requires-Dist: duckdb>=1.0.0
30
30
  Requires-Dist: pyarrow>=10.0.0
31
31
  Provides-Extra: dev
32
32
  Requires-Dist: pytest>=7.0.0; extra == "dev"
@@ -1,7 +1,7 @@
1
1
  requests>=2.25.0
2
2
  pandas>=1.3.0
3
3
  httpx>=0.24.0
4
- duckdb>=0.8.0
4
+ duckdb>=1.0.0
5
5
  pyarrow>=10.0.0
6
6
 
7
7
  [dev]
@@ -55,7 +55,7 @@ def _write_tick(root, symbol, yyyymmdd, flat=True, extra_symbol=False):
55
55
  # ---------------------------------------------------------------------------
56
56
 
57
57
  def test_registry_completeness():
58
- assert len(DATASET_REGISTRY) == 29
58
+ assert len(DATASET_REGISTRY) == 30
59
59
  assert set(DATASET_REGISTRY) == set(SYNC_DATASET_CATEGORIES)
60
60
  for dataset, info in DATASET_REGISTRY.items():
61
61
  assert info.dataset == dataset
@@ -79,13 +79,21 @@ def test_data_property_is_lazy_and_cached(client):
79
79
  def test_datasets_filters(client):
80
80
  hub = client.data
81
81
  all_df = hub.datasets()
82
- assert len(all_df) == 29
82
+ assert len(all_df) == 30
83
83
  assert list(all_df.columns) == ["dataset", "category_id", "category_dir", "name", "access", "layout"]
84
84
 
85
85
  cat6 = hub.datasets(category="6")
86
86
  assert set(cat6["dataset"]) == {"features_daily", "l1_factors", "l2_factors", "l1_l2_factors"}
87
87
  assert (cat6["access"] == "subscription").all()
88
88
 
89
+ cat3 = hub.datasets(category="3")
90
+ assert set(cat3["dataset"]) == {
91
+ "balance", "income", "cashflow", "capital", "holder_num",
92
+ "pershare_index", "per_share", "dividend_factors",
93
+ }
94
+ # per_share 为 V1 逐股票补充表,需能按 symbol 读取
95
+ assert hub.datasets(keyword="per_share").iloc[0]["dataset"] == "per_share"
96
+
89
97
  assert len(hub.datasets(access="subscription")) == 4
90
98
  assert "daily_forward" in set(hub.datasets(keyword="复权")["dataset"])
91
99
 
@@ -250,6 +258,32 @@ def test_read_local_v1_date_filter(tmp_path, client):
250
258
  assert len(df) == 2
251
259
 
252
260
 
261
+ def test_read_local_v1_int_report_period_filter(tmp_path, client):
262
+ """V1 整型报告期(如 m_timetag=20251231)也要能被日期区间正确过滤。"""
263
+ root = str(tmp_path)
264
+ path = tmp_path / "3_financial_data" / "income" / "600519.SH.parquet"
265
+ path.parent.mkdir(parents=True)
266
+ pd.DataFrame({
267
+ "Symbol": ["600519.SH"] * 3,
268
+ "m_timetag": [20251231, 20250930, 20250630],
269
+ "x": [1, 2, 3],
270
+ }).to_parquet(path)
271
+ df = client.data.read_local("income", symbol="600519.SH", root=root,
272
+ start_date="2025-09-01", end_date="2025-12-31")
273
+ assert len(df) == 2
274
+
275
+
276
+ def test_read_local_v2_columns_projection_and_symbol_pushdown(tmp_path, client):
277
+ """V2:列裁剪下推 + symbol 下推(symbol 不在 columns 里也照样过滤)。"""
278
+ root = str(tmp_path)
279
+ _write_v2(root, "daily_forward", ["20260901"])
280
+ df = client.data.read_local("daily_forward", symbol="600519.SH", columns=["close"], root=root)
281
+ assert list(df.columns) == ["close", "trade_date"]
282
+ assert len(df) == 1
283
+ assert df.iloc[0]["close"] == 1.0
284
+ assert df.iloc[0]["trade_date"] == "2026-09-01"
285
+
286
+
253
287
  def test_local_path_precise_hit(tmp_path, client):
254
288
  root = str(tmp_path)
255
289
  _write_v2(root, "daily_forward", ["20260901"])
File without changes
File without changes
File without changes
File without changes