alpha-dc-client 0.2.0__tar.gz → 0.2.2__tar.gz

{alpha_dc_client-0.2.0 → alpha_dc_client-0.2.2}/.gitignore

@@ -163,3 +163,7 @@ ic_backfill_*.err
 # 港股回补运行时状态（断点续传，本地产物）
 dc_quant/scripts/_hk_*_progress.json
 dc_quant/scripts/check/_tmp_*
+dc_quant/scripts/_us_edgar_repull_progress.json
+dc_quant/scripts/check/_layer_reconcile_done.json
+dc_quant/scripts/_cn_index_full_progress.json
+dc_quant/scripts/_etf_fundamentals_progress.json

{alpha_dc_client-0.2.0 → alpha_dc_client-0.2.2}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.5
 Name: alpha-dc-client
-Version: 0.2.0
+Version: 0.2.2
 Author-email: Hang GuangLiang <hanguangliang@alphaaidig.com>
-Requires-Dist: alpha-dc-common==0.1.24
-Requires-Dist: httpx
-Requires-Dist: python-dotenv
+Requires-Dist: alpha-dc-common>=0.1.26
+Requires-Dist: requests
 Provides-Extra: parquet
 Requires-Dist: duckdb>=0.10; extra == 'parquet'
 Requires-Dist: pandas>=2.0; extra == 'parquet'
 Requires-Dist: paramiko>=3.0; extra == 'parquet'
+Requires-Dist: python-dotenv; extra == 'parquet'

alpha_dc_client-0.2.2/pyproject.toml

@@ -0,0 +1,33 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "alpha-dc-client"
+version = "0.2.2"  # common 锁定放宽为 >=（内核可独立发版）
+authors = [
+    {name = "Hang GuangLiang", email = "hanguangliang@alphaaidig.com"},
+]
+dependencies = [
+    # client.py 直接使用 requests（POST + Timeout/ConnectionError 异常处理），
+    # workspace 内曾靠 tushare 等传递带入未暴露，独立安装必炸，必须显式声明
+    "requests",
+    "alpha-dc-common>=0.1.26"  # 内核独立演进发版，client 不强制随发  # +strategies 内核,
+]
+
+[project.optional-dependencies]
+# 因子 Parquet 同步与读取（dc_client.data）：duckdb 读取 + paramiko sftp 兜底通道
+parquet = [
+    "duckdb>=0.10",
+    "pandas>=2.0",
+    "paramiko>=3.0",
+    "python-dotenv",  # data/config.py 懒加载 .env
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dc_client"]
+# 测试与 run_tests 不进发布产物（tests/base.py 含硬编码 API token，绝不随 wheel 外发）
+exclude = ["src/dc_client/tests", "src/dc_client/run_tests.py"]
+
+[tool.uv.sources]
+alpha-dc-common = { workspace = true }

{alpha_dc_client-0.2.0 → alpha_dc_client-0.2.2}/src/dc_client/data/discovery.py

@@ -235,6 +235,7 @@ def effective_registry(cfg: SyncConfig | None = None) -> Registry:
                 date_column=bundled_meta.date_column,
                 entity_column=bundled_meta.entity_column,
                 tier=bundled_meta.tier,
+                month_continuity=bundled_meta.month_continuity,
             )
         datasets[name] = meta
 

{alpha_dc_client-0.2.0 → alpha_dc_client-0.2.2}/src/dc_client/data/io.py

@@ -172,6 +172,8 @@ def _sql_str_list(paths: list[Path]) -> str:
 def _check_month_continuity(files: list[Path], dataset: str) -> None:
     """月份连续性（P0-3）：本地覆盖区间内的中间月缺失即抛错——静默丢整月数据。
 
+    仅适用于日频表。稀疏事件/报告期表（month_continuity="skip"）走
+    _check_sparse_files（DC05, review 260918）。
     区间外（数据开始前/结束后）的月份不要求（新 dataset 起点晚是正常的）。
     """
     months = sorted(int(m.group(1)) for f in files
@@ -183,6 +185,8 @@ def _check_month_continuity(files: list[Path], dataset: str) -> None:
         nxt = cur + 1 if cur % 100 < 12 else (cur // 100 + 1) * 100 + 1
         while nxt < ym:
             gaps.append(nxt)
+            # 12 月跳月必须基于 nxt 自身推进——用 cur 会在缺口跨年时
+            # 把 nxt 跳回 cur 的下一年 1 月，形成死循环（DC05 排查时发现）
             nxt = nxt + 1 if nxt % 100 < 12 else (nxt // 100 + 1) * 100 + 1
         cur = ym
     if gaps:
@@ -192,24 +196,127 @@ def _check_month_continuity(files: list[Path], dataset: str) -> None:
             f"请 sync 修复或检查数据")
 
 
+def _check_sparse_files(
+    dataset: str, cfg: SyncConfig, start: date | None = None, end: date | None = None,
+) -> None:
+    """稀疏事件/报告期表的完整性校验（DC05，review 260918）。
+
+    事件表（分红/持仓披露等）合法无事件月不落文件，月份连续性检查会把
+    全历史读取误判为缺月。改为对照 sync manifest：manifest 记录过"已同步"
+    的文件若在本地消失，才是真正丢文件，报错；manifest 无记录（未 sync 过
+    或旧版本账本）时只放行——远端清单校验是 sync 的职责。
+    """
+    manifest_path = cfg.resolved_local_root() / ".manifest.json"
+    if not manifest_path.exists():
+        return
+    import json as _json
+
+    try:
+        manifest = _json.loads(manifest_path.read_text(encoding="utf-8"))
+    except (OSError, ValueError) as exc:
+        logger.warning("%s: manifest 损坏，跳过稀疏表文件校验: %s", dataset, exc)
+        return
+    root = cfg.resolved_local_root()
+    expected = {
+        rel for rel in manifest
+        if isinstance(rel, str) and rel.startswith(dataset + "/")
+        and (root / rel).suffix == ".parquet"
+    }
+    def in_window(rel: str) -> bool:
+        match = re.search(r"(\d{6})(?=\.parquet$)", Path(rel).name)
+        if match is None:
+            return True  # cannot prove this publication is outside the request
+        ym = int(match.group(1))
+        return ((start is None or ym >= start.year * 100 + start.month)
+                and (end is None or ym <= end.year * 100 + end.month))
+
+    missing = sorted(rel for rel in expected if in_window(rel) and not (root / rel).is_file())
+    if missing:
+        raise FileNotFoundError(
+            f"{dataset}: sync manifest 记录的已同步文件本地缺失 {missing[:6]}"
+            f"{'...' if len(missing) > 6 else ''}——事件表无事件月不落文件是正常的，"
+            f"但已发布文件丢失必须报错，请重新 sync")
+
+
 def _d(v: date | str | None) -> date | None:
     if v is None or isinstance(v, date):
         return v
     return date.fromisoformat(v)
 
 
+_IDENT_RE = None  # 惰性编译的 where 列名提取正则
+
+
+def _where_columns(where: str) -> set[str]:
+    """提取 where 表达式引用的裸列名（不含引号字符串/数字/函数/操作符）。
+
+    用于判断 where 能否安全下推到某层：只提取 [A-Za-z_][A-Za-z0-9_.]* 形态的
+    标识符，排除带引号字符串字面量；带表别名/点路径的取末段。
+    """
+    global _IDENT_RE
+    import re as _re
+    if _IDENT_RE is None:
+        _IDENT_RE = _re.compile(r'"([A-Za-z_][A-Za-z0-9_ ]*)"'  # 双引号列名
+                                r"|'[^']*'"                      # 字符串字面量（丢弃）
+                                r'|\b([A-Za-z_][A-Za-z0-9_]*)\b')  # 裸标识符
+    cols: set[str] = set()
+    text = where
+    for m in _IDENT_RE.finditer(text):
+        if m.group(1):
+            cols.add(m.group(1))
+            text = text.replace(m.group(0), " ", 1)  # 双引号列已消费，防再匹配
+        elif m.group(2):
+            cols.add(m.group(2))
+    sql_keywords = {"and", "or", "not", "in", "is", "null", "between", "like",
+                    "true", "false", "date", "timestamp"}
+    return {c for c in cols if c.lower() not in sql_keywords}
+
+
+def _layer_pushable_where(meta: "LayerMeta", hits: list[str], where: str | None) -> str | None:
+    """where 仅在引用列全部为该层可用列时下推（否则跳过，由外层 JOIN 后过滤）。
+
+    可用列 = 键列 + 命中列 + 层已声明物理列（columns 已知时）。列未知的层
+    不下推 where（保守，正确性优先）——修复跨层筛选把 pe 条件推进只有
+    momentum 的层导致的 DuckDB BinderException（review 2026-09-14 P2）。
+    """
+    if not where:
+        return None
+    available: set[str] = set(hits)
+    if meta.date_column:
+        available.add(meta.date_column)
+    if meta.entity_column:
+        available.add(meta.entity_column)
+    if meta.columns:
+        available.update(meta.columns)
+    if not available - {meta.date_column, meta.entity_column}:
+        return None  # 列未知（本机无 footer）——不下推
+    refs = _where_columns(where)
+    return where if refs <= available else None
+
+
 def _date_conditions(meta: LayerMeta, start: date | None,
                      end: date | None, where: str | None) -> str:
+    # DC06（review 260918）：历史月文件 trade_date 可能是 VARCHAR（旧写入路径），
+    # 与 DATE 字面量直接比较会抛 Binder 异常。CAST 到 DATE 统一谓词类型；
+    # 解析失败会响亮报错（缺失必须报告，不静默丢行），ISO 字符串无副作用。
+    date_expr = f"CAST({meta.date_column} AS DATE)"
     conds = []
     if start:
-        conds.append(f"{meta.date_column} >= DATE '{start.isoformat()}'")
+        conds.append(f"{date_expr} >= DATE '{start.isoformat()}'")
     if end:
-        conds.append(f"{meta.date_column} <= DATE '{end.isoformat()}'")
+        conds.append(f"{date_expr} <= DATE '{end.isoformat()}'")
     if where:
         conds.append(f"({where})")
     return " AND ".join(conds)
 
 
+def _dated_source(files: list[Path], meta: LayerMeta) -> str:
+    """Normalize the date before user predicates, joins and projections."""
+    col = '"' + meta.date_column.replace('"', '""') + '"'
+    return (f"(SELECT * REPLACE (CAST({col} AS DATE) AS {col}) "
+            f"FROM read_parquet({_sql_str_list(files)}, union_by_name=true))")
+
+
 def load_dataset(
     dataset: str,
     start: date | str | None = None,
@@ -232,6 +339,10 @@ def load_dataset(
     if meta is None:
         meta = _meta(dataset, cfg)
     start_d, end_d = _d(start), _d(end)
+    if meta.month_continuity == "skip":
+        # Even on_missing=empty cannot turn a known lost publication into a
+        # legitimate month with no events. Check before the no-files return.
+        _check_sparse_files(dataset, cfg, start_d, end_d)
     files = _month_files(
         cfg.resolved_local_root() / dataset / "monthly", start_d, end_d,
         allow_missing=(on_missing == "empty"),
@@ -240,7 +351,8 @@ def load_dataset(
         if on_missing == "empty":
             return pd_empty(columns, meta.date_column)
         raise FileNotFoundError(f"{dataset}: 日期范围内无本地数据文件")
-    _check_month_continuity(files, dataset)
+    if meta.month_continuity != "skip":
+        _check_month_continuity(files, dataset)
 
     cols = list(columns) if columns else None
     if cols and meta.date_column not in cols:
@@ -248,10 +360,11 @@ def load_dataset(
     select = ", ".join(f'"{c}"' for c in cols) if cols else "*"
     cond = _date_conditions(meta, start_d, end_d, where)
     where_sql = f" WHERE {cond}" if cond else ""
+    # DC06：ORDER BY 同样 CAST——union_by_name 下混型日期列（string/date32）
+    # 直接排序会和 WHERE 一样抛 Binder 异常
     sql = (
-        f"SELECT {select} FROM read_parquet("
-        f"{_sql_str_list(files)}, union_by_name=true){where_sql} "
-        f"ORDER BY {meta.date_column}"
+        f"SELECT {select} FROM {_dated_source(files, meta)}{where_sql} "
+        f"ORDER BY CAST({meta.date_column} AS DATE)"
         + (f", {meta.entity_column}" if meta.entity_column else "")
     )
     return _normalize_frame(_apply_dtype_contract(
@@ -383,24 +496,30 @@ def load_view(
         LogicalFactorView 语义：filters 由各层文件 schema 自行解析，
         支持引用各层自有列，如 close 在量价层）。
         """
+        if meta.month_continuity == "skip":
+            _check_sparse_files(meta.name, cfg, start_d, end_d)
         files = _month_files(root / meta.name / "monthly", start_d, end_d,
                              allow_missing=True)
         if not files:
             return None
+        # C02/DC-READ-01：参与 JOIN 的每层都查月份连续性（此前只查驱动层）——
+        # 中间缺月会被 LEFT JOIN 静默变 NULL，业务缺值与文件未同步不可区分。
+        # _check_month_continuity 只查现有文件集内部空洞，层合法晚起不误判
+        if meta.month_continuity != "skip":
+            _check_month_continuity(files, meta.name)
         hits = hits_map.get(meta.name, [])
-        cond = _date_conditions(meta, start_d, end_d, where)
+        # P2 修复（review 2026-09-14）：where 只下推到列覆盖的层做剪枝；
+        # 不覆盖的层仅日期裁剪，过滤条件由外层 JOIN 后统一应用
+        cond = _date_conditions(meta, start_d, end_d,
+                                _layer_pushable_where(meta, hits, where))
         w = f" WHERE {cond}" if cond else ""
-        inner = (f"SELECT * FROM read_parquet({_sql_str_list(files)}, "
-                 f"union_by_name=true){w}")
+        inner = f"SELECT * FROM {_dated_source(files, meta)}{w}"
         return inner, hits
 
     # 驱动层固定 = 视图注册首层（与 LogicalFactorView 的 base 语义一致）：
     # 行集稳定不随投影变化；首层**必需**——缺文件时 empty 模式返回空表、
     # raise 模式报错，绝不静默换驱动层（换驱动=换行集）。
     base_meta = layer_metas.get(view.layers[0])
-    _check_month_continuity(
-        _month_files(root / view.layers[0] / "monthly", start_d, end_d,
-                     allow_missing=True), view.layers[0])
     base_sub = _layer_sub(base_meta) if base_meta is not None else None
     if base_sub is None:
         if on_missing == "empty":
@@ -411,8 +530,13 @@ def load_view(
     # P1-8: 只请求键列 → 直接返回驱动层键行集（对齐旧实现）
     if not wanted:
         drive_sql = base_sub[0]
-        sql = f"SELECT {', '.join(key_cols)} FROM ({drive_sql}) a ORDER BY {', '.join(key_cols)}"
-        return _normalize_frame(_conn().sql(sql).df())
+        # DC06：日期键排序 CAST，防混型月文件 Binder 异常
+        order_exprs = [f"CAST({c} AS DATE)" if c == anchor.date_column else c
+                       for c in key_cols]
+        sql = (f"SELECT {', '.join(key_cols)} FROM ({drive_sql}) a "
+               f"ORDER BY {', '.join(order_exprs)}")
+        return _normalize_frame(_apply_dtype_contract(
+            _conn().sql(sql).df(), {anchor.date_column: "date32"}))
 
     # 命中层（非驱动）缺文件必须抛错（对齐旧 MissingLayerError 语义）：
     # 静默跳过会让结果缺请求列——批量因子读取静默减因子，比报错危险。
@@ -442,7 +566,14 @@ def load_view(
         select_parts.extend(f"{alias}.{c}" for c in hits)
 
     sql = f"SELECT {', '.join(select_parts)} FROM ({drive_sql}) a{joins}"
-    sql += f" ORDER BY {', '.join(key_cols)}"
+    # 外层统一过滤（P2）：层内下推只是剪枝优化，语义过滤在这里兜底——
+    # 跨层 where（如 pe 在 t1、momentum 在驱动层）此前既炸层又无外层应用
+    if where:
+        sql += f" WHERE ({where})"
+    # DC06：日期键排序 CAST，防混型月文件 Binder 异常
+    order_exprs = [f"CAST({c} AS DATE)" if c == anchor.date_column else c
+                   for c in key_cols]
+    sql += f" ORDER BY {', '.join(order_exprs)}"
     # 契约并集先到先得（与列归属同序：归属层的契约生效）
     view_dtypes: dict[str, str] = {}
     for name in view.layers: