sino-bc-cli 1.8.22 → 1.8.24

package/dist/skills/custom/biz-line-strategy-generate/scripts/cli_json.py

@@ -142,10 +142,39 @@ def parse_json_text(raw: str):
         return None
 
 
-def load_json_file(path: Path):
+def load_json_file(path: Path, drop_page_rows: bool = False):
+    """从文件解析 JSON。优先 json.load，避免先整文件读成 str 再 loads。
+
+    drop_page_rows：分页信封在 object_hook 里记下行数并丢掉行 list，调用方拿不到整页行。
+    解析失败只读文件头，不把后面埋着的 JSON 当成功回包。
+    """
     if not path.exists():
         return None
-    return parse_json_text(path.read_text(encoding="utf-8"))
+
+    def hook(obj):
+        if isinstance(obj.get("data"), list) and obj.get("total") is not None:
+            obj["_page_got"] = len(obj["data"])
+            obj["data"] = []
+        if isinstance(obj.get("result"), list) and obj.get("total") is not None:
+            obj["_page_got"] = obj.get("_page_got") or len(obj["result"])
+            obj["result"] = []
+        return obj
+
+    try:
+        with path.open(encoding="utf-8") as handle:
+            if drop_page_rows:
+                return json.load(handle, object_hook=hook)
+            return json.load(handle)
+    except json.JSONDecodeError:
+        return parse_json_text(read_text_head(path))
+
+
+def read_text_head(path: Path, limit: int = 2000) -> str:
+    """失败摘要只读文件头部，不把整页 stdout 再读进内存。"""
+    if not path.exists():
+        return ""
+    with path.open(encoding="utf-8", errors="replace") as handle:
+        return handle.read(limit)
 
 
 def unwrap_page(envelope: dict) -> tuple[list, int]:
@@ -171,8 +200,23 @@ def unwrap_page(envelope: dict) -> tuple[list, int]:
     return [], 0
 
 
-def exec_cli_to_file(cmd: list[str], out_path: Path) -> tuple[int, str]:
-    """两个输出流直写本次文件；取消时等待子进程退出后才允许清理。"""
+def take_page_stats(envelope: dict) -> tuple[int, int]:
+    """取出本页行数与 total，并丢掉行 list，让整页数据可被回收。"""
+    rows, total = unwrap_page(envelope)
+    got = len(rows)
+    data = envelope.get("data")
+    if isinstance(data, dict):
+        if isinstance(data.get("data"), list):
+            data["data"] = []
+        if isinstance(data.get("result"), list):
+            data["result"] = []
+    elif isinstance(data, list):
+        envelope["data"] = []
+    return got, total
+
+
+def exec_cli_to_file(cmd: list[str], out_path: Path) -> int:
+    """两个输出流直写本次文件，只返回退出码，不把 stdout 读回内存。"""
     out_path.parent.mkdir(parents=True, exist_ok=True)
     cancel = getattr(_local, "cancel", None)
     if cancel and cancel.is_set():
@@ -195,8 +239,7 @@ def exec_cli_to_file(cmd: list[str], out_path: Path) -> tuple[int, str]:
                 except subprocess.TimeoutExpired:
                     proc.kill()
                     proc.wait()
-    raw = out_path.read_text(encoding="utf-8")
-    return proc.returncode, raw
+    return proc.returncode
 
 
 def list_query_body(*, name: str = "", biz_id: int = 0) -> dict:
@@ -215,7 +258,7 @@ def list_query_body(*, name: str = "", biz_id: int = 0) -> dict:
     return body
 
 
-def run_cli(args: list[str], body: dict | None, out_path: Path) -> dict:
+def run_cli(args: list[str], body: dict | None, out_path: Path, *, drop_rows: bool = False) -> dict:
     out_path.parent.mkdir(parents=True, exist_ok=True)
     cmd = [*resolve_cli(), *args, "--format", "json"]
     if body is not None:
@@ -224,11 +267,12 @@ def run_cli(args: list[str], body: dict | None, out_path: Path) -> dict:
             body_path = out_path.with_suffix(".body.json")
         body_path.write_text(json.dumps(body, ensure_ascii=False), encoding="utf-8")
         cmd.extend(["--data", json.dumps(body, ensure_ascii=False)])
-    rc, raw = exec_cli_to_file(cmd, out_path)
-    parsed = parse_json_text(raw.strip())
-    stderr = out_path.with_suffix(".stderr.txt").read_text(encoding="utf-8", errors="replace")
+    rc = exec_cli_to_file(cmd, out_path)
+    parsed = load_json_file(out_path, drop_page_rows=drop_rows)
+    stderr_path = out_path.with_suffix(".stderr.txt")
     if not isinstance(parsed, dict) or rc != 0 or parsed.get("ok") is not True:
-        error = parse_json_text(stderr) or stderr or parsed or raw or "无有效 CLI 回包"
+        stderr = read_text_head(stderr_path)
+        error = parse_json_text(stderr) or stderr or parsed or read_text_head(out_path) or "无有效 CLI 回包"
         parsed = {"ok": False, "error": error, "rc": rc}
     return parsed
 
@@ -249,16 +293,46 @@ def optional_num(row, key):
         return None
 
 
-def metric_value(rows, metric="CPI"):
-    """同一粒度先合计基础指标再计算，任一缺失不当作零补齐。"""
-    formulas = {"CPI": ("spend_usd", "installs", 1), "CPC": ("spend_usd", "clicks", 1),
-                "CPM": ("spend_usd", "impressions", 1000), "ROAS": ("purchase_value", "spend_usd", 1),
-                "CTR": ("clicks", "impressions", 100)}
-    a, b, scale = formulas[metric]
-    if not rows or any(optional_num(r, key) is None for r in rows for key in (a, b)):
+METRIC_FORMULAS = {
+    "CPI": ("spend_usd", "installs", 1),
+    "CPC": ("spend_usd", "clicks", 1),
+    "CPM": ("spend_usd", "impressions", 1000),
+    "ROAS": ("purchase_value", "spend_usd", 1),
+    "CTR": ("clicks", "impressions", 100),
+}
+
+
+def new_metric_acc(metric="CPI"):
+    a, b, scale = METRIC_FORMULAS[metric]
+    return {"a": a, "b": b, "scale": scale, "num_a": 0.0, "num_b": 0.0, "ok": True, "seen": False}
+
+
+def add_metric_acc(acc, row) -> None:
+    if not acc["ok"]:
+        return
+    acc["seen"] = True
+    left, right = optional_num(row, acc["a"]), optional_num(row, acc["b"])
+    if left is None or right is None:
+        acc["ok"] = False
+        return
+    acc["num_a"] += left
+    acc["num_b"] += right
+
+
+def finish_metric_acc(acc):
+    if not acc["ok"] or not acc["seen"]:
         return None
-    denominator = sum(optional_num(r, b) for r in rows)
-    return sum(optional_num(r, a) for r in rows) / denominator * scale if denominator > 0 else None
+    return acc["num_a"] / acc["num_b"] * acc["scale"] if acc["num_b"] > 0 else None
+
+
+def metric_value(rows, metric="CPI"):
+    """同一粒度先合计基础指标再计算，任一缺失不当作零补齐。只扫一遍，可接迭代器。"""
+    acc = new_metric_acc(metric)
+    for row in rows or ():
+        add_metric_acc(acc, row)
+        if not acc["ok"]:
+            return None
+    return finish_metric_acc(acc)
 
 
 def core_rule(text):

package/dist/skills/custom/biz-line-strategy-generate/scripts/fetch_data.py

@@ -15,7 +15,7 @@ SCRIPTS = Path(__file__).resolve().parent
 if str(SCRIPTS) not in sys.path:
     sys.path.insert(0, str(SCRIPTS))
 
-from cli_json import list_query_body, run_cli, unwrap_page, require_ok, report_progress
+from cli_json import list_query_body, run_cli, unwrap_page, take_page_stats, require_ok, report_progress
 from datetime import date
 
 # 维度/成效 size=1000 时，TikTok 出价按天明细可到 4～5 万行，旧上限 30 页会截断
@@ -41,35 +41,50 @@ MET_CMD = {
 }
 
 
-def fetch_paged(shortcut: str, base_body: dict, stem: str, bl_dir: Path) -> list:
-    all_rows: list = []
+def fetch_paged(shortcut: str, base_body: dict, stem: str, bl_dir: Path) -> int:
+    """逐页把 CLI 回包落到 stem.pN.out.json，成功后再写 stem.pages.json。
+
+    只在内存里保留当前页和行数累计，不把全表 extend 成一份 list，也不写 merged.json。
+    开始前删掉本 stem 残留索引，避免同目录重试失败后仍留下上一轮完整 pages.json。
+    """
+    index_path = bl_dir / f"{stem}.pages.json"
+    if index_path.exists():
+        index_path.unlink()
+    accumulated = 0
+    counts: list[int] = []
     page = 1
     total = 0
     while True:
         body = {**base_body, "page": page, "size": 1000}
         out = bl_dir / f"{stem}.p{page}.out.json"
-        env = run_cli(["biz-line", shortcut], body, out)
+        env = run_cli(["biz-line", shortcut], body, out, drop_rows=True)
         require_ok(env, stem)
         data = env.get("data")
         if not isinstance(data, dict) or not isinstance(data.get("data"), list) or not isinstance(data.get("total"), int) or data["total"] < 0:
             raise RuntimeError(f"{stem}: 分页信封缺少有效 data/total")
-        rows, current_total = unwrap_page(env)
+        got = data.get("_page_got")
+        if isinstance(got, int):
+            current_total = data["total"]
+        else:
+            got, current_total = take_page_stats(env)
         if page > 1 and current_total != total:
             raise RuntimeError(f"{stem}: 分页期间总数变化，请重新拉取")
         total = current_total
-        all_rows.extend(rows)
-        report_progress(f"{stem} page={page} got={len(rows)} acc={len(all_rows)} total={total}")
-        if len(all_rows) > total:
+        accumulated += got
+        counts.append(got)
+        report_progress(f"{stem} page={page} got={got} acc={accumulated} total={total}")
+        if accumulated > total:
             raise RuntimeError(f"{stem}: 累计行数超过 total")
-        if len(all_rows) == total:
+        if accumulated == total:
             break
-        if not rows or page >= PAGE_CAP:
-            raise RuntimeError(f"{stem}: 分页不完整 {len(all_rows)}/{total}")
+        if got == 0 or page >= PAGE_CAP:
+            raise RuntimeError(f"{stem}: 分页不完整 {accumulated}/{total}")
         page += 1
-    (bl_dir / f"{stem}.merged.json").write_text(
-        json.dumps(all_rows, ensure_ascii=False), encoding="utf-8"
+    (bl_dir / f"{stem}.pages.json").write_text(
+        json.dumps({"total": total, "pages": len(counts), "counts": counts}, ensure_ascii=False),
+        encoding="utf-8",
     )
-    return all_rows
+    return total
 
 
 CHANNEL_ALIAS = {
@@ -336,7 +351,7 @@ def fetch_business(args):
                     "dimensions": dims,
                     **extra,
                 }
-                manifest[stem] = len(fetch_paged(DIM_CMD[ch], body, stem, bl_dir))
+                manifest[stem] = fetch_paged(DIM_CMD[ch], body, stem, bl_dir)
             for dims in met_dims(ch):
                 stem = f"met_{ch}_{per}_{'+'.join(dims)}"
                 body = {
@@ -346,7 +361,7 @@ def fetch_business(args):
                     "dimensions": dims,
                     "indicators": INDICATORS,
                 }
-                manifest[stem] = len(fetch_paged(MET_CMD[ch], body, stem, bl_dir))
+                manifest[stem] = fetch_paged(MET_CMD[ch], body, stem, bl_dir)
 
     # 只有全部所需数据完成后登记清单，汇总前按清单核对文件和数量。
     (req / 'fetch-manifest.json').write_text(json.dumps(manifest), encoding='utf-8')