sino-bc-cli 1.0.0-beta.0

package/dist/skills/biz-line-strategy-generate/scripts/fetch_data.py

@@ -0,0 +1,449 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""按业务线批量拉列表 / 项目 / 维度 / 成效，写入本次请求目录。
+
+不要在 Agent 里手写上百次 CLI：信封解包和翻页都收在这里。
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+SCRIPTS = Path(__file__).resolve().parent
+if str(SCRIPTS) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS))
+
+from cli_json import BASE_METRICS, list_query_body, run_cli, unwrap_page, require_ok, report_progress
+from datetime import date, timedelta
+from strategy_output import GenerationOutcome, precheck_projects, validate_period
+
+# 维度/成效 size=1000；大业务线广告可超过 10 万行，200 页仍作为防失控上限。
+PAGE_CAP = 200
+
+INDICATORS = list(BASE_METRICS)
+DIM_CMD = {
+    "fb": "+facebook-dimension",
+    "gg": "+google-dimension",
+    "tt": "+tiktok-dimension",
+}
+MET_CMD = {
+    "fb": "+facebook-metrics",
+    "gg": "+google-metrics",
+    "tt": "+tiktok-metrics",
+}
+
+
+def fetch_paged(shortcut: str, base_body: dict, stem: str, bl_dir: Path) -> int:
+    """逐页把 CLI 回包落到 stem.pN.out.json，成功后再写 stem.pages.json。
+
+    只在内存里保留当前页和行数累计，不把全表 extend 成一份 list，也不写 merged.json。
+    开始前删掉本 stem 残留索引，避免同目录重试失败后仍留下上一轮完整 pages.json。
+    """
+    index_path = bl_dir / f"{stem}.pages.json"
+    if index_path.exists():
+        index_path.unlink()
+    accumulated = 0
+    counts: list[int] = []
+    page = 1
+    total = 0
+    conflicts = set()
+    while True:
+        body = {**base_body, "page": page, "size": 1000}
+        out = bl_dir / f"{stem}.p{page}.out.json"
+        env = run_cli(["biz-line-metrics", shortcut], body, out)
+        require_ok(env, stem)
+        data = env.get("data")
+        if not isinstance(data, dict) or not isinstance(data.get("data"), list) or not isinstance(data.get("total"), int) or data["total"] < 0:
+            raise RuntimeError(f"{stem}: 分页信封缺少有效 data/total")
+        rows, current_total = unwrap_page(env)
+        # 接口会静默忽略不支持字段；保留反馈，分析时不能误用部分渠道的指标。
+        reported = data.get('conflictField', [])
+        # 维度接口可能用 null 表示无冲突；只兼容空值，不放过 0/false/空字符串等非法类型。
+        if reported is None:
+            reported = []
+        if not isinstance(reported, list) or any(not isinstance(value, str) for value in reported):
+            raise RuntimeError(f'{stem}: conflictField 格式无效')
+        conflicts.update(reported)
+        if page > 1 and current_total != total:
+            raise RuntimeError(f"{stem}: 分页期间总数变化，请重新拉取")
+        total = current_total
+        got = len(rows)
+        accumulated += got
+        counts.append(got)
+        report_progress(f"{stem} page={page} got={got} acc={accumulated} total={total}")
+        if accumulated > total:
+            raise RuntimeError(f"{stem}: 累计行数超过 total")
+        if accumulated == total:
+            break
+        if not rows or page >= PAGE_CAP:
+            raise RuntimeError(f"{stem}: 分页不完整 {accumulated}/{total}")
+        page += 1
+    (bl_dir / f"{stem}.pages.json").write_text(
+        json.dumps({"total": total, "pages": len(counts), "counts": counts,
+                    **({'conflictField': sorted(conflicts)} if conflicts else {})}, ensure_ascii=False),
+        encoding="utf-8",
+    )
+    return total
+
+
+CHANNEL_ALIAS = {
+    "facebook": "fb",
+    "meta": "fb",
+    "fb": "fb",
+    "google": "gg",
+    "gg": "gg",
+    "tiktok": "tt",
+    "tik tok": "tt",
+    "tt": "tt",
+}
+
+
+def load_projects(req: Path, biz_line_id: int) -> tuple[list[str], dict[str, list[str]]]:
+    """校验所有关联项目的广告账号渠道，再按渠道去重；不使用项目渠道。"""
+    pl = run_cli(
+        ["biz-line", "+project-list"],
+        {"bizLineId": biz_line_id},
+        req / "biz-line-project-list.out.json",
+    )
+    require_ok(pl, "关联项目")
+    if not isinstance(pl.get('data'), list):
+        raise RuntimeError('关联项目回包格式无效')
+    rows, _ = unwrap_page(pl)
+    names = [
+        p.get("projectName")
+        for p in rows
+        if isinstance(p, dict) and p.get("projectName")
+    ]
+    buckets: dict[str, list[str]] = {"fb": [], "gg": [], "tt": []}
+    seen: set[tuple[str, str]] = set()
+    channels_by_id = None
+    for proj in rows:
+        if not isinstance(proj, dict):
+            raise RuntimeError('关联项目回包含无效项目')
+        accounts = proj.get('adAccounts')
+        if not isinstance(accounts, list):
+            raise RuntimeError('关联项目广告账号列表格式无效')
+        for acc in accounts:
+            if not isinstance(acc, dict):
+                raise RuntimeError('关联项目广告账号格式无效')
+            # 契约允许channelName为空；有channelId时必须以渠道字典解析ID，不能用名称掩盖未知ID。
+            if acc.get('channelId') is not None:
+                if channels_by_id is None:
+                    dictionary = require_ok(run_cli(['media', '+channels'], None, req / 'channels.out.json'), '渠道字典')
+                    if not isinstance(dictionary.get('data'), list):
+                        raise RuntimeError('渠道字典格式无效')
+                    channels_by_id = {str(item['channelId']).strip(): CHANNEL_ALIAS.get(str(item.get('channelName') or '').strip().lower())
+                                      for item in dictionary['data'] if isinstance(item, dict) and item.get('channelId') is not None}
+                ch = channels_by_id.get(str(acc['channelId']).strip())
+            else:
+                ch = CHANNEL_ALIAS.get(str(acc.get('channelName') or '').strip().lower())
+            if ch is None:
+                raise GenerationOutcome('UNSUPPORTED_CHANNEL',
+                    f"项目 {proj.get('projectId') or proj.get('projectName') or '未知'} 的广告账号渠道不支持或无法识别："
+                    f"channelId={acc.get('channelId')}，channelName={acc.get('channelName')}；仅允许 Facebook、Google、TikTok")
+            aid = str(acc.get("adAccountId") or "").strip()
+            if not aid or (ch, aid) in seen:
+                continue
+            seen.add((ch, aid))
+            buckets[ch].append(aid)
+    if not names:
+        detail = run_cli(
+            ["biz-line", "+detail"],
+            {"id": biz_line_id},
+            req / "biz-line-detail.out.json",
+        )
+        require_ok(detail, "业务线详情")
+        data = detail.get("data") if isinstance(detail.get("data"), dict) else {}
+        names = list(data.get("projectNames") or [])
+    return names, buckets
+
+
+def dim_jobs(ch: str) -> list[tuple[list[str], dict]]:
+    jobs: list[tuple[list[str], dict]] = []
+    orders = {"orders": {"adgroup_id": "asc", "calendar_date": "asc"}}
+    if ch == "fb":
+        jobs.append((["calendar_date", "adgroup_id", "adgroup_name", "bid_amount", "bid_strategy"], orders))
+    elif ch == "gg":
+        jobs.append(
+            (
+                [
+                    "calendar_date",
+                    "adgroup_id",
+                    "adgroup_name",
+                    "cpc_bid_amount",
+                    "cpm_bid_amount",
+                    "cpv_bid_amount",
+                    "bid_strategy",
+                ],
+                orders,
+            )
+        )
+    else:
+        jobs.append(
+            (
+                [
+                    "calendar_date",
+                    "adgroup_id",
+                    "adgroup_name",
+                    "bid_amount",
+                    "conversion_bid",
+                    "deep_cpabid",
+                    "bid_strategy",
+                ],
+                orders,
+            )
+        )
+    jobs.append((["country_code"], {}))
+    singles = {
+        "fb": [
+            "objective",
+            "optimization_goal",
+            "bid_strategy",
+            "ad_format",
+            "audience_type",
+            "targeting_age",
+            "targeting_genders",
+        ],
+        "gg": [
+            "advertising_channel_type",
+            "optimization_goal",
+            "bid_strategy",
+            "ad_format",
+            "audience_type",
+            "targeting_age",
+            "targeting_genders",
+            "keywords",
+        ],
+        "tt": [
+            "objective",
+            "optimization_goal",
+            "bid_strategy",
+            "ad_format",
+            "audience_type",
+            "targeting_age",
+            "targeting_genders",
+            "targeting_interests",
+        ],
+    }[ch]
+    for field in singles:
+        jobs.append(([field], {}))
+    if ch == "fb":
+        jobs.append(
+            (
+                [
+                    "targeting_interests",
+                    "targeting_flexible_interests",
+                    "targeting_exclusion_interests",
+                ],
+                {},
+            )
+        )
+    jobs.append((["adgroup_id", "adgroup_name"], {}))
+    jobs.append((["ad_id", "ad_name"], {}))
+    return jobs
+
+
+def met_dims(ch: str) -> list[list[str]]:
+    common = [
+        ["account_id"],
+        ["calendar_date"],
+        ["optimization_goal"],
+        ["bid_strategy"],
+        ["audience_type"],
+        ["targeting_age"],
+        ["targeting_genders"],
+        ["adgroup_id", "adgroup_name"],
+        ["ad_id", "ad_name"],
+        ["ad_format"],
+        ["country_code"],
+    ]
+    if ch == "fb":
+        return [["objective"], ["targeting_interests"], *common]
+    if ch == "gg":
+        return [["advertising_channel_type"], ["keywords"], *common]
+    return [["objective"], ["targeting_interests"], *common]
+
+
+def core_index_override(value):
+    """覆盖值绑定真实业务线ID，按首个等号切分以保留指标阈值比较符。"""
+    ident, separator, expression = value.partition('=')
+    if not separator or not ident.isdigit() or int(ident) <= 0 or not expression.strip():
+        raise argparse.ArgumentTypeError('核心指标覆盖格式为 业务线ID=指标，例如 123=CPI <= 3')
+    return int(ident), expression.strip()
+
+
+def argument_parser(batch=False):
+    """单条诊断入口与批次入口共享日期和账号参数。"""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--core-index', action='append', type=core_index_override, default=[],
+                        help='仅本次覆盖指定业务线核心指标，格式 ID=指标，可重复传不同ID')
+    if not batch:
+        parser.add_argument("--request-dir", required=True, help="strategy_task/日期/tmp内的独立请求目录（维护诊断用）")
+    target = parser.add_mutually_exclusive_group(required=True)
+    target.add_argument("--name", default="", help="业务线名称，命中多条时报歧义")
+    if batch:
+        target.add_argument("--id", type=int, action="append", default=[], help="业务线 ID，可重复传入多个")
+    else:
+        target.add_argument("--id", type=int, default=0, help="已知业务线 ID")
+    parser.add_argument("--curr-start", default="")
+    parser.add_argument("--curr-end", default="")
+    parser.add_argument("--prev-start", default="")
+    parser.add_argument("--prev-end", default="")
+    parser.add_argument("--skip-project-spend-check", action="store_true", help="仅用户明确要求时跳过项目上周消耗预检")
+    parser.add_argument("--account-fb", action="append", default=[])
+    parser.add_argument("--account-gg", action="append", default=[])
+    parser.add_argument("--account-tt", action="append", default=[])
+    return parser
+
+
+def resolve_periods(args, today=None):
+    """按执行机器当地日期取两个已结束自然周，各期用户起止日期成对覆盖。"""
+    anchor = today or date.today()
+    monday = anchor - timedelta(days=anchor.weekday())
+    result = {}
+    for key, weeks in (('curr', 1), ('prev', 2)):
+        start, end = getattr(args, key + '_start'), getattr(args, key + '_end')
+        if bool(start) != bool(end):
+            raise ValueError('本期和上期起止日期必须分别成对提供')
+        if not start:
+            first = monday - timedelta(weeks=weeks)
+            start, end = first.isoformat(), (first + timedelta(days=6)).isoformat()
+        if date.fromisoformat(start) > date.fromisoformat(end):
+            raise ValueError('周期开始日期晚于结束日期')
+        result[key] = (start, end)
+    return result
+
+
+def fetch_business(args):
+    """一条业务线内所有调用串行；阶段失败交由统一入口收尾。"""
+
+    req = Path(args.request_dir)
+    req.mkdir(parents=True, exist_ok=True)
+    accounts = {"fb": args.account_fb, "gg": args.account_gg, "tt": args.account_tt}
+
+    listed = run_cli(
+        ["biz-line", "+list"],
+        list_query_body(name=args.name, biz_id=args.id),
+        req / "biz-line-list.out.json",
+    )
+    require_ok(listed, "查找业务线")
+    rows, total = unwrap_page(listed)
+    if not rows:
+        raise RuntimeError("业务线列表为空")
+    if len(rows) != 1 or total != 1:
+        names = "；".join(f"{r.get('id')} {r.get('name')}" for r in rows[:10])
+        raise RuntimeError(f"名称命中 {total} 条，请收窄：{names}")
+    biz = rows[0]
+    biz_id = int(biz["id"])
+    # 用户覆盖仅写本次分析配置，不修改服务端业务线字段，也不跨业务线复用。
+    overrides = {}
+    for ident, expression in getattr(args, 'core_index', []):
+        if ident in overrides:
+            raise ValueError(f'业务线 {ident} 重复指定核心指标')
+        overrides[ident] = expression
+    if not args.id and set(overrides) - {biz_id}:
+        raise ValueError('核心指标覆盖ID与名称匹配的业务线不一致')
+    if biz_id in overrides:
+        (req / 'analysis-settings.json').write_text(json.dumps(
+            {'bizLineId': biz_id, 'coreIndex': overrides[biz_id]}, ensure_ascii=False), encoding='utf-8')
+    if args.id and biz_id != args.id:
+        raise RuntimeError("返回的业务线 ID 与请求不一致")
+    (req / "biz-line.json").write_text(json.dumps(biz, ensure_ascii=False, indent=2), encoding="utf-8")
+
+    # 查询、报表和保存共享本次解析的周期，不读取业务线返回的周期字段。
+    periods = resolve_periods(args)
+    curr_start, curr_end = periods['curr']
+    prev_start, prev_end = periods['prev']
+
+    # 只有真实业务线及合法周期才建立入库上下文，后续失败可据此保存状态。
+    if getattr(args, 'output_mode', 'excel') == 'database':
+        validate_period(curr_start, curr_end)
+        (req / 'save-context.json').write_text(json.dumps({
+            'bizLineId': biz_id, 'periodStart': curr_start, 'periodEnd': curr_end,
+        }), encoding='utf-8')
+    # 业务线已唯一确定且保存上下文已准备：先按行业加载标签，再取项目和广告数据。
+    if getattr(args, 'tag_cache', None) is not None:
+        args.tag_cache.get(biz)
+    if getattr(args, 'output_mode', 'excel') == 'database':
+        expected_ids = biz.get('projectIds')
+        if not isinstance(expected_ids, list) or any(type(value) is not int or value <= 0 for value in expected_ids):
+            raise RuntimeError('业务线缺少有效 projectIds，无法校验项目预检的完整性')
+        precheck_projects(req, biz_id, expected_ids, skip_spend_check=getattr(args, 'skip_project_spend_check', False))
+
+    require_ok(run_cli(["biz-line", "+detail"], {"id": biz_id}, req / "biz-line-detail.out.json"), "业务线详情")
+    names, discovered = load_projects(req, biz_id)
+    # 用户指定账号覆盖接口返回；没指定则用项目下按 ID 去重后的账号
+    for ch in accounts:
+        if not accounts[ch]:
+            accounts[ch] = discovered[ch]
+        accounts[ch] = list(dict.fromkeys(accounts[ch]))
+    (req / "periods.json").write_text(
+        json.dumps(
+            {
+                "curr": [curr_start, curr_end],
+                "prev": [prev_start, prev_end],
+            },
+            ensure_ascii=False,
+            indent=2,
+        ),
+        encoding="utf-8",
+    )
+    (req / "projects.json").write_text(
+        json.dumps(
+            {"bizLineId": biz_id, "projectNames": names, "accounts": accounts, "allAccounts": discovered},
+            ensure_ascii=False,
+            indent=2,
+        ),
+        encoding="utf-8",
+    )
+    if not names and not any(accounts.values()):
+        raise GenerationOutcome('NO_AD_ACCOUNT' if getattr(args, 'output_mode', 'excel') == 'database' else 'NO_PROJECT', "未绑定项目/广告账号")
+    if not any(accounts.values()):
+        raise GenerationOutcome('NO_AD_ACCOUNT', "项目下没有 Meta/Google/TikTok 广告账号")
+
+    bl_dir = req / f"bl-{biz_id}"
+    bl_dir.mkdir(parents=True, exist_ok=True)
+    manifest = {}
+
+    for ch, acc in accounts.items():
+        if not acc:
+            continue
+        for per, (start, end) in periods.items():
+            for dims, extra in dim_jobs(ch):
+                stem = f"dim_{ch}_{per}_{'+'.join(dims[:4])}"
+                body = {
+                    "startDate": start,
+                    "endDate": end,
+                    "dimensionValues": {"account_id": acc},
+                    "dimensions": dims,
+                    **extra,
+                }
+                manifest[stem] = fetch_paged(DIM_CMD[ch], body, stem, bl_dir)
+            for dims in met_dims(ch):
+                stem = f"met_{ch}_{per}_{'+'.join(dims)}"
+                # 维度成效仍保留按日颗粒度，供明细和已有汇总口径使用。
+                # 文件标识保留业务维度名；账户总体与按日总体请求保持原口径。
+                query_dims = dims if dims in (["account_id"], ["calendar_date"]) else [*dims, "calendar_date"]
+                body = {
+                    "startDate": start,
+                    "endDate": end,
+                    "dimensionValues": {"account_id": acc},
+                    "dimensions": query_dims,
+                    "indicators": INDICATORS,
+                }
+                manifest[stem] = fetch_paged(MET_CMD[ch], body, stem, bl_dir)
+
+    # 只有全部所需数据完成后登记清单，汇总前按清单核对文件和数量。
+    (req / 'fetch-manifest.json').write_text(json.dumps(manifest), encoding='utf-8')
+    return biz
+
+
+def main() -> int:
+    fetch_business(argument_parser().parse_args())
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())