sino-bc-cli 1.0.0-beta.0

package/dist/skills/biz-line-strategy-generate/scripts/run_workflow.py

@@ -0,0 +1,402 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""业务线策略唯一入口：拉数与AI各最多三个滑动并发，逐条交付和清理。"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import shutil
+import signal
+import sys
+from datetime import date
+import threading
+import time
+from argparse import Namespace
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
+from pathlib import Path
+
+# 从任意 cwd 或安装目录启动均定位到技能自身；不在源码树生成缓存。
+sys.dont_write_bytecode = True
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from build_report import (default_output_path, read_ai_sheet, validate_detail_workbook, validate_xlsx,
+                          write_detail_workbook, write_xlsx)
+from ai_strategy import AiStrategyExhausted, review_ai_strategy
+from cleanup import OwnedDirectory, MARKER
+from cli_json import cli_context, load_json_file
+from fetch_data import argument_parser, fetch_business, resolve_periods
+from summarize import summarize_request
+from strategy_output import (TagCache, GenerationOutcome, IndustryConfigurationError,
+                             attach_tags, review_tags, ready_body, upload_excel, save_result)
+
+_output_lock = threading.Lock()
+
+
+class PipelineLease:
+    """一条业务线的阶段名额：拉数转AI后立即释放拉数名额，收尾才释放AI。"""
+
+    def __init__(self, fetch_slots, ai_slots):
+        # 拉数名额已由调度器非阻塞取得，不能在排队线程中再占一份。
+        self.fetch_slots, self.ai_slots = fetch_slots, ai_slots
+        self.phase = 'fetch'
+
+    def enter_ai(self, cancel):
+        """排队不生成审核请求，不消耗7/15分钟的审核预算。"""
+        if self.phase != 'fetch':
+            raise ValueError('业务线不能重复进入AI阶段')
+        self.phase = 'waiting_ai'
+        self.fetch_slots.release()
+        while not cancel.is_set():
+            if self.ai_slots.acquire(timeout=0.1):
+                self.phase = 'ai'
+                if cancel.is_set():
+                    raise InterruptedError('任务已取消')
+                return
+        raise InterruptedError('等待AI阶段名额时任务已取消')
+
+    def close(self):
+        """worker返回或异常都只归还当前实际持有的名额，允许重复收尾。"""
+        phase, self.phase = self.phase, 'closed'
+        if phase == 'fetch':
+            self.fetch_slots.release()
+        elif phase == 'ai':
+            self.ai_slots.release()
+
+
+def execute_admitted(worker, target, args, batch, cancel, progress):
+    """跨阶段调度不拆散原有单条try/finally；清理结束后才归还总在途名额。"""
+    try:
+        return worker(target, args, batch, cancel, progress)
+    finally:
+        args.pipeline_lease.close()
+
+
+class PublishError(RuntimeError):
+    """发布失败且无法移除本次不完整目标时，保留精确残留路径。"""
+    def __init__(self, message, path):
+        super().__init__(message)
+        self.residual_path = str(path)
+
+
+def emit(event):
+    """并发进度输出按 JSON 行加锁，结果无需额外磁盘日志。"""
+    with _output_lock:
+        print(json.dumps(event, ensure_ascii=False), flush=True)
+
+
+def publish(temp, final, payload):
+    """先校验临时文件，再独占创建目标；任何中途错误仅清理自己创建的目标。"""
+    validate_xlsx(temp, payload['rows'], payload['details'], ai_strategy=payload.get('ai_strategy'))
+    created = False
+    try:
+        with final.open('xb') as output:
+            created = True
+            with temp.open('rb') as source:
+                shutil.copyfileobj(source, output)
+        validate_xlsx(final, payload['rows'], payload['details'], ai_strategy=payload.get('ai_strategy'))
+    except BaseException as original:
+        if created:
+            try:
+                final.unlink()
+            except OSError as cleanup_error:
+                raise PublishError(f'发布失败：{original}；删除不完整目标失败：{cleanup_error}', final) from original
+        raise
+
+
+def execute_business(target, args, batch, cancel, progress=emit):
+    """单条保持完整生命周期，只在固定分析后交接拉数/AI阶段名额。"""
+    owned = OwnedDirectory.create(batch.path, f'line-{target or "named"}-')
+    result = {'id': target, 'ok': False, 'timings': {}, 'output_path': None}
+    current_stage = 'fetch'
+    started = time.perf_counter()
+    final = None
+    database = args.output_mode == 'database'
+    save_attempted = False
+    context = None
+
+    def stage(name, function):
+        nonlocal current_stage
+        current_stage = name
+        if cancel.is_set():
+            raise InterruptedError('任务已取消')
+        begin = time.perf_counter()
+        progress({'id': result['id'], 'stage': name, 'status': 'running'})
+        try:
+            return function()
+        finally:
+            result['timings'][name] = round(time.perf_counter() - begin, 3)
+
+    try:
+        # 最终目录必须在整个批次临时树之外，而非仅在当前业务线之外。
+        output_dir = Path(args.output_dir or Path(default_output_path()).parent).resolve() if not database else None
+        if output_dir is not None and output_dir.is_relative_to(batch.path):
+            raise ValueError('最终 Excel 目录不能位于批次临时目录内')
+        local_args = Namespace(**{**vars(args), 'id': target or 0, 'request_dir': str(owned.path)})
+        with cli_context(cancel, lambda text: progress({'id': result['id'], 'stage': 'fetch', 'detail': text})):
+            biz = stage('fetch', lambda: fetch_business(local_args))
+        result['id'], result['name'] = int(biz['id']), str(biz.get('name') or '')
+        context = load_json_file(owned.path / 'save-context.json') if database else None
+        payload, summary = stage('analyze', lambda: summarize_request(owned.path))
+        result['detail_count'] = summary['detail_count']
+        if database and summary['curr_total'] <= 0:
+            raise GenerationOutcome('NO_DELIVERY_DATA', '目标周期无投放消耗')
+        lease = getattr(args, 'pipeline_lease', None)
+        if lease is not None:
+            # 某一条准备好就补拉下一条，不等待同批其他拉数全部完成。
+            stage('wait_ai_slot', lambda: lease.enter_ai(cancel))
+        with cli_context(cancel, lambda text: progress({'id': result['id'], 'stage': 'tags', 'detail': text})):
+            dictionaries = args.tag_cache.get(biz)
+            evidence = summary.get('tag_evidence', {})
+            def tag_progress(event):
+                # 兜底必须进入最终业务线结果，不能只留在瞬时进度事件里。
+                if event.get('event') == 'tag_review_fallback':
+                    result['tag_review_fallback'] = {
+                        'reviewAttempts': event['reviewAttempts'],
+                        'failures': event['failures'],
+                        'missingOtherScenes': event['missingOtherScenes'],
+                    }
+                progress({'id': result['id'], **event})
+            selections = stage('tag_review', lambda: review_tags(payload['rows'][0], dictionaries, evidence,
+                               owned.path, cancel, tag_progress,
+                               biz_line_id=result['id'], attempt=getattr(args, 'attempt', 1)))
+            stage('tags', lambda: attach_tags(payload['rows'][0], dictionaries, evidence, selections))
+        if database:
+            if not context:
+                raise ValueError('缺少可保存业务线及周期上下文')
+        # 先把固定策略与明细写成可回读草稿，再把实际工作簿交给 Agent 撰写第三张表。
+        draft = owned.path / 'draft.xlsx'
+        stage('export_draft', lambda: write_xlsx(payload['rows'], str(draft), payload['details']))
+        stage('validate_draft', lambda: validate_xlsx(draft, payload['rows'], payload['details']))
+        # AI 只接收明细-only 工作簿；固定策略仍保留在草稿和最终交付文件中。
+        ai_detail = owned.path / 'ai-detail.xlsx'
+        stage('export_ai_detail', lambda: write_detail_workbook(payload['details'], str(ai_detail)))
+        stage('validate_ai_detail', lambda: validate_detail_workbook(ai_detail, payload['details']))
+        prompt = Path(__file__).resolve().parent.parent / 'references' / 'ai-strategy-prompt.md'
+        ai_text = stage('ai_strategy', lambda: review_ai_strategy(
+            owned.path, ai_detail, prompt, cancel,
+            lambda event: progress({'id': result['id'], **event}),
+            biz_line_id=result['id'], attempt=getattr(args, 'attempt', 1), require_learning=database))
+        payload['ai_strategy'] = ai_text
+        temp = owned.path / 'final.xlsx'
+        stage('export', lambda: write_xlsx(payload['rows'], str(temp), payload['details'], ai_strategy=ai_text))
+        stage('validate', lambda: validate_xlsx(temp, payload['rows'], payload['details'], ai_strategy=ai_text))
+        if database:
+            # 上传前从最终工作簿取值并校验，删文件后只复用这份已核对的单元格。
+            ai_sheet = stage('read_ai_sheet', lambda: read_ai_sheet(temp))
+            stage('preflight', lambda: ready_body(context, ai_sheet, payload['rows'][0], 'preflight'))
+            with cli_context(cancel, lambda text: progress({'id': result['id'], 'stage': current_stage, 'detail': text})):
+                result['storageKey'] = stage('upload', lambda: upload_excel(owned.path, temp))
+                # 仅删除自己创建且归属已验证的临时文件，上传成功后不留桌面副本。
+                owned.validate()
+                stage('remove_excel', temp.unlink)
+                body = ready_body(context, ai_sheet, payload['rows'][0], result['storageKey'])
+                if cancel.is_set():
+                    raise InterruptedError('任务已取消')
+                save_attempted = True
+                result['aiStrategyId'] = stage('save', lambda: save_result(owned.path, body))
+            result.update(generateCode='READY', database_saved=True)
+        else:
+            safe_name = re.sub(r'[<>:"/\\|?*\x00-\x1f]', '_', result['name'])[:55].rstrip(' .') or '业务线'
+            final = output_dir / f'业务线策略_{result["id"]}_{safe_name}_{owned.token[:12]}.xlsx'
+            output_dir.mkdir(parents=True, exist_ok=True)
+            stage('validate_publish', lambda: publish(temp, final, payload))
+            result['output_path'] = str(final)
+        result['ok'] = True
+    except Exception as exc:
+        result.update(error=str(exc), failed_stage=current_stage)
+        if isinstance(exc, IndustryConfigurationError):
+            # 真实业务线缺行业是配置失败，保存一次状态后不再重拉数据。
+            result.update(retryable=False, failure_kind='industry_configuration')
+        if isinstance(exc, AiStrategyExhausted):
+            # 三次 AI 尝试已经穷尽，批次不得重新拉数制造额外三轮。
+            result['ai_strategy_exhausted'] = True
+        # 保存请求发出后不补写失败状态；超时可能已成功，补写会替换有效记录。
+        if database and not save_attempted and not cancel.is_set():
+            context = context or load_json_file(owned.path / 'save-context.json')
+            if context:
+                code = exc.code if isinstance(exc, GenerationOutcome) else 'GENERATION_FAILED'
+                result.update(id=context['bizLineId'], generateCode=code)
+                try:
+                    with cli_context(cancel, lambda _: None):
+                        result['aiStrategyId'] = stage('save_status', lambda: save_result(owned.path, {
+                            **context, 'generateCode': code, 'remark': str(exc),
+                        }))
+                    result['database_saved'] = True
+                    # 不可生成是业务结果；技术失败即使状态保存成功仍返回失败。
+                    result['ok'] = isinstance(exc, GenerationOutcome)
+                except Exception as save_error:
+                    result.update(database_saved=False, save_error=str(save_error), save_result_unknown=True)
+        elif save_attempted:
+            result.update(database_saved=False, save_result_unknown=True)
+        if isinstance(exc, PublishError):
+            result['residual_output'] = exc.residual_path
+    finally:
+        begin = time.perf_counter()
+        cleanup = owned.cleanup(final)
+        result['timings']['cleanup'] = round(time.perf_counter() - begin, 3)
+        result['cleanup'] = cleanup
+        if not cleanup['ok']:
+            result.update(ok=False, cleanup_failed=True)
+        result['timings']['total'] = round(time.perf_counter() - started, 3)
+    return result
+
+
+def retry_allowed(result):
+    """只重跑未交付的技术失败；保存未知、残留文件及已处理业务状态需单独报告。"""
+    return not (result.get('retryable') is False or result.get('ok') or result.get('save_result_unknown')
+                or result.get('ai_strategy_exhausted')
+                or result.get('cleanup_failed') or result.get('residual_output')
+                or result.get('output_path')
+                or result.get('failed_stage') in ('cancelled', 'worker')
+                or (result.get('database_saved') and result.get('generateCode') != 'GENERATION_FAILED'))
+
+
+def run_batch(args, *, cancel=None, worker=execute_business, progress=emit, temp_parent=None):
+    """拉数/AI各最多三个，总在途最多六个；整轮结束才重跑失败项。"""
+    if args.concurrency not in (1, 2, 3):
+        raise ValueError('并发只能设置为 1、2 或 3')
+    if args.output_mode not in ('excel', 'database'):
+        raise ValueError('输出模式必须为 excel 或 database')
+    if args.output_mode == 'database' and args.output_dir:
+        raise ValueError('database 模式不接受 --output-dir，不向桌面发布 Excel')
+    if any(i <= 0 for i in args.id):
+        raise ValueError('业务线 ID 必须为正整数')
+    # 覆盖必须绑定本批目标，重复或拼错ID在请求业务接口前失败。
+    override_ids = [ident for ident, _ in getattr(args, 'core_index', [])]
+    if len(set(override_ids)) != len(override_ids):
+        raise ValueError('同一业务线不能重复指定核心指标')
+    if args.id and set(override_ids) - set(args.id):
+        raise ValueError('核心指标覆盖ID不在本批业务线中')
+    # 批次开始时固定日期，避免任务跨越周一零点后使用不同的默认周。
+    task_date = date.today()
+    periods = resolve_periods(args, today=task_date)
+    args = Namespace(**{**vars(args), **{
+        key + suffix: bounds[index]
+        for key, bounds in periods.items()
+        for index, suffix in enumerate(('_start', '_end'))
+    }})
+    targets = list(dict.fromkeys(args.id)) if args.id else [None]
+    cancel = cancel or threading.Event()
+    # 只使用明确的持久工作根，日期固定在本批启动时，不回退系统TEMP。
+    configured_root = getattr(args, 'task_root', None)
+    if configured_root is not None and not str(configured_root).strip():
+        raise ValueError('--task-root不能为空')
+    work_root = Path(temp_parent) if temp_parent is not None else Path(configured_root or Path.cwd())
+    if not work_root.is_absolute():
+        work_root = Path.cwd() / work_root
+    task_tmp = (work_root.resolve() / 'strategy_task' / task_date.isoformat() / 'tmp').resolve()
+    if task_tmp.is_relative_to(Path(__file__).resolve().parent.parent):
+        raise ValueError('--task-root不能位于安装技能或源码目录内，请选择持久工作目录')
+    try:
+        batch = OwnedDirectory.create(task_tmp, 'batch-')
+    except OSError as exc:
+        raise ValueError(f'无法创建策略任务目录 {task_tmp}：{exc}') from exc
+    # 共享字典独立归属；业务线清理不触碰它，批次结束单独清理。
+    tags_owned = OwnedDirectory.create(batch.path, 'tags-')
+    args = Namespace(**{**vars(args), 'tag_cache': TagCache(tags_owned.path)})
+    latest, history = {}, {target: [] for target in targets}
+    round_targets = targets
+    started = time.perf_counter()
+    fetch_slots = threading.BoundedSemaphore(args.concurrency)
+    ai_slots = threading.BoundedSemaphore(args.concurrency)
+    in_flight_limit = args.concurrency * 2
+    try:
+        # 最初只分配拉数名额；线程按实际提交惰性启动，不一次启动六条拉数。
+        with ThreadPoolExecutor(max_workers=in_flight_limit) as pool:
+            for attempt in range(1, 4):
+                if attempt > 1:
+                    if cancel.is_set():
+                        break
+                    args.tag_cache.reset_errors()
+                    progress({'event': 'retry_round', 'attempt': attempt, 'targets': round_targets})
+                pending, remaining = {}, iter(round_targets)
+                while True:
+                    while not cancel.is_set() and len(pending) < in_flight_limit:
+                        if not fetch_slots.acquire(blocking=False):
+                            break
+                        try:
+                            target = next(remaining)
+                        except StopIteration:
+                            fetch_slots.release()
+                            break
+                        # 每个 worker 持有自己的轮次快照，避免并发修改共享 args 导致错配。
+                        lease = PipelineLease(fetch_slots, ai_slots)
+                        worker_args = Namespace(**{**vars(args), 'attempt': attempt, 'pipeline_lease': lease})
+                        try:
+                            future = pool.submit(execute_admitted, worker, target, worker_args, batch, cancel, progress)
+                        except BaseException:
+                            lease.close()
+                            raise
+                        pending[future] = target
+                    if not pending:
+                        break
+                    finished, _ = wait(pending, timeout=0.2, return_when=FIRST_COMPLETED)
+                    for future in finished:
+                        target = pending.pop(future)
+                        try:
+                            result = future.result()
+                        except Exception as exc:
+                            # worker 之外异常无法保证副作用状态，不能盲目整条重放。
+                            result = {'id': target, 'ok': False, 'error': str(exc), 'failed_stage': 'worker'}
+                        result['attempt'] = attempt
+                        history[target].append(result.copy())
+                        latest[target] = result
+                        progress({'event': 'completed', **result})
+                # 取消时保留此前失败记录；首次未启动条目仍计入最终结果。
+                for target in remaining:
+                    if target not in latest:
+                        latest[target] = {'id': target, 'ok': False, 'failed_stage': 'cancelled',
+                                          'error': '任务未启动，已取消'}
+                round_targets = [target for target in round_targets if retry_allowed(latest[target])]
+                if not round_targets or cancel.is_set():
+                    break
+    finally:
+        tags_cleanup = tags_owned.cleanup()
+        # 已失败的清理目录保留并报告，不由父级递归重试掩盖错误。
+        batch.validate()
+        children = [p for p in batch.path.iterdir() if p.name != MARKER]
+        cleanup = ({'ok': False, 'residual': [{'path': str(p), 'error': '业务线目录尚有残留'} for p in children]} if children else batch.cleanup())
+        if not tags_cleanup['ok']:
+            cleanup['ok'] = False
+            cleanup.setdefault('residual', []).extend(tags_cleanup.get('residual', []))
+    # 同一业务线仅按最终一次统计，历史失败供定位，不重复计入失败数。
+    results = [{**latest[target], 'attempts': len(history[target]), 'attempt_history': history[target]}
+               for target in targets]
+    success = sum(r['ok'] for r in results)
+    return {'ok': success == len(targets) and cleanup['ok'] and not cancel.is_set(), 'success': success,
+            'failed': len(results) - success, 'results': results, 'cleanup': cleanup,
+            'elapsed_seconds': round(time.perf_counter() - started, 3)}
+
+
+def workflow_parser():
+    """入口参数集中构建，便于验证默认本地输出和显式数据库输出。"""
+    parser = argument_parser(batch=True)
+    parser.description = __doc__
+    parser.add_argument('--concurrency', type=int, choices=(1, 2, 3), default=3,
+                        help='拉数和AI阶段各自的并发上限；总在途最多为此值的两倍，默认3+3')
+    parser.add_argument('--output-dir', default=None)
+    parser.add_argument('--task-root', default=None, help='持久工作目录，默认启动cwd；临时文件在strategy_task/日期/tmp下')
+    parser.add_argument('--output-mode', choices=('excel', 'database'), default='excel')
+    return parser
+
+
+def main():
+    args = workflow_parser().parse_args()
+    cancel = threading.Event()
+    # 可处理的 Ctrl+C/SIGTERM 只发信号，worker 关闭自己的 CLI 后清理。
+    previous = {sig: signal.signal(sig, lambda *_: cancel.set()) for sig in (signal.SIGINT, signal.SIGTERM)}
+    try:
+        result = run_batch(args, cancel=cancel)
+        emit({'event': 'batch_completed', **result})
+        return 0 if result['ok'] else 1
+    except ValueError as exc:
+        emit({'ok': False, 'error': str(exc)})
+        return 2
+    finally:
+        for sig, handler in previous.items():
+            signal.signal(sig, handler)
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())