stock-analyzer-skill 1.1.0 → 1.2.0

package/scripts/common/__init__.py

@@ -0,0 +1,334 @@
+"""
+公共工具包：HTTP 请求、字段映射、工具函数、熔断器、数据源抽象。
+
+包结构:
+- common/__init__.py   # 主模块（re-export + 熔断器/数据源抽象）
+- common/http.py       # HTTP 客户端
+- common/parsers.py    # 字段映射与解析
+- common/utils.py      # 工具函数
+- common/exceptions/   # 统一异常类
+- common/validators.py # 输入验证器
+"""
+import json
+import threading
+import time
+from abc import ABC, abstractmethod
+from enum import Enum
+
+# ---------- 子模块导入 ----------
+
+# 统一异常类
+from common.exceptions import (
+    StockAnalyzerError,
+    DataError,
+    NetworkError,
+    RateLimitError,
+    ParseError,
+    DataUnavailableError,
+    BusinessError,
+    ValidationError,
+    StrategyError,
+    InsufficientDataError,
+    ConfigurationError,
+    format_error,
+    is_retryable_error,
+)
+
+# HTTP 客户端
+from common.http import (
+    USER_AGENTS,
+    http_get,
+    http_get_with_headers,
+    decode_gbk,
+)
+
+# 字段映射与解析
+from common.parsers import (
+    TENCENT_FIELDS,
+    parse_tencent_line,
+    SINA_QUOTE_URL,
+    parse_sina_quote_line,
+    EAST_MONEY_FIELDS,
+)
+
+# 工具函数
+from common.utils import (
+    PACKAGE_ROOT,
+    DATA_DIR,
+    split_codes,
+    plain_code,
+    infer_exchange,
+    normalize_quote_code,
+    normalize_finance_code,
+    to_secid,
+    board_type,
+    is_etf,
+    batchify,
+    to_float,
+    to_int,
+    clamp,
+    normalize_volume,
+    normalize_amount,
+    err,
+    parallel_map,
+)
+
+# 输入验证器
+from common.validators import (
+    validate_code,
+    normalize_code,
+    validate_codes,
+    validate_date,
+    validate_date_range,
+    validate_positive,
+    validate_in_range,
+)
+
+# 向后兼容别名
+DataSourceUnavailableError = NetworkError
+DataParseError = ParseError
+
+
+# ---------- 缓存代理（延迟导入避免循环依赖）----------
+
+_cache_module = None
+
+
+def _get_cache_module():
+    global _cache_module
+    if _cache_module is None:
+        from data import cache as _cache_module
+    return _cache_module
+
+
+def _get_cache_items():
+    cache = _get_cache_module()
+    return cache.CACHE_DIR, cache.get, cache.set, cache.cleanup, cache.cache_key, cache.cache_key_for_stock
+
+
+def __getattr__(name):
+    if name in ("CACHE_DIR", "cache_get", "cache_set", "cache_cleanup", "cache_key", "cache_key_for_stock"):
+        items = _get_cache_items()
+        names = ("CACHE_DIR", "cache_get", "cache_set", "cache_cleanup", "cache_key", "cache_key_for_stock")
+        idx = names.index(name)
+        return items[idx]
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+def http_get_cached(url: str, timeout: int = 10, ttl: int = 21600) -> bytes:
+    """带缓存的 HTTP GET。先读缓存，未命中则请求并写入缓存。"""
+    cache = _get_cache_module()
+    key = cache.cache_key(url)
+    cached = cache.get(key, ttl)
+    if cached is not None:
+        return cached
+    data = http_get(url, timeout)
+    cache.set(key, data)
+    return data
+
+
+def http_get_cached_keyed(url: str, key: str, timeout: int = 10, ttl: int = 21600) -> bytes:
+    """带语义缓存键的 HTTP GET。"""
+    cache = _get_cache_module()
+    cached = cache.get(key, ttl)
+    if cached is not None:
+        return cached
+    data = http_get(url, timeout)
+    cache.set(key, data)
+    return data
+
+
+# ---------- 熔断器 ----------
+
+class CircuitState(Enum):
+    CLOSED = "closed"        # 正常
+    OPEN = "open"            # 熔断
+    HALF_OPEN = "half_open"  # 试探
+
+
+class CircuitBreaker:
+    """线程安全的熔断器：连续失败 N 次后熔断，超时后半开试探。"""
+
+    def __init__(self, name: str, failure_threshold: int = 5,
+                 recovery_timeout: int = 60, half_open_max: int = 3):
+        self.name = name
+        self.failure_threshold = failure_threshold
+        self.recovery_timeout = recovery_timeout
+        self.half_open_max = half_open_max
+
+        self._lock = threading.Lock()
+        self.state = CircuitState.CLOSED
+        self.failure_count = 0
+        self.last_failure_time = 0
+        self.half_open_success = 0
+
+    def can_execute(self) -> bool:
+        """判断是否允许请求（线程安全）。"""
+        with self._lock:
+            if self.state == CircuitState.CLOSED:
+                return True
+            if self.state == CircuitState.OPEN:
+                if time.time() - self.last_failure_time >= self.recovery_timeout:
+                    self.state = CircuitState.HALF_OPEN
+                    self.half_open_success = 0
+                    return True
+                return False
+            if self.state == CircuitState.HALF_OPEN:
+                return True
+            return False
+
+    def record_success(self):
+        """记录成功（线程安全）。"""
+        with self._lock:
+            if self.state == CircuitState.HALF_OPEN:
+                self.half_open_success += 1
+                if self.half_open_success >= self.half_open_max:
+                    self.state = CircuitState.CLOSED
+                    self.failure_count = 0
+            elif self.state == CircuitState.CLOSED:
+                self.failure_count = 0
+
+    def record_failure(self):
+        """记录失败（线程安全）。"""
+        with self._lock:
+            self.failure_count += 1
+            self.last_failure_time = time.time()
+            if self.state == CircuitState.HALF_OPEN:
+                self.state = CircuitState.OPEN
+            elif self.failure_count >= self.failure_threshold:
+                self.state = CircuitState.OPEN
+
+    def reset(self):
+        """重置熔断器（线程安全）。"""
+        with self._lock:
+            self.state = CircuitState.CLOSED
+            self.failure_count = 0
+            self.last_failure_time = 0
+
+
+# 全局熔断器实例（线程安全）
+_circuit_breakers = {}
+_circuit_breakers_lock = threading.Lock()
+
+
+def get_circuit_breaker(name: str, **kwargs) -> CircuitBreaker:
+    """获取或创建熔断器实例（线程安全）。"""
+    with _circuit_breakers_lock:
+        if name not in _circuit_breakers:
+            _circuit_breakers[name] = CircuitBreaker(name, **kwargs)
+        return _circuit_breakers[name]
+
+
+# ---------- 数据源抽象基类 ----------
+
+class BaseFetcher(ABC):
+    """数据源抽象基类。"""
+
+    def __init__(self, name: str, priority: int = 0):
+        self.name = name
+        self.priority = priority
+        self.circuit_breaker = get_circuit_breaker(name)
+
+    @abstractmethod
+    def fetch(self, code: str, **kwargs) -> dict | list | None:
+        """获取数据。返回 None 表示失败。"""
+        pass
+
+    def is_available(self) -> bool:
+        """检查数据源是否可用（熔断器状态）。"""
+        return self.circuit_breaker.can_execute()
+
+    def on_success(self):
+        """记录成功。"""
+        self.circuit_breaker.record_success()
+
+    def on_failure(self):
+        """记录失败。"""
+        self.circuit_breaker.record_failure()
+
+
+class DataFetcherManager:
+    """数据源策略管理器：按优先级尝试，自动故障切换。"""
+
+    def __init__(self, fetchers: list):
+        self.fetchers = sorted(fetchers, key=lambda f: f.priority, reverse=True)
+
+    def fetch(self, code: str, **kwargs) -> dict | list | None:
+        """按优先级尝试各数据源。"""
+        last_error = None
+        for fetcher in self.fetchers:
+            if not fetcher.is_available():
+                continue
+            try:
+                result = fetcher.fetch(code, **kwargs)
+                if result is not None:
+                    fetcher.on_success()
+                    return result
+                fetcher.on_failure()
+            except RateLimitError:
+                fetcher.on_failure()
+                raise  # 限流直接抛出
+            except Exception as e:
+                fetcher.on_failure()
+                last_error = e
+                continue
+        return None
+
+    def fetch_with_fallback(self, code: str, fallback=None, **kwargs):
+        """带默认值的获取。"""
+        result = self.fetch(code, **kwargs)
+        return result if result is not None else fallback
+
+    def fetch_with_cache_fallback(self, code: str, cache_prefix: str = None,
+                                   cache_ttl: int = 21600, fallback=None, **kwargs):
+        """带缓存降级的获取：优先实时数据 → 缓存数据 → 默认值。"""
+        result = self.fetch(code, **kwargs)
+        if result is not None:
+            return result
+
+        # 尝试从缓存降级
+        if cache_prefix:
+            cache = _get_cache_module()
+            key = cache.cache_key_for_stock(cache_prefix, code, **kwargs)
+            cached = cache.get(key, cache_ttl)
+            if cached is not None:
+                try:
+                    return json.loads(cached)
+                except (json.JSONDecodeError, Exception):
+                    pass
+
+        return fallback
+
+
+# ---------- 导出列表 ----------
+
+__all__ = [
+    # 基础设施
+    "PACKAGE_ROOT", "DATA_DIR", "USER_AGENTS",
+    "http_get", "http_get_with_headers", "http_get_cached", "http_get_cached_keyed",
+    "decode_gbk",
+    # 字段映射与解析
+    "TENCENT_FIELDS", "parse_tencent_line",
+    "SINA_QUOTE_URL", "parse_sina_quote_line", "EAST_MONEY_FIELDS",
+    # 工具函数
+    "split_codes", "plain_code", "infer_exchange",
+    "normalize_quote_code", "normalize_finance_code", "to_secid",
+    "board_type", "is_etf", "batchify", "to_float", "to_int", "clamp",
+    "normalize_volume", "normalize_amount",
+    "err", "parallel_map",
+    # 异常类
+    "StockAnalyzerError", "DataError", "NetworkError", "RateLimitError",
+    "ParseError", "DataUnavailableError", "BusinessError",
+    "ValidationError", "StrategyError", "InsufficientDataError",
+    "ConfigurationError", "format_error", "is_retryable_error",
+    # 向后兼容别名
+    "DataSourceUnavailableError", "DataParseError",
+    # 熔断器
+    "CircuitState", "CircuitBreaker", "get_circuit_breaker",
+    # 数据源抽象
+    "BaseFetcher", "DataFetcherManager",
+    # 输入验证器
+    "validate_code", "normalize_code", "validate_codes",
+    "validate_date", "validate_date_range",
+    "validate_positive", "validate_in_range",
+]

package/scripts/common/exceptions/__init__.py

@@ -0,0 +1,172 @@
+"""
+统一异常类定义。
+
+异常层次结构:
+- StockAnalyzerError (基础)
+  ├── DataError (数据层)
+  │   ├── NetworkError
+  │   ├── RateLimitError
+  │   ├── ParseError
+  │   └── DataUnavailableError
+  │
+  └── BusinessError (业务层)
+      ├── ValidationError
+      ├── StrategyError
+      └── InsufficientDataError
+"""
+from typing import Any
+
+
+class StockAnalyzerError(Exception):
+    """项目基础异常类。"""
+
+    def __init__(self, message: str, details: dict = None):
+        self.message = message
+        self.details = details or {}
+        super().__init__(self.message)
+
+    def to_dict(self) -> dict:
+        return {
+            "error_type": self.__class__.__name__,
+            "message": self.message,
+            "details": self.details,
+        }
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}({self.message})"
+
+
+# ═══════════════════════════════════════════════════════════════
+# 数据层异常
+# ═══════════════════════════════════════════════════════════════
+
+class DataError(StockAnalyzerError):
+    """数据层基类异常。"""
+    pass
+
+
+class NetworkError(DataError):
+    """网络请求失败。"""
+
+    def __init__(self, url: str, reason: str, retry_count: int = 0):
+        self.url = url
+        self.retry_count = retry_count
+        super().__init__(
+            f"网络请求失败: {reason}",
+            {"url": url, "retry_count": retry_count}
+        )
+
+
+class RateLimitError(NetworkError):
+    """触发速率限制 (429)。"""
+
+    def __init__(self, url: str, retry_after: int = None):
+        self.retry_after = retry_after
+        super().__init__(url, "429 Too Many Requests", 0)
+        self.message = f"触发速率限制，请 {retry_after} 秒后重试"
+
+
+class ParseError(DataError):
+    """数据解析失败。"""
+
+    def __init__(self, raw_data: str, parser: str, reason: str):
+        self.parser = parser
+        self.raw_preview = raw_data[:200] if raw_data else ""
+        super().__init__(
+            f"数据解析失败 [{parser}]: {reason}",
+            {"parser": parser, "data_preview": self.raw_preview}
+        )
+
+
+class DataUnavailableError(DataError):
+    """数据源不可用（连续失败）。"""
+
+    def __init__(self, source: str, failures: int):
+        self.source = source
+        self.failures = failures
+        super().__init__(
+            f"数据源 [{source}] 不可用，已连续失败 {failures} 次",
+            {"source": source, "failures": failures}
+        )
+
+
+# ═══════════════════════════════════════════════════════════════
+# 业务层异常
+# ═══════════════════════════════════════════════════════════════
+
+class BusinessError(StockAnalyzerError):
+    """业务层基类异常。"""
+    pass
+
+
+class ValidationError(BusinessError):
+    """输入校验失败。"""
+
+    def __init__(self, field: str, value: Any, constraint: str):
+        self.field = field
+        self.value_str = str(value)[:100] if value is not None else None
+        super().__init__(
+            f"字段 {field} 校验失败: {constraint}",
+            {"field": field, "value": self.value_str, "constraint": constraint}
+        )
+
+
+class StrategyError(BusinessError):
+    """策略执行错误。"""
+    pass
+
+
+class InsufficientDataError(BusinessError):
+    """数据不足，无法执行分析。"""
+
+    def __init__(self, data_type: str, required: int, actual: int, context: str = ""):
+        self.data_type = data_type
+        self.required = required
+        self.actual = actual
+        context_info = f" [{context}]" if context else ""
+        super().__init__(
+            f"{data_type}数据不足{context_info}: 需要 {required} 条，实际 {actual} 条",
+            {"data_type": data_type, "required": required, "actual": actual, "context": context}
+        )
+
+
+class ConfigurationError(StockAnalyzerError):
+    """配置错误。"""
+    pass
+
+
+# ═══════════════════════════════════════════════════════════════
+# 辅助函数
+# ═══════════════════════════════════════════════════════════════
+
+def format_error(error: Exception) -> str:
+    """格式化异常为用户友好消息。"""
+    if isinstance(error, StockAnalyzerError):
+        return error.message
+    return f"未知错误: {str(error)}"
+
+
+def is_retryable_error(error: Exception) -> bool:
+    """判断错误是否可重试。"""
+    if isinstance(error, RateLimitError):
+        return True
+    if isinstance(error, NetworkError):
+        return True
+    return False
+
+
+__all__ = [
+    "StockAnalyzerError",
+    "DataError",
+    "NetworkError",
+    "RateLimitError",
+    "ParseError",
+    "DataUnavailableError",
+    "BusinessError",
+    "ValidationError",
+    "StrategyError",
+    "InsufficientDataError",
+    "ConfigurationError",
+    "format_error",
+    "is_retryable_error",
+]

package/scripts/common/http.py

@@ -0,0 +1,79 @@
+"""HTTP 客户端：GET 请求、重试、编码转换。"""
+import random
+import time
+import urllib.request
+import urllib.error
+
+from common.exceptions import RateLimitError, NetworkError
+
+USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "stock-analyzer-skill/1.0",
+]
+
+
+def http_get(url: str, timeout: int = 10, max_retries: int = 3) -> bytes:
+    """GET 请求，指数退避重试，UA 随机轮换。429 立即抛出不重试。"""
+    last_err = None
+    for attempt in range(max_retries):
+        req = urllib.request.Request(url, headers={
+            "User-Agent": random.choice(USER_AGENTS),
+        })
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as resp:
+                return resp.read()
+        except urllib.error.HTTPError as e:
+            last_err = e
+            if e.code == 429:
+                raise RateLimitError(url)
+            if attempt < max_retries - 1:
+                delay = min(1.0 * (2 ** attempt), 8.0)
+                jitter = random.uniform(0, delay * 0.5)
+                time.sleep(delay + jitter)
+        except (urllib.error.URLError, TimeoutError, OSError, ConnectionResetError, BrokenPipeError) as e:
+            last_err = e
+            if attempt < max_retries - 1:
+                delay = min(1.0 * (2 ** attempt), 8.0)
+                jitter = random.uniform(0, delay * 0.5)
+                time.sleep(delay + jitter)
+    raise NetworkError(url, str(last_err), max_retries)
+
+
+def http_get_with_headers(url: str, headers: dict = None, timeout: int = 10, max_retries: int = 3) -> bytes:
+    """带自定义 headers 的 GET 请求（用于新浪等需要 Referer 的源）。"""
+    last_err = None
+    req_headers = {"User-Agent": random.choice(USER_AGENTS)}
+    if headers:
+        req_headers.update(headers)
+    for attempt in range(max_retries):
+        req = urllib.request.Request(url, headers=req_headers)
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as resp:
+                return resp.read()
+        except urllib.error.HTTPError as e:
+            last_err = e
+            if e.code == 429:
+                raise RateLimitError(url)
+            if attempt < max_retries - 1:
+                delay = min(1.0 * (2 ** attempt), 8.0)
+                jitter = random.uniform(0, delay * 0.5)
+                time.sleep(delay + jitter)
+        except (urllib.error.URLError, TimeoutError, OSError, ConnectionResetError, BrokenPipeError) as e:
+            last_err = e
+            if attempt < max_retries - 1:
+                delay = min(1.0 * (2 ** attempt), 8.0)
+                jitter = random.uniform(0, delay * 0.5)
+                time.sleep(delay + jitter)
+    raise NetworkError(url, str(last_err), max_retries)
+
+
+def decode_gbk(data: bytes) -> str:
+    """腾讯接口 GBK → UTF-8。"""
+    return data.decode("gbk", errors="replace")
+
+
+__all__ = [
+    "USER_AGENTS", "http_get", "http_get_with_headers", "decode_gbk",
+]

package/scripts/common/metrics.py

@@ -0,0 +1,92 @@
+"""轻量级指标收集器：fetch 延迟、成功率、缓存命中率。"""
+import json
+import time
+import threading
+from pathlib import Path
+from collections import defaultdict
+
+
+class MetricsCollector:
+    """线程安全的指标收集器。"""
+
+    def __init__(self):
+        self._lock = threading.Lock()
+        self._counters = defaultdict(int)
+        self._latencies = defaultdict(list)
+        self._start_time = time.time()
+
+    def record_fetch(self, source: str, success: bool, latency_ms: float):
+        """记录一次 fetch 调用。"""
+        with self._lock:
+            self._counters[f"fetch.{source}.total"] += 1
+            if success:
+                self._counters[f"fetch.{source}.success"] += 1
+            else:
+                self._counters[f"fetch.{source}.failure"] += 1
+            self._latencies[f"fetch.{source}"].append(latency_ms)
+
+    def record_cache(self, hit: bool):
+        """记录一次缓存访问。"""
+        with self._lock:
+            self._counters["cache.total"] += 1
+            if hit:
+                self._counters["cache.hit"] += 1
+            else:
+                self._counters["cache.miss"] += 1
+
+    def get_summary(self) -> dict:
+        """获取指标摘要。"""
+        with self._lock:
+            summary = {
+                "uptime_seconds": round(time.time() - self._start_time, 1),
+                "counters": dict(self._counters),
+                "latency": {},
+            }
+            # 计算延迟统计
+            for key, values in self._latencies.items():
+                if values:
+                    summary["latency"][key] = {
+                        "avg_ms": round(sum(values) / len(values), 1),
+                        "min_ms": round(min(values), 1),
+                        "max_ms": round(max(values), 1),
+                        "p50_ms": round(sorted(values)[len(values) // 2], 1),
+                        "count": len(values),
+                    }
+            # 计算成功率
+            for source in set(k.split(".")[1] for k in self._counters if k.startswith("fetch.")):
+                total = self._counters.get(f"fetch.{source}.total", 0)
+                success = self._counters.get(f"fetch.{source}.success", 0)
+                if total > 0:
+                    summary["counters"][f"fetch.{source}.success_rate"] = round(success / total * 100, 1)
+            # 缓存命中率
+            cache_total = self._counters.get("cache.total", 0)
+            cache_hit = self._counters.get("cache.hit", 0)
+            if cache_total > 0:
+                summary["counters"]["cache.hit_rate"] = round(cache_hit / cache_total * 100, 1)
+            return summary
+
+    def dump(self, path: Path = None):
+        """将指标写入 JSON 文件。"""
+        if path is None:
+            from data import cache
+            path = cache.CACHE_DIR / "metrics.json"
+        path.parent.mkdir(exist_ok=True)
+        path.write_text(json.dumps(self.get_summary(), ensure_ascii=False, indent=2))
+
+
+# 全局实例
+_collector = None
+_collector_lock = threading.Lock()
+
+
+def get_collector() -> MetricsCollector:
+    """获取全局指标收集器。"""
+    global _collector
+    if _collector is None:
+        with _collector_lock:
+            if _collector is None:
+                _collector = MetricsCollector()
+    return _collector
+
+
+__all__ = ["MetricsCollector", "get_collector"]