aria-code 4.1.3__py3-none-any.whl

datasources/sources/fred_source.py

@@ -0,0 +1,219 @@
+"""
+datasources/sources/fred_source.py — FRED (Federal Reserve Economic Data)
+=========================================================================
+完全免费，无需 API key（可选 key 提升频率限制）。
+覆盖：宏观经济指标、利率、CPI、GDP、就业数据等 800,000+ 系列。
+
+配置（可选）: FRED_API_KEY 环境变量 或 ~/.aria/.env
+
+FRED series 示例：
+  DGS10      — 10年美国国债收益率
+  FEDFUNDS   — 联邦基金利率
+  CPIAUCSL   — 消费者价格指数
+  GDP        — 美国 GDP
+  UNRATE     — 失业率
+  SP500      — S&P 500 指数
+  DEXCNUS    — 人民币/美元汇率
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from datetime import date, timedelta
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from ..base import BaseDataSource, HistoryResult, QuoteResult
+
+logger = logging.getLogger(__name__)
+
+_FRED_BASE = "https://fred.stlouisfed.org/graph/fredgraph.csv?id="
+_FRED_API  = "https://api.stlouisfed.org/fred"
+
+# 常用宏观指标 symbol → FRED series id 映射
+MACRO_ALIASES: Dict[str, str] = {
+    "US10Y":    "DGS10",
+    "US2Y":     "DGS2",
+    "US3M":     "DTB3",
+    "FEDFUNDS": "FEDFUNDS",
+    "CPI":      "CPIAUCSL",
+    "CPIYOY":   "CPIAUCSL",
+    "PCE":      "PCEPI",
+    "GDP":      "GDP",
+    "UNRATE":   "UNRATE",
+    "NFP":      "PAYEMS",
+    "SP500":    "SP500",
+    "NASDAQ":   "NASDAQCOM",
+    "WILSHIRE": "WILL5000PR",
+    "USDINR":   "DEXINUS",
+    "USDCNY":   "DEXCHUS",
+    "USDEUR":   "DEXUSEU",
+    "VIX":      "VIXCLS",
+    "M2":       "M2SL",
+    "MORTGAGE": "MORTGAGE30US",
+    "HOUSING":  "HOUST",
+}
+
+
+def _load_api_key() -> str:
+    key = os.getenv("FRED_API_KEY", "")
+    if not key:
+        for p in [Path.home() / ".aria" / ".env", Path.home() / ".arthera" / ".env"]:
+            if p.exists():
+                for line in p.read_text(encoding="utf-8").splitlines():
+                    if line.startswith("FRED_API_KEY="):
+                        key = line.split("=", 1)[1].strip()
+                        break
+            if key:
+                break
+    if not key:
+        try:
+            import json as _json
+            _p = Path.home() / ".arthera" / "providers.json"
+            if _p.exists():
+                _raw = _json.loads(_p.read_text(encoding="utf-8"))
+                key = _raw.get("data", {}).get("fred", {}).get("api_key", "")
+        except Exception:
+            pass
+    return key
+
+
+class FREDSource(BaseDataSource):
+    """
+    Federal Reserve Economic Data — 宏观经济数据源。
+    无需 key 即可使用 CSV 下载接口；有 key 可使用 JSON API 获得更多元数据。
+    """
+
+    name         = "fred"
+    markets      = ["us", "macro"]
+    requires_key = False
+
+    def __init__(self, config=None):
+        super().__init__(config)
+        self._api_key = _load_api_key()
+
+    def is_configured(self) -> bool:
+        return True  # 无需 key，免费开放
+
+    def supports(self, symbol: str) -> bool:
+        s = symbol.upper()
+        return s in MACRO_ALIASES or s in MACRO_ALIASES.values()
+
+    def _resolve_series(self, symbol: str) -> str:
+        s = symbol.upper()
+        return MACRO_ALIASES.get(s, s)
+
+    def quote(self, symbol: str) -> Optional[QuoteResult]:
+        try:
+            h = self.history(symbol, days=30)
+            if h is None or h.data is None or h.data.empty:
+                return None
+            last = h.data.iloc[-1]
+            val  = float(last.get("close", last.iloc[0]))
+            series_id = self._resolve_series(symbol)
+            return QuoteResult(
+                symbol    = symbol,
+                name      = f"FRED:{series_id}",
+                price     = val,
+                currency  = "USD",
+                market    = "macro",
+                source    = self.name,
+                timestamp = str(h.data.index[-1].date()),
+            )
+        except Exception as e:
+            logger.debug(f"[fred] quote {symbol} 失败: {e}")
+            return None
+
+    def history(
+        self,
+        symbol: str,
+        days: int = 365,
+        interval: str = "1d",
+        _timeout: int = 12,
+    ) -> Optional[HistoryResult]:
+        try:
+            import pandas as pd
+            import urllib.request, json
+
+            series_id = self._resolve_series(symbol)
+            start = (date.today() - timedelta(days=days)).strftime("%Y-%m-%d")
+            end   = date.today().strftime("%Y-%m-%d")
+
+            def _parse_api(url: str) -> Optional[pd.DataFrame]:
+                req = urllib.request.Request(url, headers={"User-Agent": "aria-code/1.0"})
+                with urllib.request.urlopen(req, timeout=_timeout) as resp:
+                    data = json.loads(resp.read())
+                obs = data.get("observations", [])
+                rows = []
+                for o in obs:
+                    try:
+                        rows.append({"date": o["date"], "close": float(o["value"])})
+                    except (ValueError, KeyError):
+                        pass
+                if not rows:
+                    return None
+                _df = pd.DataFrame(rows)
+                _df["date"] = pd.to_datetime(_df["date"])
+                return _df.set_index("date").sort_index()
+
+            def _parse_csv(url: str) -> Optional[pd.DataFrame]:
+                req = urllib.request.Request(url, headers={"User-Agent": "aria-code/1.0"})
+                with urllib.request.urlopen(req, timeout=_timeout) as resp:
+                    _df = pd.read_csv(resp, parse_dates=["DATE"], index_col="DATE")
+                _df.columns = ["close"]
+                _df = _df.replace(".", float("nan")).dropna()
+                _df["close"] = _df["close"].astype(float)
+                return _df if not _df.empty else None
+
+            df = None
+
+            # 优先使用 JSON API（有 key 速率更高）
+            if self._api_key:
+                api_url = (
+                    f"{_FRED_API}/series/observations?series_id={series_id}"
+                    f"&observation_start={start}&observation_end={end}"
+                    f"&api_key={self._api_key}&file_type=json"
+                )
+                try:
+                    df = _parse_api(api_url)
+                except Exception as _e1:
+                    logger.debug(f"[fred] JSON API 失败，尝试 CSV: {_e1}")
+
+            # 回落：免费 CSV 下载接口（无需 key，但国内可能超时）
+            if df is None:
+                csv_url = (
+                    f"{_FRED_BASE}{series_id}"
+                    f"&cosd={start}&coed={end}"
+                )
+                try:
+                    df = _parse_csv(csv_url)
+                except Exception as _e2:
+                    logger.debug(f"[fred] CSV 也失败: {_e2}")
+
+            if df is None:
+                return None
+            return HistoryResult(symbol=symbol, data=df, source=self.name, interval="1d")
+        except Exception as e:
+            logger.debug(f"[fred] history {symbol} 失败: {e}")
+            return None
+
+    def search_series(self, query: str, limit: int = 10) -> List[Dict[str, Any]]:
+        """用关键词搜索 FRED 系列（需要 API key）。"""
+        if not self._api_key:
+            return [{"error": "需要 FRED_API_KEY 才能搜索系列"}]
+        try:
+            import urllib.request, json, urllib.parse
+            q   = urllib.parse.quote(query)
+            url = (f"{_FRED_API}/series/search?search_text={q}"
+                   f"&limit={limit}&api_key={self._api_key}&file_type=json")
+            req = urllib.request.Request(url, headers={"User-Agent": "aria-code/1.0"})
+            with urllib.request.urlopen(req, timeout=10) as resp:
+                data = json.loads(resp.read())
+            return [
+                {"id": s["id"], "title": s["title"], "frequency": s.get("frequency_short"),
+                 "units": s.get("units_short"), "last_updated": s.get("last_updated")}
+                for s in data.get("seriess", [])
+            ]
+        except Exception as e:
+            return [{"error": str(e)}]

datasources/sources/tushare_source.py

@@ -0,0 +1,141 @@
+"""
+datasources/sources/tushare_source.py — Tushare A股数据源
+==========================================================
+需要 TUSHARE_TOKEN 环境变量（已在 .env 中配置）。
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from datetime import date, timedelta
+from pathlib import Path
+from typing import Optional
+
+from ..base import BaseDataSource, FundamentalsResult, HistoryResult, QuoteResult
+
+logger = logging.getLogger(__name__)
+
+
+def _load_token() -> str:
+    token = os.getenv("TUSHARE_TOKEN", "")
+    if not token:
+        env_paths = [
+            Path.cwd() / ".env",
+            Path.cwd().parent / ".env",
+            Path.home() / ".aria" / ".env",
+            Path.home() / ".arthera" / ".env",
+        ]
+        for p in env_paths:
+            if p.exists():
+                for line in p.read_text(encoding="utf-8").splitlines():
+                    if line.startswith("TUSHARE_TOKEN="):
+                        token = line.split("=", 1)[1].strip()
+                        break
+            if token:
+                break
+    return token
+
+
+class TushareSource(BaseDataSource):
+
+    name         = "tushare"
+    markets      = ["a_share"]
+    requires_key = True
+
+    def __init__(self, config=None):
+        super().__init__(config)
+        self._token = _load_token()
+        self._pro   = None
+
+    def is_configured(self) -> bool:
+        return bool(self._token)
+
+    def _get_pro(self):
+        if self._pro is None:
+            import tushare as ts
+            ts.set_token(self._token)
+            self._pro = ts.pro_api()
+        return self._pro
+
+    def _to_ts_code(self, symbol: str) -> str:
+        s = symbol.upper().replace("SH","").replace("SZ","").replace(".","").strip()
+        code = s[:6]
+        return f"{code}.SH" if code.startswith(("6","9")) else f"{code}.SZ"
+
+    def quote(self, symbol: str) -> Optional[QuoteResult]:
+        """
+        使用 pro.daily 获取最近交易日收盘价（不调用 daily_basic 避免限频）。
+        PE/PB 留给 fundamentals() 单独获取。
+        """
+        try:
+            import time as _t
+            pro     = self._get_pro()
+            ts_code = self._to_ts_code(symbol)
+            end     = date.today().strftime("%Y%m%d")
+            start   = (date.today() - timedelta(days=10)).strftime("%Y%m%d")
+            df      = pro.daily(ts_code=ts_code, start_date=start, end_date=end)
+            _t.sleep(0.35)
+            if df is None or df.empty:
+                return None
+            # trade_date 格式 YYYYMMDD，字符串排序等价于日期排序
+            latest = df.sort_values("trade_date", ascending=False).iloc[0]
+            price  = float(latest.get("close",     0))
+            prev   = float(latest.get("pre_close", price))
+            return QuoteResult(
+                symbol     = symbol,
+                price      = price,
+                change     = round(price - prev, 4),
+                change_pct = float(latest.get("pct_chg", 0)),
+                volume     = float(latest.get("vol",    0)),
+                currency   = "CNY",
+                market     = "a_share",
+                source     = self.name,
+                timestamp  = str(latest.get("trade_date", "")),
+            )
+        except Exception as e:
+            logger.debug(f"[tushare] quote {symbol} 失败: {e}")
+            return None
+
+    def history(self, symbol: str, days: int = 90, interval: str = "1d") -> Optional[HistoryResult]:
+        try:
+            import time as _t, pandas as pd
+            pro     = self._get_pro()
+            ts_code = self._to_ts_code(symbol)
+            end     = date.today().strftime("%Y%m%d")
+            start   = (date.today() - timedelta(days=days + 15)).strftime("%Y%m%d")
+            df      = pro.daily(ts_code=ts_code, start_date=start, end_date=end)
+            _t.sleep(0.4)
+            if df is None or df.empty:
+                return None
+            df["trade_date"] = pd.to_datetime(df["trade_date"])
+            df = df.rename(columns={"trade_date": "date", "vol": "volume"})
+            df = df.set_index("date").sort_index()
+            return HistoryResult(symbol=symbol, data=df, source=self.name, interval=interval)
+        except Exception as e:
+            logger.debug(f"[tushare] history {symbol} 失败: {e}")
+            return None
+
+    def fundamentals(self, symbol: str) -> Optional[FundamentalsResult]:
+        try:
+            import time as _t
+            pro     = self._get_pro()
+            ts_code = self._to_ts_code(symbol)
+            for period in ["20241231", "20231231"]:
+                df = pro.fina_indicator(
+                    ts_code=ts_code, period=period,
+                    fields="ts_code,roe,netprofit_yoy,revenue_yoy"
+                )
+                _t.sleep(0.4)
+                if df is not None and not df.empty:
+                    r = df.iloc[0]
+                    return FundamentalsResult(
+                        symbol          = symbol,
+                        roe             = float(r.get("roe", 0) or 0),
+                        revenue_growth  = float(r.get("revenue_yoy", 0) or 0),
+                        net_profit_growth = float(r.get("netprofit_yoy", 0) or 0),
+                        source          = self.name,
+                    )
+        except Exception as e:
+            logger.debug(f"[tushare] fundamentals {symbol} 失败: {e}")
+        return None

datasources/sources/web_scraper_source.py

@@ -0,0 +1,278 @@
+"""
+datasources/sources/web_scraper_source.py — 公开财务数据网络爬虫
+================================================================
+爬取完全公开的金融数据网站，无需账号或 API key。
+所有请求遵守 robots.txt 和频率限制。
+
+覆盖:
+  - Macrotrends   — 历史财务数据（收入/利润/现金流/估值）
+  - Wisesheets    — 免费财务摘要
+  - 东方财富       — A股公告/研报摘要
+  - 同花顺         — A股财务数据
+  - Yahoo Finance  — 基本面摘要（HTML解析）
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import time
+import urllib.parse
+import urllib.request
+from typing import Any, Dict, List, Optional
+
+from ..base import BaseDataSource, FundamentalsResult, HistoryResult, QuoteResult
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/120.0.0.0 Safari/537.36"
+    ),
+    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+    "Accept": "text/html,application/xhtml+xml,application/json,*/*;q=0.8",
+}
+
+
+def _fetch_html(url: str, timeout: int = 12, encoding: str = "utf-8") -> Optional[str]:
+    try:
+        req = urllib.request.Request(url, headers=_DEFAULT_HEADERS)
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            raw = resp.read()
+            try:
+                return raw.decode(encoding)
+            except UnicodeDecodeError:
+                return raw.decode("gbk", errors="replace")
+    except Exception as e:
+        logger.debug(f"[web_scraper] fetch {url[:80]} 失败: {e}")
+        return None
+
+
+def _fetch_json(url: str, timeout: int = 12, extra_headers: dict = None) -> Optional[Any]:
+    import json
+    try:
+        headers = {**_DEFAULT_HEADERS, **(extra_headers or {})}
+        req = urllib.request.Request(url, headers=headers)
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return json.loads(resp.read())
+    except Exception as e:
+        logger.debug(f"[web_scraper] json {url[:80]} 失败: {e}")
+        return None
+
+
+# ─── 东方财富 A股公告爬虫 ────────────────────────────────────────────────────
+
+def scrape_eastmoney_announcements(symbol: str, count: int = 10) -> List[Dict]:
+    """
+    爬取东方财富 A股公告列表（免费公开页面）。
+    symbol: 6位A股代码（如 '600519'）
+    """
+    s = symbol.replace(".SS", "").replace(".SZ", "").zfill(6)
+    url = (
+        f"https://np-anotice-stock.eastmoney.com/api/security/ann?"
+        f"sr=-1&page_size={count}&page_index=1&ann_type=A&client_source=web"
+        f"&stock_list={s}&f_node=0&s_node=0"
+    )
+    data = _fetch_json(url, extra_headers={"Referer": "https://www.eastmoney.com/"})
+    if not data or "data" not in data:
+        return []
+    anns = data["data"].get("list", [])
+    result = []
+    for a in anns[:count]:
+        result.append({
+            "title":   a.get("notice_title", ""),
+            "date":    a.get("notice_date", ""),
+            "type":    a.get("notice_type", ""),
+            "url":     f"https://data.eastmoney.com/notices/detail/{s}/{a.get('art_code','')}.html",
+            "source":  "eastmoney",
+        })
+    return result
+
+
+def scrape_eastmoney_news(symbol: str, count: int = 10) -> List[Dict]:
+    """爬取东方财富个股新闻。"""
+    s = symbol.replace(".SS", "").replace(".SZ", "").zfill(6)
+    url = (
+        f"https://newsapi.eastmoney.com/kuaixun/v1/getlist_101_ajaxResult_50_{s},,50,1.html"
+    )
+    html = _fetch_html(url)
+    if not html:
+        return []
+    # 尝试从 JSON 响应中解析
+    try:
+        import json
+        data = json.loads(html)
+        items = data.get("LiveList", [])[:count]
+        return [{
+            "title":  i.get("title", ""),
+            "time":   i.get("showtime", ""),
+            "source": i.get("medianame", ""),
+            "url":    i.get("url", ""),
+        } for i in items]
+    except Exception:
+        pass
+    return []
+
+
+# ─── Macrotrends 历史财务数据爬虫 ────────────────────────────────────────────
+
+def scrape_macrotrends_revenue(symbol: str) -> Optional[List[Dict]]:
+    """
+    从 Macrotrends 爬取年度营收历史（免费公开页面，无需登录）。
+    注意：依赖页面结构，如网站改版可能失效。
+    """
+    slug_map = {"AAPL": "apple", "MSFT": "microsoft", "GOOGL": "alphabet",
+                "AMZN": "amazon", "META": "meta-platforms", "TSLA": "tesla",
+                "NVDA": "nvidia", "JPM": "jpmorgan-chase", "V": "visa"}
+    slug = slug_map.get(symbol.upper())
+    if not slug:
+        slug = symbol.lower()
+
+    url = f"https://www.macrotrends.net/stocks/charts/{symbol.upper()}/{slug}/revenue"
+    html = _fetch_html(url)
+    if not html:
+        return None
+
+    # 提取嵌入的 JSON 数据
+    match = re.search(r'var originalData\s*=\s*(\[.*?\]);', html, re.DOTALL)
+    if not match:
+        return None
+    try:
+        import json
+        rows = json.loads(match.group(1))
+        return [
+            {"date": r.get("date", ""), "revenue_usd": r.get("v1", 0)}
+            for r in rows
+        ][-20:]  # 最近20个数据点
+    except Exception:
+        return None
+
+
+# ─── SSE/SZSE 交易所公开数据爬虫 ─────────────────────────────────────────────
+
+def scrape_sse_financials(symbol: str) -> Optional[Dict]:
+    """
+    爬取上交所公开的公司财务摘要（适用于 6 开头的A股）。
+    """
+    s = symbol.replace(".SS", "").zfill(6)
+    if not s.startswith(("6", "9")):
+        return None
+
+    url = (
+        f"https://query.sse.com.cn/commonSoaQuery.do?"
+        f"isPagination=false&sqlId=COMMON_SSE_CP_GPLB_GPXG_JBXX_L&stockCode={s}"
+    )
+    data = _fetch_json(url, extra_headers={
+        "Referer": "https://www.sse.com.cn/",
+        "Host": "query.sse.com.cn",
+    })
+    if not data or "result" not in data:
+        return None
+    r = data["result"]
+    if isinstance(r, list) and r:
+        r = r[0]
+    if not isinstance(r, dict):
+        return None
+    return {
+        "symbol":     s,
+        "name":       r.get("FULL_NAME", ""),
+        "industry":   r.get("INDUSTRY", ""),
+        "listing_date": r.get("LISTING_DATE", ""),
+        "total_shares": r.get("TOTAL_SHARES", ""),
+        "source":     "sse",
+    }
+
+
+def scrape_szse_financials(symbol: str) -> Optional[Dict]:
+    """爬取深交所公开的公司基本信息（适用于 0/3 开头的A股）。"""
+    s = symbol.replace(".SZ", "").zfill(6)
+    if not s.startswith(("0", "3", "2")):
+        return None
+
+    url = (
+        f"https://www.szse.cn/api/report/show/nature/detail?"
+        f"id=&"
+    )
+    # 深交所 API
+    data = _fetch_json(
+        f"https://www.szse.cn/api/market/smstock/list?random=0.1&tab2PAGENUM=1"
+        f"&tab2PAGENO=1&tab2COUNT=1&tab2SORTTAB=tab2&tab2SORTKEY=",
+        extra_headers={"Referer": "https://www.szse.cn/"},
+    )
+    return None  # 深交所API结构复杂，返回None让上层fallback
+
+
+# ─── 统一爬虫接口 ─────────────────────────────────────────────────────────────
+
+class WebScraperSource(BaseDataSource):
+    """
+    公开网页数据爬虫 — 汇总多个免费公开金融数据网站。
+    实现 BaseDataSource 接口以便接入路由器；quote/history 委托给主数据源，
+    fundamentals() 提供爬取的补充数据（公告、交易所基本信息等）。
+    """
+
+    name         = "web_scraper"
+    markets      = ["a_share", "us"]
+    requires_key = False
+
+    def is_configured(self) -> bool:
+        return True
+
+    def quote(self, symbol: str) -> Optional[QuoteResult]:
+        return None  # 不提供实时行情，由 akshare/yfinance 负责
+
+    def fundamentals(self, symbol: str) -> Optional[FundamentalsResult]:
+        """从交易所公开页面补充基本信息（仅 A 股）。"""
+        s = symbol.replace(".SS", "").replace(".SZ", "").zfill(6)
+        info = None
+        if s.startswith(("6", "9")):
+            info = scrape_sse_financials(s)
+        elif s.startswith(("0", "3")):
+            info = scrape_szse_financials(s)
+        if not info:
+            return None
+        return FundamentalsResult(
+            symbol = symbol,
+            source = self.name,
+        )
+
+    # ── 扩展方法（供上层直接调用） ─────────────────────────────────────────────
+
+    def get_announcements(self, symbol: str, count: int = 10) -> List[Dict]:
+        """A股公告列表（东方财富）。"""
+        return scrape_eastmoney_announcements(symbol, count)
+
+    def get_news(self, symbol: str, count: int = 10) -> List[Dict]:
+        """个股新闻。"""
+        return scrape_eastmoney_news(symbol, count)
+
+    def get_historical_revenue(self, symbol: str) -> Optional[List[Dict]]:
+        """美股历史营收（Macrotrends）。"""
+        return scrape_macrotrends_revenue(symbol)
+
+    def get_exchange_info(self, symbol: str) -> Optional[Dict]:
+        """A股交易所公开信息。"""
+        s = symbol.replace(".SS", "").replace(".SZ", "").zfill(6)
+        if s.startswith(("6", "9")):
+            return scrape_sse_financials(s)
+        elif s.startswith(("0", "3")):
+            return scrape_szse_financials(s)
+        return None
+
+    def bulk_scrape(self, symbols: List[str], data_type: str = "announcements") -> Dict[str, Any]:
+        """批量爬取，带速率限制（每次请求间隔 1 秒）。"""
+        results = {}
+        for sym in symbols:
+            try:
+                if data_type == "announcements":
+                    results[sym] = self.get_announcements(sym)
+                elif data_type == "news":
+                    results[sym] = self.get_news(sym)
+                elif data_type == "revenue":
+                    results[sym] = self.get_historical_revenue(sym)
+                time.sleep(1.0)
+            except Exception as e:
+                results[sym] = {"error": str(e)}
+        return results