aria-code 0.52.0__py3-none-any.whl

aria_code/datasources/sources/web_scraper_source.py

@@ -0,0 +1,278 @@
+"""
+datasources/sources/web_scraper_source.py — 公开财务数据网络爬虫
+================================================================
+爬取完全公开的金融数据网站，无需账号或 API key。
+所有请求遵守 robots.txt 和频率限制。
+
+覆盖:
+  - Macrotrends   — 历史财务数据（收入/利润/现金流/估值）
+  - Wisesheets    — 免费财务摘要
+  - 东方财富       — A股公告/研报摘要
+  - 同花顺         — A股财务数据
+  - Yahoo Finance  — 基本面摘要（HTML解析）
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import time
+import urllib.parse
+import urllib.request
+from typing import Any, Dict, List, Optional
+
+from ..base import BaseDataSource, FundamentalsResult, HistoryResult, QuoteResult
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/120.0.0.0 Safari/537.36"
+    ),
+    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+    "Accept": "text/html,application/xhtml+xml,application/json,*/*;q=0.8",
+}
+
+
+def _fetch_html(url: str, timeout: int = 12, encoding: str = "utf-8") -> Optional[str]:
+    try:
+        req = urllib.request.Request(url, headers=_DEFAULT_HEADERS)
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            raw = resp.read()
+            try:
+                return raw.decode(encoding)
+            except UnicodeDecodeError:
+                return raw.decode("gbk", errors="replace")
+    except Exception as e:
+        logger.debug(f"[web_scraper] fetch {url[:80]} 失败: {e}")
+        return None
+
+
+def _fetch_json(url: str, timeout: int = 12, extra_headers: dict = None) -> Optional[Any]:
+    import json
+    try:
+        headers = {**_DEFAULT_HEADERS, **(extra_headers or {})}
+        req = urllib.request.Request(url, headers=headers)
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return json.loads(resp.read())
+    except Exception as e:
+        logger.debug(f"[web_scraper] json {url[:80]} 失败: {e}")
+        return None
+
+
+# ─── 东方财富 A股公告爬虫 ────────────────────────────────────────────────────
+
+def scrape_eastmoney_announcements(symbol: str, count: int = 10) -> List[Dict]:
+    """
+    爬取东方财富 A股公告列表（免费公开页面）。
+    symbol: 6位A股代码（如 '600519'）
+    """
+    s = symbol.replace(".SS", "").replace(".SZ", "").zfill(6)
+    url = (
+        f"https://np-anotice-stock.eastmoney.com/api/security/ann?"
+        f"sr=-1&page_size={count}&page_index=1&ann_type=A&client_source=web"
+        f"&stock_list={s}&f_node=0&s_node=0"
+    )
+    data = _fetch_json(url, extra_headers={"Referer": "https://www.eastmoney.com/"})
+    if not data or "data" not in data:
+        return []
+    anns = data["data"].get("list", [])
+    result = []
+    for a in anns[:count]:
+        result.append({
+            "title":   a.get("notice_title", ""),
+            "date":    a.get("notice_date", ""),
+            "type":    a.get("notice_type", ""),
+            "url":     f"https://data.eastmoney.com/notices/detail/{s}/{a.get('art_code','')}.html",
+            "source":  "eastmoney",
+        })
+    return result
+
+
+def scrape_eastmoney_news(symbol: str, count: int = 10) -> List[Dict]:
+    """爬取东方财富个股新闻。"""
+    s = symbol.replace(".SS", "").replace(".SZ", "").zfill(6)
+    url = (
+        f"https://newsapi.eastmoney.com/kuaixun/v1/getlist_101_ajaxResult_50_{s},,50,1.html"
+    )
+    html = _fetch_html(url)
+    if not html:
+        return []
+    # 尝试从 JSON 响应中解析
+    try:
+        import json
+        data = json.loads(html)
+        items = data.get("LiveList", [])[:count]
+        return [{
+            "title":  i.get("title", ""),
+            "time":   i.get("showtime", ""),
+            "source": i.get("medianame", ""),
+            "url":    i.get("url", ""),
+        } for i in items]
+    except Exception:
+        pass
+    return []
+
+
+# ─── Macrotrends 历史财务数据爬虫 ────────────────────────────────────────────
+
+def scrape_macrotrends_revenue(symbol: str) -> Optional[List[Dict]]:
+    """
+    从 Macrotrends 爬取年度营收历史（免费公开页面，无需登录）。
+    注意：依赖页面结构，如网站改版可能失效。
+    """
+    slug_map = {"AAPL": "apple", "MSFT": "microsoft", "GOOGL": "alphabet",
+                "AMZN": "amazon", "META": "meta-platforms", "TSLA": "tesla",
+                "NVDA": "nvidia", "JPM": "jpmorgan-chase", "V": "visa"}
+    slug = slug_map.get(symbol.upper())
+    if not slug:
+        slug = symbol.lower()
+
+    url = f"https://www.macrotrends.net/stocks/charts/{symbol.upper()}/{slug}/revenue"
+    html = _fetch_html(url)
+    if not html:
+        return None
+
+    # 提取嵌入的 JSON 数据
+    match = re.search(r'var originalData\s*=\s*(\[.*?\]);', html, re.DOTALL)
+    if not match:
+        return None
+    try:
+        import json
+        rows = json.loads(match.group(1))
+        return [
+            {"date": r.get("date", ""), "revenue_usd": r.get("v1", 0)}
+            for r in rows
+        ][-20:]  # 最近20个数据点
+    except Exception:
+        return None
+
+
+# ─── SSE/SZSE 交易所公开数据爬虫 ─────────────────────────────────────────────
+
+def scrape_sse_financials(symbol: str) -> Optional[Dict]:
+    """
+    爬取上交所公开的公司财务摘要（适用于 6 开头的A股）。
+    """
+    s = symbol.replace(".SS", "").zfill(6)
+    if not s.startswith(("6", "9")):
+        return None
+
+    url = (
+        f"https://query.sse.com.cn/commonSoaQuery.do?"
+        f"isPagination=false&sqlId=COMMON_SSE_CP_GPLB_GPXG_JBXX_L&stockCode={s}"
+    )
+    data = _fetch_json(url, extra_headers={
+        "Referer": "https://www.sse.com.cn/",
+        "Host": "query.sse.com.cn",
+    })
+    if not data or "result" not in data:
+        return None
+    r = data["result"]
+    if isinstance(r, list) and r:
+        r = r[0]
+    if not isinstance(r, dict):
+        return None
+    return {
+        "symbol":     s,
+        "name":       r.get("FULL_NAME", ""),
+        "industry":   r.get("INDUSTRY", ""),
+        "listing_date": r.get("LISTING_DATE", ""),
+        "total_shares": r.get("TOTAL_SHARES", ""),
+        "source":     "sse",
+    }
+
+
+def scrape_szse_financials(symbol: str) -> Optional[Dict]:
+    """爬取深交所公开的公司基本信息（适用于 0/3 开头的A股）。"""
+    s = symbol.replace(".SZ", "").zfill(6)
+    if not s.startswith(("0", "3", "2")):
+        return None
+
+    url = (
+        "https://www.szse.cn/api/report/show/nature/detail?"
+        "id=&"
+    )
+    # 深交所 API
+    data = _fetch_json(
+        "https://www.szse.cn/api/market/smstock/list?random=0.1&tab2PAGENUM=1"
+        "&tab2PAGENO=1&tab2COUNT=1&tab2SORTTAB=tab2&tab2SORTKEY=",
+        extra_headers={"Referer": "https://www.szse.cn/"},
+    )
+    return None  # 深交所API结构复杂，返回None让上层fallback
+
+
+# ─── 统一爬虫接口 ─────────────────────────────────────────────────────────────
+
+class WebScraperSource(BaseDataSource):
+    """
+    公开网页数据爬虫 — 汇总多个免费公开金融数据网站。
+    实现 BaseDataSource 接口以便接入路由器；quote/history 委托给主数据源，
+    fundamentals() 提供爬取的补充数据（公告、交易所基本信息等）。
+    """
+
+    name         = "web_scraper"
+    markets      = ["a_share", "us"]
+    requires_key = False
+
+    def is_configured(self) -> bool:
+        return True
+
+    def quote(self, symbol: str) -> Optional[QuoteResult]:
+        return None  # 不提供实时行情，由 akshare/yfinance 负责
+
+    def fundamentals(self, symbol: str) -> Optional[FundamentalsResult]:
+        """从交易所公开页面补充基本信息（仅 A 股）。"""
+        s = symbol.replace(".SS", "").replace(".SZ", "").zfill(6)
+        info = None
+        if s.startswith(("6", "9")):
+            info = scrape_sse_financials(s)
+        elif s.startswith(("0", "3")):
+            info = scrape_szse_financials(s)
+        if not info:
+            return None
+        return FundamentalsResult(
+            symbol = symbol,
+            source = self.name,
+        )
+
+    # ── 扩展方法（供上层直接调用） ─────────────────────────────────────────────
+
+    def get_announcements(self, symbol: str, count: int = 10) -> List[Dict]:
+        """A股公告列表（东方财富）。"""
+        return scrape_eastmoney_announcements(symbol, count)
+
+    def get_news(self, symbol: str, count: int = 10) -> List[Dict]:
+        """个股新闻。"""
+        return scrape_eastmoney_news(symbol, count)
+
+    def get_historical_revenue(self, symbol: str) -> Optional[List[Dict]]:
+        """美股历史营收（Macrotrends）。"""
+        return scrape_macrotrends_revenue(symbol)
+
+    def get_exchange_info(self, symbol: str) -> Optional[Dict]:
+        """A股交易所公开信息。"""
+        s = symbol.replace(".SS", "").replace(".SZ", "").zfill(6)
+        if s.startswith(("6", "9")):
+            return scrape_sse_financials(s)
+        elif s.startswith(("0", "3")):
+            return scrape_szse_financials(s)
+        return None
+
+    def bulk_scrape(self, symbols: List[str], data_type: str = "announcements") -> Dict[str, Any]:
+        """批量爬取，带速率限制（每次请求间隔 1 秒）。"""
+        results = {}
+        for sym in symbols:
+            try:
+                if data_type == "announcements":
+                    results[sym] = self.get_announcements(sym)
+                elif data_type == "news":
+                    results[sym] = self.get_news(sym)
+                elif data_type == "revenue":
+                    results[sym] = self.get_historical_revenue(sym)
+                time.sleep(1.0)
+            except Exception as e:
+                results[sym] = {"error": str(e)}
+        return results

aria_code/datasources/sources/world_bank_source.py

@@ -0,0 +1,206 @@
+"""
+datasources/sources/world_bank_source.py — 世界银行开放数据
+============================================================
+完全免费，无需 API key。数据来源：https://api.worldbank.org/v2/
+覆盖：GDP/人均GDP/通胀/贸易/外债/人口/能源等 16,000+ 指标，200+ 国家。
+
+常用指标:
+  NY.GDP.MKTP.CD  — GDP（当前美元）
+  NY.GDP.PCAP.CD  — 人均GDP
+  FP.CPI.TOTL.ZG  — 通货膨胀率（CPI 年增长率）
+  NE.TRD.GNFS.ZS  — 贸易占GDP比重
+  SL.UEM.TOTL.ZS  — 失业率
+  SP.POP.TOTL     — 总人口
+  BX.KLT.DINV.CD.WD — 外商直接投资（净流入）
+"""
+
+from __future__ import annotations
+
+import logging
+import urllib.request
+from typing import Any, Dict, List, Optional
+
+from ..base import BaseDataSource, HistoryResult, QuoteResult
+
+logger = logging.getLogger(__name__)
+
+_BASE = "https://api.worldbank.org/v2"
+
+# 常用指标别名
+INDICATOR_ALIASES: Dict[str, str] = {
+    "GDP":       "NY.GDP.MKTP.CD",
+    "GDPPC":     "NY.GDP.PCAP.CD",
+    "GDPGROWTH": "NY.GDP.MKTP.KD.ZG",
+    "CPI":       "FP.CPI.TOTL.ZG",
+    "INFLATION": "FP.CPI.TOTL.ZG",
+    "UNRATE":    "SL.UEM.TOTL.ZS",
+    "POPULATION":"SP.POP.TOTL",
+    "TRADE":     "NE.TRD.GNFS.ZS",
+    "FDI":       "BX.KLT.DINV.CD.WD",
+    "DEBT":      "GC.DOD.TOTL.GD.ZS",
+    "EXPORTS":   "NE.EXP.GNFS.ZS",
+    "IMPORTS":   "NE.IMP.GNFS.ZS",
+}
+
+# 国家代码别名
+COUNTRY_ALIASES: Dict[str, str] = {
+    "CHINA": "CN", "CN": "CN", "CHN": "CN",
+    "US": "US", "USA": "US", "AMERICA": "US",
+    "JAPAN": "JP", "JP": "JP",
+    "GERMANY": "DE", "DE": "DE",
+    "UK": "GB", "GB": "GB",
+    "INDIA": "IN", "IN": "IN",
+    "BRAZIL": "BR",
+    "WORLD": "WLD", "GLOBAL": "WLD",
+    "G7": "G7", "G20": "G20",
+}
+
+
+def _fetch(url: str, timeout: int = 15) -> Optional[List]:
+    import json
+    try:
+        req = urllib.request.Request(
+            url + "&format=json&per_page=100",
+            headers={"User-Agent": "aria-code/1.0"}
+        )
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            data = json.loads(resp.read())
+        if isinstance(data, list) and len(data) >= 2:
+            return data[1]
+        return None
+    except Exception as e:
+        logger.debug(f"[world_bank] fetch 失败: {e}")
+        return None
+
+
+class WorldBankSource(BaseDataSource):
+    """世界银行开放数据 — 宏观经济与发展指标。"""
+
+    name         = "world_bank"
+    markets      = ["macro", "us", "cn"]
+    requires_key = False
+
+    def is_configured(self) -> bool:
+        return True
+
+    def supports(self, symbol: str) -> bool:
+        s = symbol.upper()
+        return (
+            "." in s and len(s) > 5 or  # looks like WB indicator (NY.GDP.*)
+            s in INDICATOR_ALIASES
+        )
+
+    def _resolve_indicator(self, indicator: str) -> str:
+        return INDICATOR_ALIASES.get(indicator.upper(), indicator)
+
+    def _resolve_country(self, country: str) -> str:
+        return COUNTRY_ALIASES.get(country.upper(), country.upper())
+
+    def get_indicator(
+        self,
+        indicator: str,
+        country: str = "WLD",
+        start_year: int = 2000,
+        end_year: int = 2024,
+    ) -> Optional[HistoryResult]:
+        """
+        获取指定国家/地区的经济指标历史序列。
+
+        indicator: 世界银行指标代码或别名（GDP/CPI/UNRATE 等）
+        country:   ISO 2位代码或别名（CN/US/JP/WORLD 等）
+        """
+        try:
+            import pandas as pd
+            ind = self._resolve_indicator(indicator)
+            cty = self._resolve_country(country)
+            url = (f"{_BASE}/country/{cty}/indicator/{ind}"
+                   f"?date={start_year}:{end_year}")
+            rows_raw = _fetch(url)
+            if not rows_raw:
+                return None
+            rows = []
+            for r in rows_raw:
+                if r.get("value") is not None:
+                    rows.append({
+                        "date": f"{r['date']}-12-31",
+                        "close": float(r["value"]),
+                        "country": r.get("country", {}).get("value", cty),
+                    })
+            if not rows:
+                return None
+            df = pd.DataFrame(rows)
+            df["date"] = pd.to_datetime(df["date"])
+            df = df.set_index("date").sort_index()
+            return HistoryResult(symbol=f"{country}:{indicator}", data=df,
+                                 source=self.name, interval="1y")
+        except Exception as e:
+            logger.debug(f"[world_bank] get_indicator 失败: {e}")
+            return None
+
+    def compare_countries(
+        self,
+        indicator: str,
+        countries: List[str],
+        start_year: int = 2010,
+    ) -> Optional[Dict[str, Any]]:
+        """
+        多国横向对比同一指标。
+        返回: {country: [(year, value), ...], ...}
+        """
+        ind = self._resolve_indicator(indicator)
+        cty_codes = ";".join(self._resolve_country(c) for c in countries)
+        url = (f"{_BASE}/country/{cty_codes}/indicator/{ind}"
+               f"?date={start_year}:2024")
+        rows_raw = _fetch(url)
+        if not rows_raw:
+            return None
+
+        result: Dict[str, list] = {}
+        for r in rows_raw:
+            if r.get("value") is None:
+                continue
+            cty_name = r.get("country", {}).get("value", "?")
+            yr = r.get("date", "")
+            val = float(r["value"])
+            result.setdefault(cty_name, []).append((yr, val))
+
+        for k in result:
+            result[k].sort(key=lambda x: x[0])
+        return result
+
+    def search_indicators(self, query: str, limit: int = 10) -> List[Dict]:
+        """关键词搜索世界银行指标。"""
+        import json
+        import urllib.parse
+        q = urllib.parse.quote(query)
+        url = f"{_BASE}/indicator?format=json&per_page={limit}&mrv=1"
+        try:
+            req = urllib.request.Request(
+                f"{_BASE}/indicator?format=json&per_page={limit}&source=2",
+                headers={"User-Agent": "aria-code/1.0"}
+            )
+            with urllib.request.urlopen(req, timeout=10) as resp:
+                data = json.loads(resp.read())
+            if isinstance(data, list) and len(data) >= 2:
+                return [
+                    {"id": i["id"], "name": i["name"],
+                     "source": i.get("source", {}).get("value", "")}
+                    for i in data[1] if query.lower() in i.get("name", "").lower()
+                ][:limit]
+        except Exception as e:
+            logger.debug(f"[world_bank] search 失败: {e}")
+        return []
+
+    def quote(self, symbol: str) -> Optional[QuoteResult]:
+        # World Bank indicators aren't real-time quotes
+        return None
+
+    def history(self, symbol: str, days: int = 365 * 5, interval: str = "1y") -> Optional[HistoryResult]:
+        # Parse "CN:GDP" or just "GDP" (default to WLD)
+        if ":" in symbol:
+            country, indicator = symbol.split(":", 1)
+        else:
+            country, indicator = "WLD", symbol
+        years = max(1, days // 365)
+        start = 2024 - years
+        return self.get_indicator(indicator, country, start_year=start)

aria_code/datasources/sources/yfinance_source.py

@@ -0,0 +1,152 @@
+"""
+datasources/sources/yfinance_source.py — yfinance 美股/港股/加密 数据源
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from ..base import BaseDataSource, FundamentalsResult, HistoryResult, QuoteResult, _detect_market
+
+logger = logging.getLogger(__name__)
+
+
+class YFinanceSource(BaseDataSource):
+
+    name         = "yfinance"
+    markets      = ["us", "hk", "crypto"]
+    requires_key = False
+
+    def _to_yf_symbol(self, symbol: str) -> str:
+        s = symbol.upper()
+        if "/" in s:   # BTC/USDT → BTC-USD
+            base, quote = s.split("/", 1)
+            return f"{base}-{'USD' if 'USDT' in quote else quote}"
+        return s
+
+    def quote(self, symbol: str) -> Optional[QuoteResult]:
+        try:
+            import yfinance as yf
+            yf_sym = self._to_yf_symbol(symbol)
+            ticker = yf.Ticker(yf_sym)
+
+            price = None
+            prev  = None
+            info  = {}
+
+            # Attempt 1: fast_info (lighter, often succeeds when info is rate-limited)
+            try:
+                fi = ticker.fast_info
+                price = getattr(fi, "last_price", None) or getattr(fi, "previous_close", None)
+                prev  = getattr(fi, "previous_close", None) or price
+            except Exception:
+                pass
+
+            # Attempt 2: full info dict
+            if not price:
+                try:
+                    info  = ticker.info or {}
+                    price = (info.get("currentPrice") or info.get("regularMarketPrice")
+                             or info.get("previousClose"))
+                    prev  = info.get("previousClose") or price
+                except Exception:
+                    pass
+
+            # Attempt 3: last close from recent history
+            if not price:
+                try:
+                    hist = ticker.history(period="5d", interval="1d", auto_adjust=True)
+                    if hist is not None and not hist.empty:
+                        close_col = next((c for c in hist.columns if c.lower() == "close"), None)
+                        if close_col:
+                            price = float(hist[close_col].iloc[-1])
+                            prev  = float(hist[close_col].iloc[-2]) if len(hist) > 1 else price
+                except Exception:
+                    pass
+
+            if not price:
+                return None
+
+            # Fetch info lazily if not already fetched
+            if not info:
+                try:
+                    info = ticker.info or {}
+                except Exception:
+                    pass
+
+            chg   = price - prev if prev else 0.0
+            chg_p = (chg / prev * 100) if prev else 0.0
+            market   = _detect_market(symbol)
+            currency = "HKD" if market == "hk" else "USD"
+
+            return QuoteResult(
+                symbol      = symbol,
+                name        = info.get("shortName") or info.get("longName") or symbol,
+                price       = float(price),
+                change      = float(chg),
+                change_pct  = float(chg_p),
+                volume      = float(info.get("volume") or info.get("regularMarketVolume") or 0),
+                market_cap  = float(info.get("marketCap") or 0),
+                pe_ttm      = float(info.get("trailingPE") or 0),
+                pb          = float(info.get("priceToBook") or 0),
+                high_52w    = float(info.get("fiftyTwoWeekHigh") or 0),
+                low_52w     = float(info.get("fiftyTwoWeekLow") or 0),
+                currency    = currency,
+                market      = market,
+                source      = self.name,
+            )
+        except Exception as e:
+            logger.debug(f"[yfinance] quote {symbol} 失败: {e}")
+            return None
+
+    def history(self, symbol: str, days: int = 90, interval: str = "1d") -> Optional[HistoryResult]:
+        try:
+            import yfinance as yf
+            ticker = yf.Ticker(self._to_yf_symbol(symbol))
+            period = f"{days}d" if days <= 730 else "2y"
+            df     = ticker.history(period=period, interval=interval, auto_adjust=True)
+            if df is None or df.empty:
+                return None
+            df.columns = [c.lower() for c in df.columns]
+            return HistoryResult(symbol=symbol, data=df, source=self.name, interval=interval)
+        except Exception as e:
+            logger.debug(f"[yfinance] history {symbol} 失败: {e}")
+            return None
+
+    def fundamentals(self, symbol: str) -> Optional[FundamentalsResult]:
+        try:
+            import yfinance as yf
+            info = yf.Ticker(self._to_yf_symbol(symbol)).info or {}
+            if not info:
+                return None
+
+            def _f(key: str, mult: float = 1.0) -> Optional[float]:
+                v = info.get(key)
+                if v is None or v != v:  # None or NaN
+                    return None
+                fv = float(v) * mult
+                return fv if fv != 0.0 else None
+
+            result = FundamentalsResult(
+                symbol            = symbol,
+                pe_ttm            = _f("trailingPE"),
+                pb                = _f("priceToBook"),
+                roe               = _f("returnOnEquity", 100),
+                revenue_growth    = _f("revenueGrowth",  100),
+                net_profit_growth = _f("earningsGrowth", 100),
+                # trailingAnnualDividendYield is consistently a fraction (e.g. 0.0035 = 0.35%)
+                # dividendYield is unreliable (sometimes already pct, sometimes fraction)
+                dividend_yield    = _f("trailingAnnualDividendYield", 100),
+                total_mv          = _f("marketCap"),
+                source            = self.name,
+            )
+            # Return None only if every field is None (data completely missing)
+            has_any = any(
+                getattr(result, f) is not None
+                for f in ("pe_ttm", "pb", "roe", "revenue_growth", "total_mv")
+            )
+            return result if has_any else None
+        except Exception as e:
+            logger.debug(f"[yfinance] fundamentals {symbol} 失败: {e}")
+            return None