stock-analysis-plugin 0.2.11 → 0.2.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +234 -382
  2. package/docs/mcp.md +60 -0
  3. package/hermes/__init__.py +35 -0
  4. package/hermes/plugin.yaml +132 -0
  5. package/hermes/schemas.py +6 -0
  6. package/hermes/tools.py +647 -0
  7. package/mcp_host/__init__.py +1 -0
  8. package/mcp_host/__main__.py +3 -0
  9. package/mcp_host/server.py +222 -0
  10. package/package.json +7 -1
  11. package/pi/index.ts +52 -2906
  12. package/scripts/cli-json.mjs +23 -0
  13. package/scripts/exec-process.mjs +89 -0
  14. package/scripts/setup-python.mjs +37 -4
  15. package/scripts/tool-calls.mjs +493 -0
  16. package/skills/bull-trend/SKILL.md +2 -2
  17. package/skills/bull-trend/references/fat-tails.md +76 -3
  18. package/skills/bull-trend/scripts/tail_analysis.py +92 -3
  19. package/skills/event-driven/SKILL.md +2 -2
  20. package/skills/growth-quality/SKILL.md +1 -1
  21. package/skills/market-review/SKILL.md +1 -0
  22. package/skills/stock-analysis/SKILL.md +12 -7
  23. package/skills/stock-analysis/references/stateless-analysis.md +9 -1
  24. package/skills/stock-screener/SKILL.md +1 -1
  25. package/skills/strategy-backtest/SKILL.md +1 -1
  26. package/tools/_cache.py +13 -0
  27. package/tools/_subproc.py +46 -0
  28. package/tools/alert_rules.py +2 -2
  29. package/tools/backtest.py +3 -3
  30. package/tools/capabilities.py +33 -9
  31. package/tools/cli_budgets.json +1 -0
  32. package/tools/credit_data.py +293 -0
  33. package/tools/diagnostics.py +3 -4
  34. package/tools/gather.py +2 -6
  35. package/tools/import_parser.py +3 -4
  36. package/tools/lixinger.py +119 -27
  37. package/tools/market_regime.py +5 -2
  38. package/tools/market_review.py +5 -6
  39. package/tools/name_resolver.py +45 -33
  40. package/tools/report_renderer.py +3 -3
  41. package/tools/requirements.txt +5 -4
  42. package/tools/risk_screening.py +3 -2
  43. package/tools/screener.py +41 -30
  44. package/tools/search_intel.py +79 -10
  45. package/tools/sec_edgar.py +268 -1
  46. package/tools/sec_holders.py +6 -0
  47. package/tools/signal_tracker.py +5 -5
  48. package/tools/stateless_analysis.py +132 -19
  49. package/tools/stock_data.py +1020 -817
  50. package/tools/tiaa_disclosures.py +4 -0
  51. package/tools/tool_schemas.json +2734 -0
  52. package/tools/trading_calendar.py +65 -30
  53. package/tools/value_data.py +423 -175
  54. package/tools/watchlist.py +43 -5
@@ -18,6 +18,9 @@ INDEX_MAP = {
18
18
  "A": ("000001", "上证综指"),
19
19
  "HK": ("^HSI", "恒生指数"),
20
20
  "US": ("^GSPC", "标普500"),
21
+ "JP": ("^N225", "日经225"),
22
+ "KR": ("^KS11", "韩国KOSPI"),
23
+ "TW": ("^TWII", "台湾加权指数"),
21
24
  }
22
25
 
23
26
  REGIME_CN = {
@@ -166,7 +169,7 @@ def _detect_sector_heat(market: str) -> bool:
166
169
  def detect_regime(market: str = "A") -> dict:
167
170
  market = market.upper()
168
171
  if market not in INDEX_MAP:
169
- return {"error": f"Unsupported market: {market}. Use A/HK/US"}
172
+ return {"error": f"Unsupported market: {market}. Use {'/'.join(INDEX_MAP)}"}
170
173
 
171
174
  code, name = INDEX_MAP[market]
172
175
  klines = _get_index_kline(market)
@@ -229,7 +232,7 @@ def main():
229
232
  sub = parser.add_subparsers(dest="command")
230
233
 
231
234
  p_detect = sub.add_parser("detect")
232
- p_detect.add_argument("market", nargs="?", default="A", choices=["A", "HK", "US"])
235
+ p_detect.add_argument("market", nargs="?", default="A", choices=list(INDEX_MAP))
233
236
 
234
237
  args = parser.parse_args()
235
238
  if not args.command:
@@ -18,9 +18,10 @@ from pathlib import Path
18
18
 
19
19
  sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
20
20
 
21
- from _subproc import command_timeout, utf8_stdio
21
+ from _subproc import command_timeout, error_message, utf8_stdio
22
22
  from _subproc import run_tool as _run_tool
23
23
  from signal_tracker import _append_record, _load_records, _store_lock
24
+ from trading_calendar import market_now
24
25
 
25
26
 
26
27
  def _index_change(row):
@@ -81,8 +82,6 @@ def _signal_to_stance(signal: str) -> str | None:
81
82
 
82
83
  def review_market(market: str = "A", *, run=None) -> dict:
83
84
  """Gather market data and compute review for a single market."""
84
- from datetime import datetime
85
-
86
85
  tasks = {}
87
86
 
88
87
  if market == "A":
@@ -135,7 +134,7 @@ def review_market(market: str = "A", *, run=None) -> dict:
135
134
 
136
135
  output = {
137
136
  "market": market,
138
- "date": datetime.now().strftime("%Y-%m-%d"),
137
+ "date": market_now("CN" if market == "A" else market).strftime("%Y-%m-%d"),
139
138
  "indices": results.get("indices"),
140
139
  "stats": results.get("stats"),
141
140
  "sectors": results.get("sectors"),
@@ -169,7 +168,7 @@ def save_review(report: dict, store: Path | None = None) -> dict:
169
168
  with _store_lock(store):
170
169
  _append_record(store, report)
171
170
  except (OSError, ValueError) as e:
172
- return {"error": str(e)}
171
+ return {"error": error_message(e)}
173
172
  return {"saved": True, "date": report.get("review_date") or report.get("date")}
174
173
 
175
174
 
@@ -181,7 +180,7 @@ def cmd_history(args) -> dict:
181
180
  records = _load_records(store)
182
181
  except (OSError, UnicodeDecodeError) as e:
183
182
  # store is a directory / not UTF-8 — same clean-error contract as save_review
184
- return {"error": str(e)}
183
+ return {"error": error_message(e)}
185
184
  limit = max(args.limit, 0)
186
185
  recent = records[-limit:][::-1] if limit else []
187
186
  return {"count": len(recent), "records": recent}
@@ -3,40 +3,40 @@
3
3
 
4
4
  import argparse
5
5
  import json
6
+ import re
6
7
  import sys
7
8
  from difflib import SequenceMatcher
8
9
  from pathlib import Path
9
10
 
10
11
  sys.path.insert(0, str(Path(__file__).resolve().parent))
11
12
 
12
- from _cache import DAY, cached # noqa: E402
13
- from _subproc import utf8_stdio # noqa: E402
14
- from stock_data import _akshare_retry # noqa: E402
15
-
16
-
17
- def _load_stock_map() -> list[dict]:
18
- def fetch():
19
- import akshare as ak
20
-
21
- df = _akshare_retry(ak.stock_zh_a_spot_em)
22
- records = []
23
- for row in df.to_dict("records"):
24
- code, name = str(row["代码"]), row["名称"]
25
- if (
26
- len(code) != 6
27
- or not code.isascii()
28
- or not code.isdigit()
29
- or not isinstance(name, str)
30
- or not name.strip()
31
- ):
32
- raise ValueError("Stock name map contains invalid codes or names")
33
- records.append({"code": code, "name": name})
34
- return records
35
-
36
- try:
37
- return cached("stock-name-map:v1", DAY, fetch)
38
- except Exception as e:
39
- raise RuntimeError(f"Failed to load stock list: {e}") from e
13
+ from _subproc import error_message, utf8_stdio # noqa: E402
14
+ from stock_data import _em_codetable, normalize_us_symbol # noqa: E402
15
+
16
+ _A_TYPES = {"沪A", "深A", "京A", "科创板", "创业板"}
17
+
18
+
19
+ def _search(query: str, count: int) -> list[dict]:
20
+ """Eastmoney code-table search (code / name / pinyin / initials), kept to A/HK/US listed stocks."""
21
+ stocks = []
22
+ for row in _em_codetable(query, count):
23
+ code, name, market, kind = (
24
+ str(row.get("code", "")),
25
+ row.get("shortName"),
26
+ row.get("market"),
27
+ row.get("securityTypeName"),
28
+ )
29
+ if not isinstance(name, str) or not name.strip():
30
+ continue
31
+ if market in (0, 1) and kind in _A_TYPES and re.fullmatch(r"\d{6}", code):
32
+ stocks.append({"code": code, "name": name})
33
+ elif market == 116 and kind == "港股" and re.fullmatch(r"\d{5}", code):
34
+ stocks.append({"code": f"{code}.HK", "name": name})
35
+ elif market in (105, 106, 107) and kind == "美股":
36
+ symbol = normalize_us_symbol(code)
37
+ if re.fullmatch(r"[A-Z0-9-]{1,10}", symbol):
38
+ stocks.append({"code": symbol, "name": name})
39
+ return stocks
40
40
 
41
41
 
42
42
  def _to_pinyin(text: str) -> str:
@@ -57,16 +57,16 @@ def _to_pinyin_initials(text: str) -> str:
57
57
  return ""
58
58
 
59
59
 
60
- def resolve(query: str, top: int = 5) -> list[dict]:
61
- stocks = _load_stock_map()
60
+ def _resolve_stocks(query: str, top: int, stocks: list[dict]) -> list[dict]:
62
61
  query = query.strip()
63
62
  q_lower = query.lower()
63
+ q_code = query.upper()
64
64
 
65
- exact = [s for s in stocks if s["code"] == query or s["name"] == query]
65
+ exact = [s for s in stocks if s["code"].upper() == q_code or s["name"] == query]
66
66
  if exact:
67
67
  return exact[:top]
68
68
 
69
- prefix_code = [s for s in stocks if s["code"].startswith(query)]
69
+ prefix_code = [s for s in stocks if s["code"].upper().startswith(q_code)]
70
70
  if prefix_code:
71
71
  return prefix_code[:top]
72
72
 
@@ -104,6 +104,18 @@ def resolve(query: str, top: int = 5) -> list[dict]:
104
104
  return [{"code": s["code"], "name": s["name"], "score": round(sc, 2)} for sc, s in scored[:top]]
105
105
 
106
106
 
107
+ def resolve(query: str, top: int = 5) -> list[dict]:
108
+ query = query.strip()
109
+ if not query:
110
+ return []
111
+ try:
112
+ # The provider ranks loosely; over-fetch so the local exact/prefix/contains/pinyin order decides.
113
+ stocks = _search(query, max(top, 20))
114
+ except Exception as e:
115
+ raise RuntimeError(f"Failed to search stock names: {error_message(e)}") from e
116
+ return _resolve_stocks(query, top, stocks)
117
+
118
+
107
119
  def main():
108
120
  parser = argparse.ArgumentParser(description="Stock name resolver")
109
121
  sub = parser.add_subparsers(dest="command")
@@ -120,7 +132,7 @@ def main():
120
132
  try:
121
133
  result = resolve(args.query, args.top)
122
134
  except RuntimeError as e:
123
- result = {"error": str(e)}
135
+ result = {"error": error_message(e)}
124
136
  print(json.dumps(result, ensure_ascii=False))
125
137
 
126
138
 
@@ -12,7 +12,7 @@ from datetime import datetime, timezone
12
12
  from pathlib import Path
13
13
 
14
14
  sys.path.insert(0, str(Path(__file__).resolve().parent))
15
- from _subproc import utf8_stdio
15
+ from _subproc import error_message, utf8_stdio
16
16
  from market_review import save_review
17
17
 
18
18
  try:
@@ -55,12 +55,12 @@ def render(kind: str, template: str, report: dict) -> dict:
55
55
  try:
56
56
  tpl = _env().get_template(tpl_path)
57
57
  except Exception as e:
58
- return {"error": f"failed to load template {tpl_path}: {e}"}
58
+ return {"error": f"failed to load template {tpl_path}: {error_message(e)}"}
59
59
 
60
60
  try:
61
61
  content = tpl.render(**report)
62
62
  except Exception as e:
63
- return {"error": f"render failed: {e}", "template": tpl_path}
63
+ return {"error": f"render failed: {error_message(e)}", "template": tpl_path}
64
64
 
65
65
  return {
66
66
  "meta": {
@@ -1,16 +1,17 @@
1
1
  akshare>=1.20.24; python_version >= "3.11"
2
2
  akshare>=1.14.0; python_version < "3.11"
3
3
  yfinance>=0.2.36
4
- pandas>=2.0.0
4
+ pandas>=2.0.0,<4
5
5
  numpy>=1.24.0
6
6
  pyyaml>=6.0
7
7
  pypinyin>=0.50.0
8
+ pypdf>=5.0.0
8
9
  requests>=2.33.0
9
10
  exchange-calendars>=4.5.0
10
- newspaper4k>=0.9.6.1
11
+ jinja2>=3.1.6
12
+ # optional providers: failover-only sources; setup-python.mjs only warns when one cannot install
11
13
  efinance>=0.5.0
12
14
  baostock>=0.8.8
13
- pytdx>=1.72
14
15
  longport>=1.0.0
15
16
  finnhub-python>=2.4.0
16
- jinja2>=3.1.6
17
+ newspaper4k>=0.9.6.1
@@ -7,12 +7,13 @@ import math
7
7
  import os
8
8
  import sys
9
9
  from concurrent.futures import Future, ThreadPoolExecutor
10
- from datetime import datetime, timedelta
10
+ from datetime import timedelta
11
11
 
12
12
  sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
13
13
 
14
14
  from _subproc import run_tool as _run_tool
15
15
  from _subproc import utf8_stdio
16
+ from trading_calendar import market_now
16
17
 
17
18
 
18
19
  def _safe_float(val) -> float | None:
@@ -181,7 +182,7 @@ def _fetch_lockup(symbol: str) -> dict:
181
182
  if df.empty:
182
183
  return result
183
184
 
184
- now = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
185
+ now = market_now("CN").replace(hour=0, minute=0, second=0, microsecond=0)
185
186
  cutoff = now + timedelta(days=30)
186
187
 
187
188
  date_col = None
package/tools/screener.py CHANGED
@@ -21,7 +21,6 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
21
21
 
22
22
  from _subproc import command_timeout, run_tool, utf8_stdio
23
23
  from market_review import calc_temperature
24
- from stock_data import compute_market_stats
25
24
 
26
25
  DEFAULT_FILTERS = {
27
26
  "pe_min": 0,
@@ -56,12 +55,10 @@ def _fetch_json(args: list[str], timeout: int | None = None) -> dict | list | No
56
55
  return run_tool("stock_data.py", args, timeout=timeout, preserve_errors=True)
57
56
 
58
57
 
59
- def fetch_snapshot(market: str) -> list:
60
- # a cold full-market snapshot can exceed 30s (CI runners) — allow longer; with
61
- # all eastmoney/sina legs timing out, the tencent fallback leg needs ~2min more
62
- return _fetch_json(
63
- ["market_snapshot", "--market", market], timeout=command_timeout("stock_data.py", ["market_snapshot"])
64
- )
58
+ def fetch_snapshot(market: str, filters: dict) -> dict:
59
+ """Stocks at or above filters["market_cap_min"], largest first (A also filters server-side)."""
60
+ args = ["market_snapshot", "--market", market, "--filters", json.dumps(filters)]
61
+ return _fetch_json(args, timeout=command_timeout("stock_data.py", ["market_snapshot"]))
65
62
 
66
63
 
67
64
  def load_config(path: str) -> dict:
@@ -148,12 +145,13 @@ def compute_scores(df: pd.DataFrame) -> pd.DataFrame:
148
145
  mom_parts.append(_minmax(df["volume_ratio"]))
149
146
  df["momentum_score"] = pd.concat(mom_parts, axis=1).mean(axis=1) if mom_parts else np.nan
150
147
 
151
- # Liquidity: higher turnover_rate and volume → higher score
148
+ # Liquidity: higher turnover_rate and money traded → higher score. Share volume would
149
+ # favour low-priced stocks, so the cross-stock leg ranks turnover (成交额).
152
150
  liq_parts = []
153
151
  if "turnover_rate" in df.columns:
154
152
  liq_parts.append(_minmax(df["turnover_rate"]))
155
- if "volume" in df.columns:
156
- liq_parts.append(_minmax(df["volume"].astype(float)))
153
+ if "turnover" in df.columns:
154
+ liq_parts.append(_minmax(df["turnover"].astype(float)))
157
155
  df["liquidity_score"] = pd.concat(liq_parts, axis=1).mean(axis=1) if liq_parts else np.nan
158
156
 
159
157
  groups = df[[f"{group}_score" for group in L1_WEIGHTS]].rename(columns=lambda name: name.removesuffix("_score"))
@@ -166,19 +164,18 @@ def sentiment_multiplier(score: float) -> float:
166
164
  return round(0.8 + score / 100 * 0.4, 3) # linear map to [0.8, 1.2]
167
165
 
168
166
 
169
- def compute_sentiment(market: str, snapshot: list | None = None) -> dict:
167
+ def compute_sentiment(market: str) -> dict:
170
168
  """Market sentiment gate; degrades to a neutral multiplier of 1.0 on any failure."""
171
169
  fallback = {"score": None, "level": "unknown", "signal": None, "multiplier": 1.0}
172
170
  region = _INDEX_REGION.get(market)
173
171
  if region is None:
174
172
  return fallback
175
173
  try:
176
- stats = None
177
- if isinstance(snapshot, list) and snapshot:
178
- try:
179
- stats = compute_market_stats(snapshot)
180
- except Exception:
181
- stats = None
174
+ stats = _fetch_json(
175
+ ["market_stats", "--market", market], timeout=command_timeout("stock_data.py", ["market_stats"])
176
+ )
177
+ if not isinstance(stats, dict) or "error" in stats:
178
+ stats = None
182
179
  indices = _fetch_json(["market_indices", "--region", region])
183
180
  if not isinstance(indices, list) or not indices:
184
181
  return fallback
@@ -371,22 +368,29 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
371
368
  )
372
369
  if dividend_requested and market not in ("A", "HK"):
373
370
  return {"error": "dividend screening currently covers A/HK only", "market": market}
374
- raw = fetch_snapshot(market)
375
- if isinstance(raw, dict) and "error" in raw:
376
- return raw
377
- # snapshot legs wrap a total failure as a one-row [{"error": ...}] list — surface it
378
- # instead of letting the error row walk the scoring pipeline as a ghost candidate
379
- if isinstance(raw, list) and len(raw) == 1 and isinstance(raw[0], dict) and "error" in raw[0]:
380
- return {**raw[0], "market": market}
381
- if raw is None:
371
+ # Sentiment is an independent market_stats call; overlap it with the snapshot fetch.
372
+ pool = ThreadPoolExecutor(max_workers=1)
373
+ sentiment_future = pool.submit(compute_sentiment, market)
374
+ pool.shutdown(wait=False)
375
+ snapshot = fetch_snapshot(market, filters)
376
+ if isinstance(snapshot, dict) and "error" in snapshot:
377
+ return snapshot
378
+ if not isinstance(snapshot, dict) or not isinstance(snapshot.get("rows"), list):
382
379
  return {"error": "failed to fetch market snapshot", "market": market}
383
380
 
384
381
  l2_enabled = bool(l2 or (config or {}).get("l2"))
385
- sentiment = compute_sentiment(market, raw if isinstance(raw, list) else None)
382
+ sentiment = sentiment_future.result()
386
383
  multiplier = sentiment["multiplier"]
387
384
 
388
- df = pd.DataFrame(raw)
389
- total = len(df)
385
+ # An empty server-filtered page still reports every requested filter as applied.
386
+ columns = ["symbol", *{key.rsplit("_", 1)[0] for key in filters}]
387
+ df = pd.DataFrame(snapshot["rows"]) if snapshot["rows"] else pd.DataFrame(columns=columns)
388
+ total = snapshot.get("total_stocks")
389
+ scan_meta = {}
390
+ if snapshot.get("truncated"):
391
+ scan_meta["warnings"] = [
392
+ f"scanned only the {len(df)} largest stocks by market cap; raise market_cap_min to cover the rest"
393
+ ]
390
394
 
391
395
  for c in ["price", "change_pct", "volume", "turnover", "pe", "pb", "market_cap", "turnover_rate", "volume_ratio"]:
392
396
  if c in df.columns:
@@ -402,7 +406,11 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
402
406
  try:
403
407
  from value_data import dividend_ranking
404
408
 
405
- ranked = dividend_ranking(report_date=(config or {}).get("dividend_report_date", ""), market=market)
409
+ ranked = dividend_ranking(
410
+ report_date=(config or {}).get("dividend_report_date", ""),
411
+ market=market,
412
+ codes=list(df["symbol"]) if market == "A" else None,
413
+ )
406
414
 
407
415
  def dividend_key(symbol):
408
416
  return symbol.split(".")[0].lstrip("0") if market == "HK" else symbol
@@ -440,6 +448,7 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
440
448
  "market_sentiment": sentiment,
441
449
  "l2_enabled": l2_enabled,
442
450
  **dividend_meta,
451
+ **scan_meta,
443
452
  }
444
453
 
445
454
  df = compute_scores(df)
@@ -465,6 +474,7 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
465
474
  "price",
466
475
  "change_pct",
467
476
  "volume",
477
+ "turnover",
468
478
  "turnover_rate",
469
479
  "pe",
470
480
  "pe_basis",
@@ -489,7 +499,7 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
489
499
  factor_inputs = {
490
500
  "value": ("pe", "pb", "dividend_yield"),
491
501
  "momentum": ("change_pct", "volume_ratio"),
492
- "liquidity": ("turnover_rate", "volume"),
502
+ "liquidity": ("turnover_rate", "turnover"),
493
503
  }
494
504
  weights = L1_WEIGHTS
495
505
  available = [
@@ -529,6 +539,7 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
529
539
  "market_sentiment": sentiment,
530
540
  "l2_enabled": l2_enabled,
531
541
  **dividend_meta,
542
+ **scan_meta,
532
543
  }
533
544
 
534
545
 
@@ -9,9 +9,10 @@ import re
9
9
  import sys
10
10
  import time as _time
11
11
  from contextvars import ContextVar
12
+ from urllib.parse import urlparse
12
13
 
13
14
  sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
14
- from _subproc import scrub_secrets, utf8_stdio
15
+ from _subproc import error_message, scrub_secrets, utf8_stdio
15
16
 
16
17
  _SEARCH_STATE = ContextVar("search_budget", default=None)
17
18
 
@@ -474,25 +475,90 @@ def get_trending_sentiment() -> dict:
474
475
  # --------------- Article Extraction ---------------
475
476
 
476
477
 
477
- def extract_article(url: str) -> dict:
478
+ def _extract_pdf(url: str, max_chars: int, response) -> dict:
479
+ import io
480
+
481
+ content = bytearray()
482
+ try:
483
+ from pypdf import PdfReader
484
+
485
+ for chunk in response.iter_content(65536):
486
+ content.extend(chunk)
487
+ if len(content) > 25 * 1024 * 1024:
488
+ raise ValueError("PDF exceeds the 25 MiB download limit")
489
+ finally:
490
+ response.close()
491
+ if not content.startswith(b"%PDF-"):
492
+ raise ValueError("URL did not return a PDF document")
493
+
494
+ reader = PdfReader(io.BytesIO(content))
495
+ parts = []
496
+ for page in reader.pages:
497
+ parts.append(page.extract_text() or "")
498
+ if sum(map(len, parts)) > max_chars:
499
+ break
500
+ text = "\n".join(parts)
501
+ metadata = reader.metadata or {}
502
+ return {
503
+ "url": url,
504
+ "title": str(metadata.get("/Title") or ""),
505
+ "text": text[:max_chars],
506
+ "authors": [str(metadata["/Author"])] if metadata.get("/Author") else [],
507
+ "publish_date": str(metadata["/CreationDate"]) if metadata.get("/CreationDate") else None,
508
+ "top_image": None,
509
+ "format": "pdf",
510
+ "page_count": len(reader.pages),
511
+ "truncated": len(text) > max_chars,
512
+ }
513
+
514
+
515
+ def extract_article(url: str, max_chars: int = 3000) -> dict:
516
+ if isinstance(max_chars, bool) or not isinstance(max_chars, int) or not 1 <= max_chars <= 200000:
517
+ return {"url": url, "error": "max_chars must be an integer between 1 and 200000"}
518
+ is_pdf = url.lower().split("?", 1)[0].endswith(".pdf")
478
519
  try:
479
- from newspaper import Article
520
+ from newspaper import Article, network
521
+
522
+ config = network.Configuration()
523
+ config.allow_binary_content = True
524
+ config.requests_params.update(timeout=30, stream=True)
525
+ host = (urlparse(url).hostname or "").lower()
526
+ if host == "sec.gov" or host.endswith(".sec.gov"):
527
+ from sec_edgar import identity
528
+
529
+ config.browser_user_agent = identity() # SEC answers 403 to requests without a declared contact
530
+ response = network.do_request(url, config)
531
+ response.raise_for_status()
532
+ content_type = response.headers.get("content-type", "").lower()
533
+ is_pdf = is_pdf or "application/pdf" in content_type
534
+ if is_pdf:
535
+ return _extract_pdf(url, max_chars, response)
536
+ # allow_binary_content is on for PDFs only; refuse other binaries before reading the whole body.
537
+ if content_type.startswith(("image/", "video/", "audio/", "font/")) or (
538
+ content_type.startswith("application/") and "json" not in content_type and "xml" not in content_type
539
+ ):
540
+ response.close()
541
+ return {"url": url, "error": f"Unsupported binary content-type: {content_type}"}
480
542
 
481
543
  a = Article(url, language="zh")
482
- a.download()
544
+ a.download(input_html=network.get_html(url, config, response=response))
483
545
  a.parse()
546
+ text = a.text or ""
484
547
  return {
485
548
  "url": url,
486
549
  "title": a.title or "",
487
- "text": (a.text or "")[:3000],
550
+ "text": text[:max_chars],
488
551
  "authors": a.authors or [],
489
552
  "publish_date": str(a.publish_date) if a.publish_date else None,
490
553
  "top_image": a.top_image or None,
554
+ "format": "html",
555
+ "truncated": len(text) > max_chars,
491
556
  }
492
- except ImportError:
493
- return {"url": url, "error": "newspaper4k not installed. pip install newspaper4k"}
557
+ except ImportError as exc:
558
+ dependency = "pypdf" if exc.name == "pypdf" else "newspaper4k"
559
+ return {"url": url, "error": f"{dependency} not installed. pip install {dependency}"}
494
560
  except Exception as e:
495
- return {"url": url, "error": str(e)}
561
+ return {"url": url, "error": error_message(e)}
496
562
 
497
563
 
498
564
  # --------------- CLI Commands ---------------
@@ -515,7 +581,7 @@ def cmd_trending(args):
515
581
 
516
582
 
517
583
  def cmd_extract(args):
518
- return extract_article(args.url)
584
+ return extract_article(args.url, max_chars=args.max_chars)
519
585
 
520
586
 
521
587
  def main():
@@ -536,7 +602,8 @@ def main():
536
602
  sub.add_parser("trending")
537
603
 
538
604
  p_ext = sub.add_parser("extract")
539
- p_ext.add_argument("url", help="Article URL to extract")
605
+ p_ext.add_argument("url", help="Article or PDF URL to extract")
606
+ p_ext.add_argument("--max-chars", type=int, default=3000, help="Maximum extracted characters (1-200000)")
540
607
 
541
608
  args = parser.parse_args()
542
609
  if not args.command:
@@ -552,6 +619,8 @@ def main():
552
619
  }
553
620
  result = dispatch[args.command](args)
554
621
  print(json.dumps(result, ensure_ascii=False, default=str))
622
+ if isinstance(result, dict) and "error" in result:
623
+ sys.exit(1)
555
624
 
556
625
 
557
626
  if __name__ == "__main__":