stock-analysis-plugin 0.2.11 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +234 -382
- package/docs/mcp.md +60 -0
- package/hermes/__init__.py +35 -0
- package/hermes/plugin.yaml +132 -0
- package/hermes/schemas.py +6 -0
- package/hermes/tools.py +647 -0
- package/mcp_host/__init__.py +1 -0
- package/mcp_host/__main__.py +3 -0
- package/mcp_host/server.py +222 -0
- package/package.json +7 -1
- package/pi/index.ts +52 -2906
- package/scripts/cli-json.mjs +23 -0
- package/scripts/exec-process.mjs +89 -0
- package/scripts/setup-python.mjs +37 -4
- package/scripts/tool-calls.mjs +493 -0
- package/skills/bull-trend/SKILL.md +2 -2
- package/skills/bull-trend/references/fat-tails.md +76 -3
- package/skills/bull-trend/scripts/tail_analysis.py +92 -3
- package/skills/event-driven/SKILL.md +2 -2
- package/skills/growth-quality/SKILL.md +1 -1
- package/skills/market-review/SKILL.md +1 -0
- package/skills/stock-analysis/SKILL.md +12 -7
- package/skills/stock-analysis/references/stateless-analysis.md +9 -1
- package/skills/stock-screener/SKILL.md +1 -1
- package/skills/strategy-backtest/SKILL.md +1 -1
- package/tools/_cache.py +13 -0
- package/tools/_subproc.py +46 -0
- package/tools/alert_rules.py +2 -2
- package/tools/backtest.py +3 -3
- package/tools/capabilities.py +33 -9
- package/tools/cli_budgets.json +1 -0
- package/tools/credit_data.py +293 -0
- package/tools/diagnostics.py +3 -4
- package/tools/gather.py +2 -6
- package/tools/import_parser.py +3 -4
- package/tools/lixinger.py +119 -27
- package/tools/market_regime.py +5 -2
- package/tools/market_review.py +5 -6
- package/tools/name_resolver.py +45 -33
- package/tools/report_renderer.py +3 -3
- package/tools/requirements.txt +5 -4
- package/tools/risk_screening.py +3 -2
- package/tools/screener.py +41 -30
- package/tools/search_intel.py +79 -10
- package/tools/sec_edgar.py +268 -1
- package/tools/sec_holders.py +6 -0
- package/tools/signal_tracker.py +5 -5
- package/tools/stateless_analysis.py +132 -19
- package/tools/stock_data.py +1020 -817
- package/tools/tiaa_disclosures.py +4 -0
- package/tools/tool_schemas.json +2734 -0
- package/tools/trading_calendar.py +65 -30
- package/tools/value_data.py +423 -175
- package/tools/watchlist.py +43 -5
package/tools/market_regime.py
CHANGED
|
@@ -18,6 +18,9 @@ INDEX_MAP = {
|
|
|
18
18
|
"A": ("000001", "上证综指"),
|
|
19
19
|
"HK": ("^HSI", "恒生指数"),
|
|
20
20
|
"US": ("^GSPC", "标普500"),
|
|
21
|
+
"JP": ("^N225", "日经225"),
|
|
22
|
+
"KR": ("^KS11", "韩国KOSPI"),
|
|
23
|
+
"TW": ("^TWII", "台湾加权指数"),
|
|
21
24
|
}
|
|
22
25
|
|
|
23
26
|
REGIME_CN = {
|
|
@@ -166,7 +169,7 @@ def _detect_sector_heat(market: str) -> bool:
|
|
|
166
169
|
def detect_regime(market: str = "A") -> dict:
|
|
167
170
|
market = market.upper()
|
|
168
171
|
if market not in INDEX_MAP:
|
|
169
|
-
return {"error": f"Unsupported market: {market}. Use
|
|
172
|
+
return {"error": f"Unsupported market: {market}. Use {'/'.join(INDEX_MAP)}"}
|
|
170
173
|
|
|
171
174
|
code, name = INDEX_MAP[market]
|
|
172
175
|
klines = _get_index_kline(market)
|
|
@@ -229,7 +232,7 @@ def main():
|
|
|
229
232
|
sub = parser.add_subparsers(dest="command")
|
|
230
233
|
|
|
231
234
|
p_detect = sub.add_parser("detect")
|
|
232
|
-
p_detect.add_argument("market", nargs="?", default="A", choices=
|
|
235
|
+
p_detect.add_argument("market", nargs="?", default="A", choices=list(INDEX_MAP))
|
|
233
236
|
|
|
234
237
|
args = parser.parse_args()
|
|
235
238
|
if not args.command:
|
package/tools/market_review.py
CHANGED
|
@@ -18,9 +18,10 @@ from pathlib import Path
|
|
|
18
18
|
|
|
19
19
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
20
20
|
|
|
21
|
-
from _subproc import command_timeout, utf8_stdio
|
|
21
|
+
from _subproc import command_timeout, error_message, utf8_stdio
|
|
22
22
|
from _subproc import run_tool as _run_tool
|
|
23
23
|
from signal_tracker import _append_record, _load_records, _store_lock
|
|
24
|
+
from trading_calendar import market_now
|
|
24
25
|
|
|
25
26
|
|
|
26
27
|
def _index_change(row):
|
|
@@ -81,8 +82,6 @@ def _signal_to_stance(signal: str) -> str | None:
|
|
|
81
82
|
|
|
82
83
|
def review_market(market: str = "A", *, run=None) -> dict:
|
|
83
84
|
"""Gather market data and compute review for a single market."""
|
|
84
|
-
from datetime import datetime
|
|
85
|
-
|
|
86
85
|
tasks = {}
|
|
87
86
|
|
|
88
87
|
if market == "A":
|
|
@@ -135,7 +134,7 @@ def review_market(market: str = "A", *, run=None) -> dict:
|
|
|
135
134
|
|
|
136
135
|
output = {
|
|
137
136
|
"market": market,
|
|
138
|
-
"date":
|
|
137
|
+
"date": market_now("CN" if market == "A" else market).strftime("%Y-%m-%d"),
|
|
139
138
|
"indices": results.get("indices"),
|
|
140
139
|
"stats": results.get("stats"),
|
|
141
140
|
"sectors": results.get("sectors"),
|
|
@@ -169,7 +168,7 @@ def save_review(report: dict, store: Path | None = None) -> dict:
|
|
|
169
168
|
with _store_lock(store):
|
|
170
169
|
_append_record(store, report)
|
|
171
170
|
except (OSError, ValueError) as e:
|
|
172
|
-
return {"error":
|
|
171
|
+
return {"error": error_message(e)}
|
|
173
172
|
return {"saved": True, "date": report.get("review_date") or report.get("date")}
|
|
174
173
|
|
|
175
174
|
|
|
@@ -181,7 +180,7 @@ def cmd_history(args) -> dict:
|
|
|
181
180
|
records = _load_records(store)
|
|
182
181
|
except (OSError, UnicodeDecodeError) as e:
|
|
183
182
|
# store is a directory / not UTF-8 — same clean-error contract as save_review
|
|
184
|
-
return {"error":
|
|
183
|
+
return {"error": error_message(e)}
|
|
185
184
|
limit = max(args.limit, 0)
|
|
186
185
|
recent = records[-limit:][::-1] if limit else []
|
|
187
186
|
return {"count": len(recent), "records": recent}
|
package/tools/name_resolver.py
CHANGED
|
@@ -3,40 +3,40 @@
|
|
|
3
3
|
|
|
4
4
|
import argparse
|
|
5
5
|
import json
|
|
6
|
+
import re
|
|
6
7
|
import sys
|
|
7
8
|
from difflib import SequenceMatcher
|
|
8
9
|
from pathlib import Path
|
|
9
10
|
|
|
10
11
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
11
12
|
|
|
12
|
-
from
|
|
13
|
-
from
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
)
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
raise RuntimeError(f"Failed to load stock list: {e}") from e
|
|
13
|
+
from _subproc import error_message, utf8_stdio # noqa: E402
|
|
14
|
+
from stock_data import _em_codetable, normalize_us_symbol # noqa: E402
|
|
15
|
+
|
|
16
|
+
_A_TYPES = {"沪A", "深A", "京A", "科创板", "创业板"}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _search(query: str, count: int) -> list[dict]:
|
|
20
|
+
"""Eastmoney code-table search (code / name / pinyin / initials), kept to A/HK/US listed stocks."""
|
|
21
|
+
stocks = []
|
|
22
|
+
for row in _em_codetable(query, count):
|
|
23
|
+
code, name, market, kind = (
|
|
24
|
+
str(row.get("code", "")),
|
|
25
|
+
row.get("shortName"),
|
|
26
|
+
row.get("market"),
|
|
27
|
+
row.get("securityTypeName"),
|
|
28
|
+
)
|
|
29
|
+
if not isinstance(name, str) or not name.strip():
|
|
30
|
+
continue
|
|
31
|
+
if market in (0, 1) and kind in _A_TYPES and re.fullmatch(r"\d{6}", code):
|
|
32
|
+
stocks.append({"code": code, "name": name})
|
|
33
|
+
elif market == 116 and kind == "港股" and re.fullmatch(r"\d{5}", code):
|
|
34
|
+
stocks.append({"code": f"{code}.HK", "name": name})
|
|
35
|
+
elif market in (105, 106, 107) and kind == "美股":
|
|
36
|
+
symbol = normalize_us_symbol(code)
|
|
37
|
+
if re.fullmatch(r"[A-Z0-9-]{1,10}", symbol):
|
|
38
|
+
stocks.append({"code": symbol, "name": name})
|
|
39
|
+
return stocks
|
|
40
40
|
|
|
41
41
|
|
|
42
42
|
def _to_pinyin(text: str) -> str:
|
|
@@ -57,16 +57,16 @@ def _to_pinyin_initials(text: str) -> str:
|
|
|
57
57
|
return ""
|
|
58
58
|
|
|
59
59
|
|
|
60
|
-
def
|
|
61
|
-
stocks = _load_stock_map()
|
|
60
|
+
def _resolve_stocks(query: str, top: int, stocks: list[dict]) -> list[dict]:
|
|
62
61
|
query = query.strip()
|
|
63
62
|
q_lower = query.lower()
|
|
63
|
+
q_code = query.upper()
|
|
64
64
|
|
|
65
|
-
exact = [s for s in stocks if s["code"] ==
|
|
65
|
+
exact = [s for s in stocks if s["code"].upper() == q_code or s["name"] == query]
|
|
66
66
|
if exact:
|
|
67
67
|
return exact[:top]
|
|
68
68
|
|
|
69
|
-
prefix_code = [s for s in stocks if s["code"].startswith(
|
|
69
|
+
prefix_code = [s for s in stocks if s["code"].upper().startswith(q_code)]
|
|
70
70
|
if prefix_code:
|
|
71
71
|
return prefix_code[:top]
|
|
72
72
|
|
|
@@ -104,6 +104,18 @@ def resolve(query: str, top: int = 5) -> list[dict]:
|
|
|
104
104
|
return [{"code": s["code"], "name": s["name"], "score": round(sc, 2)} for sc, s in scored[:top]]
|
|
105
105
|
|
|
106
106
|
|
|
107
|
+
def resolve(query: str, top: int = 5) -> list[dict]:
|
|
108
|
+
query = query.strip()
|
|
109
|
+
if not query:
|
|
110
|
+
return []
|
|
111
|
+
try:
|
|
112
|
+
# The provider ranks loosely; over-fetch so the local exact/prefix/contains/pinyin order decides.
|
|
113
|
+
stocks = _search(query, max(top, 20))
|
|
114
|
+
except Exception as e:
|
|
115
|
+
raise RuntimeError(f"Failed to search stock names: {error_message(e)}") from e
|
|
116
|
+
return _resolve_stocks(query, top, stocks)
|
|
117
|
+
|
|
118
|
+
|
|
107
119
|
def main():
|
|
108
120
|
parser = argparse.ArgumentParser(description="Stock name resolver")
|
|
109
121
|
sub = parser.add_subparsers(dest="command")
|
|
@@ -120,7 +132,7 @@ def main():
|
|
|
120
132
|
try:
|
|
121
133
|
result = resolve(args.query, args.top)
|
|
122
134
|
except RuntimeError as e:
|
|
123
|
-
result = {"error":
|
|
135
|
+
result = {"error": error_message(e)}
|
|
124
136
|
print(json.dumps(result, ensure_ascii=False))
|
|
125
137
|
|
|
126
138
|
|
package/tools/report_renderer.py
CHANGED
|
@@ -12,7 +12,7 @@ from datetime import datetime, timezone
|
|
|
12
12
|
from pathlib import Path
|
|
13
13
|
|
|
14
14
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
15
|
-
from _subproc import utf8_stdio
|
|
15
|
+
from _subproc import error_message, utf8_stdio
|
|
16
16
|
from market_review import save_review
|
|
17
17
|
|
|
18
18
|
try:
|
|
@@ -55,12 +55,12 @@ def render(kind: str, template: str, report: dict) -> dict:
|
|
|
55
55
|
try:
|
|
56
56
|
tpl = _env().get_template(tpl_path)
|
|
57
57
|
except Exception as e:
|
|
58
|
-
return {"error": f"failed to load template {tpl_path}: {e}"}
|
|
58
|
+
return {"error": f"failed to load template {tpl_path}: {error_message(e)}"}
|
|
59
59
|
|
|
60
60
|
try:
|
|
61
61
|
content = tpl.render(**report)
|
|
62
62
|
except Exception as e:
|
|
63
|
-
return {"error": f"render failed: {e}", "template": tpl_path}
|
|
63
|
+
return {"error": f"render failed: {error_message(e)}", "template": tpl_path}
|
|
64
64
|
|
|
65
65
|
return {
|
|
66
66
|
"meta": {
|
package/tools/requirements.txt
CHANGED
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
akshare>=1.20.24; python_version >= "3.11"
|
|
2
2
|
akshare>=1.14.0; python_version < "3.11"
|
|
3
3
|
yfinance>=0.2.36
|
|
4
|
-
pandas>=2.0.0
|
|
4
|
+
pandas>=2.0.0,<4
|
|
5
5
|
numpy>=1.24.0
|
|
6
6
|
pyyaml>=6.0
|
|
7
7
|
pypinyin>=0.50.0
|
|
8
|
+
pypdf>=5.0.0
|
|
8
9
|
requests>=2.33.0
|
|
9
10
|
exchange-calendars>=4.5.0
|
|
10
|
-
|
|
11
|
+
jinja2>=3.1.6
|
|
12
|
+
# optional providers: failover-only sources; setup-python.mjs only warns when one cannot install
|
|
11
13
|
efinance>=0.5.0
|
|
12
14
|
baostock>=0.8.8
|
|
13
|
-
pytdx>=1.72
|
|
14
15
|
longport>=1.0.0
|
|
15
16
|
finnhub-python>=2.4.0
|
|
16
|
-
|
|
17
|
+
newspaper4k>=0.9.6.1
|
package/tools/risk_screening.py
CHANGED
|
@@ -7,12 +7,13 @@ import math
|
|
|
7
7
|
import os
|
|
8
8
|
import sys
|
|
9
9
|
from concurrent.futures import Future, ThreadPoolExecutor
|
|
10
|
-
from datetime import
|
|
10
|
+
from datetime import timedelta
|
|
11
11
|
|
|
12
12
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
13
13
|
|
|
14
14
|
from _subproc import run_tool as _run_tool
|
|
15
15
|
from _subproc import utf8_stdio
|
|
16
|
+
from trading_calendar import market_now
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
def _safe_float(val) -> float | None:
|
|
@@ -181,7 +182,7 @@ def _fetch_lockup(symbol: str) -> dict:
|
|
|
181
182
|
if df.empty:
|
|
182
183
|
return result
|
|
183
184
|
|
|
184
|
-
now =
|
|
185
|
+
now = market_now("CN").replace(hour=0, minute=0, second=0, microsecond=0)
|
|
185
186
|
cutoff = now + timedelta(days=30)
|
|
186
187
|
|
|
187
188
|
date_col = None
|
package/tools/screener.py
CHANGED
|
@@ -21,7 +21,6 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
|
21
21
|
|
|
22
22
|
from _subproc import command_timeout, run_tool, utf8_stdio
|
|
23
23
|
from market_review import calc_temperature
|
|
24
|
-
from stock_data import compute_market_stats
|
|
25
24
|
|
|
26
25
|
DEFAULT_FILTERS = {
|
|
27
26
|
"pe_min": 0,
|
|
@@ -56,12 +55,10 @@ def _fetch_json(args: list[str], timeout: int | None = None) -> dict | list | No
|
|
|
56
55
|
return run_tool("stock_data.py", args, timeout=timeout, preserve_errors=True)
|
|
57
56
|
|
|
58
57
|
|
|
59
|
-
def fetch_snapshot(market: str) ->
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
return _fetch_json(
|
|
63
|
-
["market_snapshot", "--market", market], timeout=command_timeout("stock_data.py", ["market_snapshot"])
|
|
64
|
-
)
|
|
58
|
+
def fetch_snapshot(market: str, filters: dict) -> dict:
|
|
59
|
+
"""Stocks at or above filters["market_cap_min"], largest first (A also filters server-side)."""
|
|
60
|
+
args = ["market_snapshot", "--market", market, "--filters", json.dumps(filters)]
|
|
61
|
+
return _fetch_json(args, timeout=command_timeout("stock_data.py", ["market_snapshot"]))
|
|
65
62
|
|
|
66
63
|
|
|
67
64
|
def load_config(path: str) -> dict:
|
|
@@ -148,12 +145,13 @@ def compute_scores(df: pd.DataFrame) -> pd.DataFrame:
|
|
|
148
145
|
mom_parts.append(_minmax(df["volume_ratio"]))
|
|
149
146
|
df["momentum_score"] = pd.concat(mom_parts, axis=1).mean(axis=1) if mom_parts else np.nan
|
|
150
147
|
|
|
151
|
-
# Liquidity: higher turnover_rate and
|
|
148
|
+
# Liquidity: higher turnover_rate and money traded → higher score. Share volume would
|
|
149
|
+
# favour low-priced stocks, so the cross-stock leg ranks turnover (成交额).
|
|
152
150
|
liq_parts = []
|
|
153
151
|
if "turnover_rate" in df.columns:
|
|
154
152
|
liq_parts.append(_minmax(df["turnover_rate"]))
|
|
155
|
-
if "
|
|
156
|
-
liq_parts.append(_minmax(df["
|
|
153
|
+
if "turnover" in df.columns:
|
|
154
|
+
liq_parts.append(_minmax(df["turnover"].astype(float)))
|
|
157
155
|
df["liquidity_score"] = pd.concat(liq_parts, axis=1).mean(axis=1) if liq_parts else np.nan
|
|
158
156
|
|
|
159
157
|
groups = df[[f"{group}_score" for group in L1_WEIGHTS]].rename(columns=lambda name: name.removesuffix("_score"))
|
|
@@ -166,19 +164,18 @@ def sentiment_multiplier(score: float) -> float:
|
|
|
166
164
|
return round(0.8 + score / 100 * 0.4, 3) # linear map to [0.8, 1.2]
|
|
167
165
|
|
|
168
166
|
|
|
169
|
-
def compute_sentiment(market: str
|
|
167
|
+
def compute_sentiment(market: str) -> dict:
|
|
170
168
|
"""Market sentiment gate; degrades to a neutral multiplier of 1.0 on any failure."""
|
|
171
169
|
fallback = {"score": None, "level": "unknown", "signal": None, "multiplier": 1.0}
|
|
172
170
|
region = _INDEX_REGION.get(market)
|
|
173
171
|
if region is None:
|
|
174
172
|
return fallback
|
|
175
173
|
try:
|
|
176
|
-
stats =
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
stats = None
|
|
174
|
+
stats = _fetch_json(
|
|
175
|
+
["market_stats", "--market", market], timeout=command_timeout("stock_data.py", ["market_stats"])
|
|
176
|
+
)
|
|
177
|
+
if not isinstance(stats, dict) or "error" in stats:
|
|
178
|
+
stats = None
|
|
182
179
|
indices = _fetch_json(["market_indices", "--region", region])
|
|
183
180
|
if not isinstance(indices, list) or not indices:
|
|
184
181
|
return fallback
|
|
@@ -371,22 +368,29 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
|
|
|
371
368
|
)
|
|
372
369
|
if dividend_requested and market not in ("A", "HK"):
|
|
373
370
|
return {"error": "dividend screening currently covers A/HK only", "market": market}
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
if isinstance(
|
|
380
|
-
return
|
|
381
|
-
if
|
|
371
|
+
# Sentiment is an independent market_stats call; overlap it with the snapshot fetch.
|
|
372
|
+
pool = ThreadPoolExecutor(max_workers=1)
|
|
373
|
+
sentiment_future = pool.submit(compute_sentiment, market)
|
|
374
|
+
pool.shutdown(wait=False)
|
|
375
|
+
snapshot = fetch_snapshot(market, filters)
|
|
376
|
+
if isinstance(snapshot, dict) and "error" in snapshot:
|
|
377
|
+
return snapshot
|
|
378
|
+
if not isinstance(snapshot, dict) or not isinstance(snapshot.get("rows"), list):
|
|
382
379
|
return {"error": "failed to fetch market snapshot", "market": market}
|
|
383
380
|
|
|
384
381
|
l2_enabled = bool(l2 or (config or {}).get("l2"))
|
|
385
|
-
sentiment =
|
|
382
|
+
sentiment = sentiment_future.result()
|
|
386
383
|
multiplier = sentiment["multiplier"]
|
|
387
384
|
|
|
388
|
-
|
|
389
|
-
|
|
385
|
+
# An empty server-filtered page still reports every requested filter as applied.
|
|
386
|
+
columns = ["symbol", *{key.rsplit("_", 1)[0] for key in filters}]
|
|
387
|
+
df = pd.DataFrame(snapshot["rows"]) if snapshot["rows"] else pd.DataFrame(columns=columns)
|
|
388
|
+
total = snapshot.get("total_stocks")
|
|
389
|
+
scan_meta = {}
|
|
390
|
+
if snapshot.get("truncated"):
|
|
391
|
+
scan_meta["warnings"] = [
|
|
392
|
+
f"scanned only the {len(df)} largest stocks by market cap; raise market_cap_min to cover the rest"
|
|
393
|
+
]
|
|
390
394
|
|
|
391
395
|
for c in ["price", "change_pct", "volume", "turnover", "pe", "pb", "market_cap", "turnover_rate", "volume_ratio"]:
|
|
392
396
|
if c in df.columns:
|
|
@@ -402,7 +406,11 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
|
|
|
402
406
|
try:
|
|
403
407
|
from value_data import dividend_ranking
|
|
404
408
|
|
|
405
|
-
ranked = dividend_ranking(
|
|
409
|
+
ranked = dividend_ranking(
|
|
410
|
+
report_date=(config or {}).get("dividend_report_date", ""),
|
|
411
|
+
market=market,
|
|
412
|
+
codes=list(df["symbol"]) if market == "A" else None,
|
|
413
|
+
)
|
|
406
414
|
|
|
407
415
|
def dividend_key(symbol):
|
|
408
416
|
return symbol.split(".")[0].lstrip("0") if market == "HK" else symbol
|
|
@@ -440,6 +448,7 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
|
|
|
440
448
|
"market_sentiment": sentiment,
|
|
441
449
|
"l2_enabled": l2_enabled,
|
|
442
450
|
**dividend_meta,
|
|
451
|
+
**scan_meta,
|
|
443
452
|
}
|
|
444
453
|
|
|
445
454
|
df = compute_scores(df)
|
|
@@ -465,6 +474,7 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
|
|
|
465
474
|
"price",
|
|
466
475
|
"change_pct",
|
|
467
476
|
"volume",
|
|
477
|
+
"turnover",
|
|
468
478
|
"turnover_rate",
|
|
469
479
|
"pe",
|
|
470
480
|
"pe_basis",
|
|
@@ -489,7 +499,7 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
|
|
|
489
499
|
factor_inputs = {
|
|
490
500
|
"value": ("pe", "pb", "dividend_yield"),
|
|
491
501
|
"momentum": ("change_pct", "volume_ratio"),
|
|
492
|
-
"liquidity": ("turnover_rate", "
|
|
502
|
+
"liquidity": ("turnover_rate", "turnover"),
|
|
493
503
|
}
|
|
494
504
|
weights = L1_WEIGHTS
|
|
495
505
|
available = [
|
|
@@ -529,6 +539,7 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
|
|
|
529
539
|
"market_sentiment": sentiment,
|
|
530
540
|
"l2_enabled": l2_enabled,
|
|
531
541
|
**dividend_meta,
|
|
542
|
+
**scan_meta,
|
|
532
543
|
}
|
|
533
544
|
|
|
534
545
|
|
package/tools/search_intel.py
CHANGED
|
@@ -9,9 +9,10 @@ import re
|
|
|
9
9
|
import sys
|
|
10
10
|
import time as _time
|
|
11
11
|
from contextvars import ContextVar
|
|
12
|
+
from urllib.parse import urlparse
|
|
12
13
|
|
|
13
14
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
14
|
-
from _subproc import scrub_secrets, utf8_stdio
|
|
15
|
+
from _subproc import error_message, scrub_secrets, utf8_stdio
|
|
15
16
|
|
|
16
17
|
_SEARCH_STATE = ContextVar("search_budget", default=None)
|
|
17
18
|
|
|
@@ -474,25 +475,90 @@ def get_trending_sentiment() -> dict:
|
|
|
474
475
|
# --------------- Article Extraction ---------------
|
|
475
476
|
|
|
476
477
|
|
|
477
|
-
def
|
|
478
|
+
def _extract_pdf(url: str, max_chars: int, response) -> dict:
|
|
479
|
+
import io
|
|
480
|
+
|
|
481
|
+
content = bytearray()
|
|
482
|
+
try:
|
|
483
|
+
from pypdf import PdfReader
|
|
484
|
+
|
|
485
|
+
for chunk in response.iter_content(65536):
|
|
486
|
+
content.extend(chunk)
|
|
487
|
+
if len(content) > 25 * 1024 * 1024:
|
|
488
|
+
raise ValueError("PDF exceeds the 25 MiB download limit")
|
|
489
|
+
finally:
|
|
490
|
+
response.close()
|
|
491
|
+
if not content.startswith(b"%PDF-"):
|
|
492
|
+
raise ValueError("URL did not return a PDF document")
|
|
493
|
+
|
|
494
|
+
reader = PdfReader(io.BytesIO(content))
|
|
495
|
+
parts = []
|
|
496
|
+
for page in reader.pages:
|
|
497
|
+
parts.append(page.extract_text() or "")
|
|
498
|
+
if sum(map(len, parts)) > max_chars:
|
|
499
|
+
break
|
|
500
|
+
text = "\n".join(parts)
|
|
501
|
+
metadata = reader.metadata or {}
|
|
502
|
+
return {
|
|
503
|
+
"url": url,
|
|
504
|
+
"title": str(metadata.get("/Title") or ""),
|
|
505
|
+
"text": text[:max_chars],
|
|
506
|
+
"authors": [str(metadata["/Author"])] if metadata.get("/Author") else [],
|
|
507
|
+
"publish_date": str(metadata["/CreationDate"]) if metadata.get("/CreationDate") else None,
|
|
508
|
+
"top_image": None,
|
|
509
|
+
"format": "pdf",
|
|
510
|
+
"page_count": len(reader.pages),
|
|
511
|
+
"truncated": len(text) > max_chars,
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def extract_article(url: str, max_chars: int = 3000) -> dict:
|
|
516
|
+
if isinstance(max_chars, bool) or not isinstance(max_chars, int) or not 1 <= max_chars <= 200000:
|
|
517
|
+
return {"url": url, "error": "max_chars must be an integer between 1 and 200000"}
|
|
518
|
+
is_pdf = url.lower().split("?", 1)[0].endswith(".pdf")
|
|
478
519
|
try:
|
|
479
|
-
from newspaper import Article
|
|
520
|
+
from newspaper import Article, network
|
|
521
|
+
|
|
522
|
+
config = network.Configuration()
|
|
523
|
+
config.allow_binary_content = True
|
|
524
|
+
config.requests_params.update(timeout=30, stream=True)
|
|
525
|
+
host = (urlparse(url).hostname or "").lower()
|
|
526
|
+
if host == "sec.gov" or host.endswith(".sec.gov"):
|
|
527
|
+
from sec_edgar import identity
|
|
528
|
+
|
|
529
|
+
config.browser_user_agent = identity() # SEC answers 403 to requests without a declared contact
|
|
530
|
+
response = network.do_request(url, config)
|
|
531
|
+
response.raise_for_status()
|
|
532
|
+
content_type = response.headers.get("content-type", "").lower()
|
|
533
|
+
is_pdf = is_pdf or "application/pdf" in content_type
|
|
534
|
+
if is_pdf:
|
|
535
|
+
return _extract_pdf(url, max_chars, response)
|
|
536
|
+
# allow_binary_content is on for PDFs only; refuse other binaries before reading the whole body.
|
|
537
|
+
if content_type.startswith(("image/", "video/", "audio/", "font/")) or (
|
|
538
|
+
content_type.startswith("application/") and "json" not in content_type and "xml" not in content_type
|
|
539
|
+
):
|
|
540
|
+
response.close()
|
|
541
|
+
return {"url": url, "error": f"Unsupported binary content-type: {content_type}"}
|
|
480
542
|
|
|
481
543
|
a = Article(url, language="zh")
|
|
482
|
-
a.download()
|
|
544
|
+
a.download(input_html=network.get_html(url, config, response=response))
|
|
483
545
|
a.parse()
|
|
546
|
+
text = a.text or ""
|
|
484
547
|
return {
|
|
485
548
|
"url": url,
|
|
486
549
|
"title": a.title or "",
|
|
487
|
-
"text":
|
|
550
|
+
"text": text[:max_chars],
|
|
488
551
|
"authors": a.authors or [],
|
|
489
552
|
"publish_date": str(a.publish_date) if a.publish_date else None,
|
|
490
553
|
"top_image": a.top_image or None,
|
|
554
|
+
"format": "html",
|
|
555
|
+
"truncated": len(text) > max_chars,
|
|
491
556
|
}
|
|
492
|
-
except ImportError:
|
|
493
|
-
|
|
557
|
+
except ImportError as exc:
|
|
558
|
+
dependency = "pypdf" if exc.name == "pypdf" else "newspaper4k"
|
|
559
|
+
return {"url": url, "error": f"{dependency} not installed. pip install {dependency}"}
|
|
494
560
|
except Exception as e:
|
|
495
|
-
return {"url": url, "error":
|
|
561
|
+
return {"url": url, "error": error_message(e)}
|
|
496
562
|
|
|
497
563
|
|
|
498
564
|
# --------------- CLI Commands ---------------
|
|
@@ -515,7 +581,7 @@ def cmd_trending(args):
|
|
|
515
581
|
|
|
516
582
|
|
|
517
583
|
def cmd_extract(args):
|
|
518
|
-
return extract_article(args.url)
|
|
584
|
+
return extract_article(args.url, max_chars=args.max_chars)
|
|
519
585
|
|
|
520
586
|
|
|
521
587
|
def main():
|
|
@@ -536,7 +602,8 @@ def main():
|
|
|
536
602
|
sub.add_parser("trending")
|
|
537
603
|
|
|
538
604
|
p_ext = sub.add_parser("extract")
|
|
539
|
-
p_ext.add_argument("url", help="Article URL to extract")
|
|
605
|
+
p_ext.add_argument("url", help="Article or PDF URL to extract")
|
|
606
|
+
p_ext.add_argument("--max-chars", type=int, default=3000, help="Maximum extracted characters (1-200000)")
|
|
540
607
|
|
|
541
608
|
args = parser.parse_args()
|
|
542
609
|
if not args.command:
|
|
@@ -552,6 +619,8 @@ def main():
|
|
|
552
619
|
}
|
|
553
620
|
result = dispatch[args.command](args)
|
|
554
621
|
print(json.dumps(result, ensure_ascii=False, default=str))
|
|
622
|
+
if isinstance(result, dict) and "error" in result:
|
|
623
|
+
sys.exit(1)
|
|
555
624
|
|
|
556
625
|
|
|
557
626
|
if __name__ == "__main__":
|