stock-analysis-plugin 0.2.4 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +105 -14
- package/package.json +1 -1
- package/pi/index.ts +290 -14
- package/schemas/report_schema.json +3 -3
- package/scripts/setup-python.mjs +6 -6
- package/skills/bull-trend/SKILL.md +34 -10
- package/skills/bull-trend/references/fat-tails.md +91 -0
- package/skills/bull-trend/scripts/tail_analysis.py +132 -0
- package/skills/event-driven/SKILL.md +8 -0
- package/skills/growth-quality/SKILL.md +8 -0
- package/skills/stock-analysis/SKILL.md +31 -3
- package/skills/stock-screener/SKILL.md +20 -0
- package/skills/strategy-backtest/SKILL.md +6 -0
- package/skills/wisburg-research/SKILL.md +2 -0
- package/templates/stock_analysis/brief.md.j2 +4 -1
- package/templates/stock_analysis/full.md.j2 +6 -3
- package/tools/_cache.py +61 -0
- package/tools/alert_rules.py +32 -4
- package/tools/backtest.py +22 -4
- package/tools/capabilities.py +12 -0
- package/tools/diagnostics.py +6 -0
- package/tools/gather.py +20 -10
- package/tools/lixinger.py +100 -0
- package/tools/market_regime.py +7 -3
- package/tools/market_review.py +7 -6
- package/tools/position_context.py +20 -2
- package/tools/report_renderer.py +2 -0
- package/tools/risk_screening.py +45 -17
- package/tools/screener.py +64 -4
- package/tools/sec_edgar.py +346 -0
- package/tools/sec_holders.py +263 -0
- package/tools/signal_tracker.py +142 -19
- package/tools/stock_data.py +321 -116
- package/tools/technical.py +16 -2
- package/tools/value_data.py +1527 -0
- package/tools/watchlist.py +24 -1
- package/tools/watchlist_context.py +7 -3
package/tools/risk_screening.py
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
import argparse
|
|
5
5
|
import json
|
|
6
|
+
import math
|
|
6
7
|
import os
|
|
7
8
|
import sys
|
|
8
9
|
from concurrent.futures import ThreadPoolExecutor
|
|
@@ -19,7 +20,7 @@ def _safe_float(val) -> float | None:
|
|
|
19
20
|
return None
|
|
20
21
|
try:
|
|
21
22
|
v = float(val)
|
|
22
|
-
return v if v
|
|
23
|
+
return v if math.isfinite(v) else None
|
|
23
24
|
except (ValueError, TypeError):
|
|
24
25
|
return None
|
|
25
26
|
|
|
@@ -29,9 +30,12 @@ def _safe_float(val) -> float | None:
|
|
|
29
30
|
|
|
30
31
|
def check_valuation(symbol: str) -> dict:
|
|
31
32
|
quote = _run_tool("stock_data.py", ["quote", symbol]) or {}
|
|
33
|
+
if not isinstance(quote, dict) or "error" in quote:
|
|
34
|
+
quote = {}
|
|
32
35
|
pe = _safe_float(quote.get("pe"))
|
|
33
36
|
pb = _safe_float(quote.get("pb"))
|
|
34
|
-
|
|
37
|
+
available = pe is not None or pb is not None
|
|
38
|
+
result = {"pe": pe, "pb": pb, "status": "normal" if available else "unknown", "flags": [], "available": available}
|
|
35
39
|
|
|
36
40
|
if pe is not None:
|
|
37
41
|
if pe < 0:
|
|
@@ -91,14 +95,15 @@ def check_valuation(symbol: str) -> dict:
|
|
|
91
95
|
|
|
92
96
|
def check_technical(symbol: str) -> dict:
|
|
93
97
|
ta = _run_tool("technical.py", ["analyze", symbol])
|
|
94
|
-
result = {"flags": []}
|
|
98
|
+
result = {"flags": [], "available": False}
|
|
95
99
|
|
|
96
|
-
if not ta or "error" in ta:
|
|
100
|
+
if not isinstance(ta, dict) or not ta or "error" in ta:
|
|
97
101
|
return result
|
|
98
102
|
|
|
99
103
|
signal = ta.get("buy_signal", "")
|
|
100
104
|
score = _safe_float(ta.get("signal_score"))
|
|
101
|
-
|
|
105
|
+
result["available"] = score is not None
|
|
106
|
+
trend = (ta.get("trend") or {}).get("overall", "")
|
|
102
107
|
|
|
103
108
|
if signal in ("STRONG_SELL",):
|
|
104
109
|
result["flags"].append(
|
|
@@ -130,7 +135,7 @@ def check_technical(symbol: str) -> dict:
|
|
|
130
135
|
)
|
|
131
136
|
|
|
132
137
|
macd = ta.get("macd", {})
|
|
133
|
-
if macd.get("
|
|
138
|
+
if macd.get("cross") == "death_cross":
|
|
134
139
|
result["flags"].append(
|
|
135
140
|
{"category": "technical", "severity": "medium", "description": "MACD死叉", "source": "data"}
|
|
136
141
|
)
|
|
@@ -145,15 +150,27 @@ def check_technical(symbol: str) -> dict:
|
|
|
145
150
|
|
|
146
151
|
|
|
147
152
|
def check_lockup(symbol: str) -> dict:
|
|
148
|
-
result =
|
|
153
|
+
result = _run_tool("risk_screening.py", ["lockup", symbol], timeout=25)
|
|
154
|
+
return (
|
|
155
|
+
result
|
|
156
|
+
if isinstance(result, dict) and "error" not in result
|
|
157
|
+
else {"upcoming_30d": [], "flags": [], "available": False}
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _fetch_lockup(symbol: str) -> dict:
|
|
162
|
+
result = {"upcoming_30d": [], "flags": [], "available": False}
|
|
149
163
|
try:
|
|
150
164
|
import akshare as ak
|
|
151
165
|
|
|
152
166
|
df = ak.stock_restricted_release_queue_em(symbol=symbol)
|
|
153
|
-
if df is None
|
|
167
|
+
if df is None:
|
|
168
|
+
return result
|
|
169
|
+
result["available"] = True
|
|
170
|
+
if df.empty:
|
|
154
171
|
return result
|
|
155
172
|
|
|
156
|
-
now = datetime.now()
|
|
173
|
+
now = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
|
|
157
174
|
cutoff = now + timedelta(days=30)
|
|
158
175
|
|
|
159
176
|
date_col = None
|
|
@@ -180,7 +197,7 @@ def check_lockup(symbol: str) -> dict:
|
|
|
180
197
|
}
|
|
181
198
|
)
|
|
182
199
|
except Exception:
|
|
183
|
-
|
|
200
|
+
result["available"] = False
|
|
184
201
|
return result
|
|
185
202
|
|
|
186
203
|
|
|
@@ -195,7 +212,7 @@ def check_news_risks(symbol: str, name: str = "") -> dict:
|
|
|
195
212
|
"regulatory": f"{search_name} 处罚 违规 监管 立案",
|
|
196
213
|
"industry": f"{search_name} 行业政策 监管 限制",
|
|
197
214
|
}
|
|
198
|
-
result = {"flags": []}
|
|
215
|
+
result = {"flags": [], "available": True}
|
|
199
216
|
# four independent search_intel subprocesses — fan out concurrently (same
|
|
200
217
|
# ThreadPoolExecutor pattern as gather._run_all); iterating `futures` in
|
|
201
218
|
# declaration order keeps result keys and flags stably ordered
|
|
@@ -209,8 +226,10 @@ def check_news_risks(symbol: str, name: str = "") -> dict:
|
|
|
209
226
|
items = []
|
|
210
227
|
if isinstance(data, list):
|
|
211
228
|
items = data
|
|
212
|
-
elif isinstance(data, dict) and "
|
|
229
|
+
elif isinstance(data, dict) and "error" not in data and isinstance(data.get("results"), list):
|
|
213
230
|
items = data["results"]
|
|
231
|
+
else:
|
|
232
|
+
result["available"] = False
|
|
214
233
|
|
|
215
234
|
result[cat] = items
|
|
216
235
|
if items:
|
|
@@ -265,7 +284,7 @@ def screen_risk(symbol: str, name: str = "") -> dict:
|
|
|
265
284
|
checks = {}
|
|
266
285
|
|
|
267
286
|
val = check_valuation(symbol)
|
|
268
|
-
checks["valuation"] = {
|
|
287
|
+
checks["valuation"] = {k: val[k] for k in ("pe", "pb", "status", "available")}
|
|
269
288
|
all_flags.extend(val["flags"])
|
|
270
289
|
|
|
271
290
|
tech = check_technical(symbol)
|
|
@@ -273,11 +292,12 @@ def screen_risk(symbol: str, name: str = "") -> dict:
|
|
|
273
292
|
"buy_signal": tech.get("buy_signal"),
|
|
274
293
|
"signal_score": tech.get("signal_score"),
|
|
275
294
|
"trend": tech.get("trend"),
|
|
295
|
+
"available": tech["available"],
|
|
276
296
|
}
|
|
277
297
|
all_flags.extend(tech["flags"])
|
|
278
298
|
|
|
279
299
|
lockup = check_lockup(symbol)
|
|
280
|
-
checks["lockup"] = {"upcoming_30d": lockup["upcoming_30d"]}
|
|
300
|
+
checks["lockup"] = {"upcoming_30d": lockup["upcoming_30d"], "available": lockup["available"]}
|
|
281
301
|
all_flags.extend(lockup["flags"])
|
|
282
302
|
|
|
283
303
|
news = check_news_risks(symbol, name)
|
|
@@ -285,12 +305,16 @@ def screen_risk(symbol: str, name: str = "") -> dict:
|
|
|
285
305
|
all_flags.extend(news["flags"])
|
|
286
306
|
|
|
287
307
|
risk = compute_risk(all_flags)
|
|
308
|
+
missing = [key for key, value in checks.items() if not value.get("available")]
|
|
309
|
+
complete = not missing
|
|
288
310
|
return {
|
|
289
311
|
"symbol": symbol,
|
|
290
312
|
"name": name,
|
|
291
|
-
"risk_level": risk["risk_level"],
|
|
292
|
-
"risk_score": risk["risk_score"],
|
|
293
|
-
"veto_buy": risk["veto_buy"],
|
|
313
|
+
"risk_level": "unknown" if missing and risk["risk_level"] == "low" else risk["risk_level"],
|
|
314
|
+
"risk_score": risk["risk_score"] if all_flags or complete else None,
|
|
315
|
+
"veto_buy": risk["veto_buy"] if risk["veto_buy"] or complete else None,
|
|
316
|
+
"data_complete": complete,
|
|
317
|
+
"warnings": [f"{key} risk data unavailable or incomplete" for key in missing],
|
|
294
318
|
"flags": all_flags,
|
|
295
319
|
"checks": checks,
|
|
296
320
|
}
|
|
@@ -303,6 +327,8 @@ def main():
|
|
|
303
327
|
p_screen = sub.add_parser("screen")
|
|
304
328
|
p_screen.add_argument("symbol")
|
|
305
329
|
p_screen.add_argument("--name", default="")
|
|
330
|
+
p_lockup = sub.add_parser("lockup", help="Internal bounded lockup query")
|
|
331
|
+
p_lockup.add_argument("symbol")
|
|
306
332
|
|
|
307
333
|
args = parser.parse_args()
|
|
308
334
|
if not args.command:
|
|
@@ -312,6 +338,8 @@ def main():
|
|
|
312
338
|
if args.command == "screen":
|
|
313
339
|
result = screen_risk(args.symbol, args.name)
|
|
314
340
|
print(json.dumps(result, ensure_ascii=False, default=str))
|
|
341
|
+
elif args.command == "lockup":
|
|
342
|
+
print(json.dumps(_fetch_lockup(args.symbol), ensure_ascii=False, default=str))
|
|
315
343
|
|
|
316
344
|
|
|
317
345
|
if __name__ == "__main__":
|
package/tools/screener.py
CHANGED
|
@@ -73,6 +73,13 @@ def merge_filters(config: dict | None) -> dict:
|
|
|
73
73
|
|
|
74
74
|
def apply_filters(df: pd.DataFrame, filters: dict) -> pd.DataFrame:
|
|
75
75
|
conditions = []
|
|
76
|
+
for bound, op in [("dividend_yield_min", "min"), ("dividend_yield_max", "max")]:
|
|
77
|
+
if bound in filters:
|
|
78
|
+
if "dividend_yield" not in df.columns:
|
|
79
|
+
return df.iloc[0:0].copy()
|
|
80
|
+
conditions.append(
|
|
81
|
+
df["dividend_yield"] >= filters[bound] if op == "min" else df["dividend_yield"] <= filters[bound]
|
|
82
|
+
)
|
|
76
83
|
if "pe" in df.columns:
|
|
77
84
|
conditions.append(df["pe"] >= filters["pe_min"])
|
|
78
85
|
conditions.append(df["pe"] <= filters["pe_max"])
|
|
@@ -121,7 +128,10 @@ def compute_scores(df: pd.DataFrame) -> pd.DataFrame:
|
|
|
121
128
|
value_parts.append(100 - _minmax(df["pe"]))
|
|
122
129
|
if "pb" in df.columns:
|
|
123
130
|
value_parts.append(100 - _minmax(df["pb"]))
|
|
124
|
-
|
|
131
|
+
if "dividend_yield" in df.columns:
|
|
132
|
+
df["dividend_score"] = _minmax(df["dividend_yield"])
|
|
133
|
+
value_parts.append(df["dividend_score"])
|
|
134
|
+
df["value_score"] = pd.concat(value_parts, axis=1).mean(axis=1) if value_parts else 0.0
|
|
125
135
|
|
|
126
136
|
# Momentum: higher change_pct and volume_ratio → higher score
|
|
127
137
|
mom_parts = []
|
|
@@ -304,6 +314,21 @@ def _apply_l2(candidates: list, market: str, multiplier: float) -> None:
|
|
|
304
314
|
|
|
305
315
|
|
|
306
316
|
def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict:
|
|
317
|
+
if config is not None and (not isinstance(config, dict) or not isinstance(config.get("filters", {}), dict)):
|
|
318
|
+
return {"error": "config and filters must be objects", "market": market}
|
|
319
|
+
filters = merge_filters(config)
|
|
320
|
+
supported = set(DEFAULT_FILTERS) | {"dividend_yield_min", "dividend_yield_max"}
|
|
321
|
+
for key, value in filters.items():
|
|
322
|
+
if key not in supported:
|
|
323
|
+
return {"error": f"Unsupported filter: {key}", "market": market}
|
|
324
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not np.isfinite(value):
|
|
325
|
+
return {"error": f"{key} must be a finite number", "market": market}
|
|
326
|
+
dividend_requested = any(k.startswith("dividend_yield_") for k in filters) or (config or {}).get("sort_by") in (
|
|
327
|
+
"dividend_yield",
|
|
328
|
+
"dividend_score",
|
|
329
|
+
)
|
|
330
|
+
if dividend_requested and market not in ("A", "HK"):
|
|
331
|
+
return {"error": "dividend screening currently covers A/HK only", "market": market}
|
|
307
332
|
raw = fetch_snapshot(market)
|
|
308
333
|
if isinstance(raw, dict) and "error" in raw:
|
|
309
334
|
return raw
|
|
@@ -330,7 +355,35 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
|
|
|
330
355
|
if existing:
|
|
331
356
|
df = df.dropna(subset=existing)
|
|
332
357
|
|
|
333
|
-
|
|
358
|
+
dividend_meta = {}
|
|
359
|
+
if dividend_requested:
|
|
360
|
+
try:
|
|
361
|
+
from value_data import dividend_ranking
|
|
362
|
+
|
|
363
|
+
ranked = dividend_ranking(report_date=(config or {}).get("dividend_report_date", ""), market=market)
|
|
364
|
+
|
|
365
|
+
def dividend_key(symbol):
|
|
366
|
+
return symbol.split(".")[0].lstrip("0") if market == "HK" else symbol
|
|
367
|
+
|
|
368
|
+
yields = {dividend_key(r["symbol"]): r["dividend_yield"] for r in ranked["items"]}
|
|
369
|
+
df["dividend_yield"] = df["symbol"].map(dividend_key).map(yields)
|
|
370
|
+
dividend_meta = {
|
|
371
|
+
"dividend_factor": {
|
|
372
|
+
"report_date": ranked["report_date"],
|
|
373
|
+
"yield_basis": ranked.get("yield_basis", "provider report-period cash dividend yield, percent"),
|
|
374
|
+
"missing": "unknown; excluded by dividend filters",
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
except Exception:
|
|
378
|
+
return {
|
|
379
|
+
"error": "dividend data unavailable; requested dividend criterion was not applied",
|
|
380
|
+
"market": market,
|
|
381
|
+
}
|
|
382
|
+
explicit = (config or {}).get("filters") or {}
|
|
383
|
+
missing = [key for key in explicit if key.rsplit("_", 1)[0] not in df]
|
|
384
|
+
if missing:
|
|
385
|
+
return {"error": f"Requested filter data unavailable: {', '.join(missing)}", "market": market}
|
|
386
|
+
applied_filters = {key: value for key, value in filters.items() if key.rsplit("_", 1)[0] in df}
|
|
334
387
|
df = apply_filters(df, filters)
|
|
335
388
|
filtered_count = len(df)
|
|
336
389
|
|
|
@@ -340,10 +393,11 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
|
|
|
340
393
|
"total_stocks": total,
|
|
341
394
|
"filtered_count": 0,
|
|
342
395
|
"returned_count": 0,
|
|
343
|
-
"filters_applied":
|
|
396
|
+
"filters_applied": applied_filters,
|
|
344
397
|
"candidates": [],
|
|
345
398
|
"market_sentiment": sentiment,
|
|
346
399
|
"l2_enabled": l2_enabled,
|
|
400
|
+
**dividend_meta,
|
|
347
401
|
}
|
|
348
402
|
|
|
349
403
|
df = compute_scores(df)
|
|
@@ -374,6 +428,7 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
|
|
|
374
428
|
"pb",
|
|
375
429
|
"market_cap",
|
|
376
430
|
"volume_ratio",
|
|
431
|
+
"dividend_yield",
|
|
377
432
|
]:
|
|
378
433
|
v = _finite(row.get(field))
|
|
379
434
|
if v is not None:
|
|
@@ -386,6 +441,8 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
|
|
|
386
441
|
"composite": None if composite is None else round(composite, 1),
|
|
387
442
|
"final": None if composite is None else round(composite * multiplier, 1),
|
|
388
443
|
}
|
|
444
|
+
if "dividend_score" in row:
|
|
445
|
+
entry["scores"]["dividend"] = _finite(round(float(row["dividend_score"]), 1))
|
|
389
446
|
candidates.append(entry)
|
|
390
447
|
|
|
391
448
|
if l2_enabled:
|
|
@@ -399,10 +456,11 @@ def screen(market: str, top: int, config: dict | None, l2: bool = False) -> dict
|
|
|
399
456
|
"total_stocks": total,
|
|
400
457
|
"filtered_count": filtered_count,
|
|
401
458
|
"returned_count": len(candidates),
|
|
402
|
-
"filters_applied":
|
|
459
|
+
"filters_applied": applied_filters,
|
|
403
460
|
"candidates": candidates,
|
|
404
461
|
"market_sentiment": sentiment,
|
|
405
462
|
"l2_enabled": l2_enabled,
|
|
463
|
+
**dividend_meta,
|
|
406
464
|
}
|
|
407
465
|
|
|
408
466
|
|
|
@@ -426,6 +484,8 @@ def main():
|
|
|
426
484
|
config = load_config(args.config) if args.config else None
|
|
427
485
|
result = screen(args.market, args.top, config, l2=args.l2)
|
|
428
486
|
print(json.dumps(result, ensure_ascii=False, default=str))
|
|
487
|
+
if "error" in result:
|
|
488
|
+
sys.exit(1)
|
|
429
489
|
|
|
430
490
|
|
|
431
491
|
if __name__ == "__main__":
|
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""SEC institutional 13F and issuer Form 4 disclosures, never realtime holdings."""
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import math
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import sys
|
|
10
|
+
import time
|
|
11
|
+
from datetime import date, timedelta
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from xml.etree import ElementTree as ET
|
|
14
|
+
|
|
15
|
+
import requests
|
|
16
|
+
|
|
17
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
18
|
+
from _cache import DAY, WEEK, cached
|
|
19
|
+
from _subproc import utf8_stdio
|
|
20
|
+
|
|
21
|
+
_last_request = 0.0
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def identity():
|
|
25
|
+
agent = os.environ.get("SEC_USER_AGENT", "").strip()
|
|
26
|
+
if "\n" in agent or "\r" in agent or not re.search(r"[^\s@]+@[^\s@]+\.[^\s@]+", agent):
|
|
27
|
+
raise ValueError("Set SEC_USER_AGENT to your application name and real contact email")
|
|
28
|
+
return agent
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def sec_get(url, as_json=True, stream=False):
|
|
32
|
+
global _last_request
|
|
33
|
+
agent = identity()
|
|
34
|
+
time.sleep(max(0, 0.5 - (time.monotonic() - _last_request)))
|
|
35
|
+
_last_request = time.monotonic()
|
|
36
|
+
try:
|
|
37
|
+
response = requests.get(
|
|
38
|
+
url,
|
|
39
|
+
headers={"User-Agent": agent, "Accept-Encoding": "gzip, deflate"},
|
|
40
|
+
timeout=(10, 30) if stream else 20,
|
|
41
|
+
stream=stream,
|
|
42
|
+
)
|
|
43
|
+
response.raise_for_status()
|
|
44
|
+
if stream:
|
|
45
|
+
return response
|
|
46
|
+
return response.json() if as_json else response.text
|
|
47
|
+
except Exception as exc:
|
|
48
|
+
raise ValueError(
|
|
49
|
+
f"SEC data unavailable ({type(exc).__name__}); check identity/access and retry later"
|
|
50
|
+
) from None
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def resolve_cik(symbol):
|
|
54
|
+
if re.fullmatch(r"\d{1,10}", symbol) and int(symbol) > 0:
|
|
55
|
+
return symbol.zfill(10)
|
|
56
|
+
if not re.fullmatch(r"[A-Za-z][A-Za-z0-9.-]{0,14}", symbol):
|
|
57
|
+
raise ValueError("A US ticker or numeric CIK is required")
|
|
58
|
+
mapping = cached("sec-tickers:US", WEEK, lambda: sec_get("https://www.sec.gov/files/company_tickers.json"))
|
|
59
|
+
for row in mapping.values():
|
|
60
|
+
if row["ticker"].upper().replace(".", "-") == symbol.upper().replace(".", "-"):
|
|
61
|
+
return str(row["cik_str"]).zfill(10)
|
|
62
|
+
raise ValueError("Ticker not found in SEC directory; supply issuer CIK")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def filing_rows(data):
|
|
66
|
+
return [
|
|
67
|
+
{key: values[i] for key, values in data.items() if isinstance(values, list) and i < len(values)}
|
|
68
|
+
for i in range(len(data.get("accessionNumber", [])))
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def submissions(cik, forms, cutoff=None):
|
|
73
|
+
data = cached(f"sec-submissions:{cik}", DAY, lambda: sec_get(f"https://data.sec.gov/submissions/CIK{cik}.json"))
|
|
74
|
+
if not isinstance(data.get("filings", {}).get("recent"), dict):
|
|
75
|
+
raise ValueError("SEC submissions metadata unavailable")
|
|
76
|
+
rows = filing_rows(data.get("filings", {}).get("recent", {}))
|
|
77
|
+
files = data.get("filings", {}).get("files", [])
|
|
78
|
+
relevant = [f for f in files if not cutoff or f.get("filingTo", "") >= cutoff]
|
|
79
|
+
relevant.sort(key=lambda f: f.get("filingTo", ""), reverse=True)
|
|
80
|
+
loaded = 0
|
|
81
|
+
# shortcut: at most three older submission shards; expose incomplete coverage when more are needed.
|
|
82
|
+
for file in relevant[:3]:
|
|
83
|
+
latest = max((r.get("reportDate") or "" for r in rows if r.get("form") in forms), default="")
|
|
84
|
+
if (
|
|
85
|
+
not cutoff
|
|
86
|
+
and file.get("filingTo", "") <= latest
|
|
87
|
+
and any(r.get("form") == "13F-HR" and (r.get("reportDate") or "") == latest for r in rows)
|
|
88
|
+
):
|
|
89
|
+
break
|
|
90
|
+
name = file.get("name", "")
|
|
91
|
+
if not re.fullmatch(r"CIK\d{10}-submissions-\d+\.json", name):
|
|
92
|
+
raise ValueError("Unexpected SEC submissions filename")
|
|
93
|
+
extra = cached(
|
|
94
|
+
f"sec-submission-file:{name}", DAY, lambda name=name: sec_get(f"https://data.sec.gov/submissions/{name}")
|
|
95
|
+
)
|
|
96
|
+
rows.extend(filing_rows(extra))
|
|
97
|
+
loaded += 1
|
|
98
|
+
result = [r for r in rows if r.get("form") in forms and (not cutoff or r.get("filingDate", "") >= cutoff)]
|
|
99
|
+
result.sort(key=lambda r: (r.get("filingDate", ""), r.get("accessionNumber", "")), reverse=True)
|
|
100
|
+
earliest_unsearched = cutoff or max((r.get("reportDate") or "" for r in result), default="")
|
|
101
|
+
incomplete = any(f.get("filingTo", "") >= earliest_unsearched for f in relevant[loaded:])
|
|
102
|
+
return data.get("name"), result, incomplete
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def archive_base(cik, filing):
|
|
106
|
+
accession = filing.get("accessionNumber", "")
|
|
107
|
+
if not re.fullmatch(r"\d{10}-\d{2}-\d{6}", accession):
|
|
108
|
+
raise ValueError("Invalid SEC accession number")
|
|
109
|
+
return f"https://www.sec.gov/Archives/edgar/data/{int(cik)}/{accession.replace('-', '')}"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def xml_root(text):
|
|
113
|
+
if "<!DOCTYPE" in text.upper() or "<!ENTITY" in text.upper():
|
|
114
|
+
raise ValueError("Unexpected XML entity declaration")
|
|
115
|
+
root = ET.fromstring(text)
|
|
116
|
+
for element in root.iter():
|
|
117
|
+
element.tag = element.tag.rsplit("}", 1)[-1]
|
|
118
|
+
return root
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def footnote_refs(node, path=""):
|
|
122
|
+
refs = {}
|
|
123
|
+
for child in node:
|
|
124
|
+
child_path = f"{path}/{child.tag}" if path else child.tag
|
|
125
|
+
if child.tag == "footnoteId":
|
|
126
|
+
refs.setdefault(path, []).append(child.get("id"))
|
|
127
|
+
else:
|
|
128
|
+
refs.update(footnote_refs(child, child_path))
|
|
129
|
+
return refs
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def numeric(node, path):
|
|
133
|
+
value = node.findtext(path)
|
|
134
|
+
try:
|
|
135
|
+
result = float(value)
|
|
136
|
+
return result if math.isfinite(result) else None
|
|
137
|
+
except (ValueError, TypeError):
|
|
138
|
+
return None
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def holdings(args):
|
|
142
|
+
identity()
|
|
143
|
+
if not re.fullmatch(r"\d{1,10}", args.cik) or int(args.cik) == 0:
|
|
144
|
+
raise ValueError("13F requires a manager or issuer CIK (1..10 digits)")
|
|
145
|
+
cik = args.cik.zfill(10)
|
|
146
|
+
mode = getattr(args, "mode", "auto")
|
|
147
|
+
if mode not in ("auto", "manager", "issuer"):
|
|
148
|
+
raise ValueError("mode must be auto, manager or issuer")
|
|
149
|
+
if mode != "manager":
|
|
150
|
+
data = cached(f"sec-submissions:{cik}", DAY, lambda: sec_get(f"https://data.sec.gov/submissions/CIK{cik}.json"))
|
|
151
|
+
if not isinstance(data.get("filings", {}).get("recent"), dict):
|
|
152
|
+
raise ValueError("SEC submissions metadata unavailable")
|
|
153
|
+
forms = data["filings"]["recent"].get("form", [])
|
|
154
|
+
is_manager = any(form in ("13F-HR", "13F-HR/A") for form in forms)
|
|
155
|
+
if mode == "auto" and not is_manager:
|
|
156
|
+
_, archived, _ = submissions(cik, {"13F-HR", "13F-HR/A"})
|
|
157
|
+
is_manager = any(row["form"] == "13F-HR" for row in archived)
|
|
158
|
+
if mode == "issuer" or not is_manager:
|
|
159
|
+
from sec_holders import issuer_holders
|
|
160
|
+
|
|
161
|
+
return issuer_holders(cik, data)
|
|
162
|
+
name, filings, incomplete = submissions(cik, {"13F-HR", "13F-HR/A"})
|
|
163
|
+
originals = [r for r in filings if r["form"] == "13F-HR"]
|
|
164
|
+
if not originals:
|
|
165
|
+
raise ValueError("No original 13F-HR in available submission history; issuer CIK is not a holder index")
|
|
166
|
+
filing = max(originals, key=lambda r: (r.get("reportDate", ""), r.get("filingDate", "")))
|
|
167
|
+
if max((r.get("reportDate") or "" for r in filings), default="") > (filing.get("reportDate") or ""):
|
|
168
|
+
raise ValueError("Latest 13F report period has amendments but its original is unavailable in searched history")
|
|
169
|
+
amendments = [r for r in filings if r["form"] == "13F-HR/A" and r.get("reportDate") == filing.get("reportDate")]
|
|
170
|
+
base = archive_base(cik, filing)
|
|
171
|
+
|
|
172
|
+
def fetch_table():
|
|
173
|
+
listing = sec_get(f"{base}/index.json")["directory"]["item"]
|
|
174
|
+
candidates = [r["name"] for r in listing if re.fullmatch(r"[\w.-]+\.xml", r.get("name", ""), re.I)]
|
|
175
|
+
for filename in candidates[:10]:
|
|
176
|
+
try:
|
|
177
|
+
root = xml_root(sec_get(f"{base}/{filename}", as_json=False))
|
|
178
|
+
except ET.ParseError:
|
|
179
|
+
continue
|
|
180
|
+
if root.tag == "informationTable":
|
|
181
|
+
return [
|
|
182
|
+
{
|
|
183
|
+
"issuer": row.findtext("nameOfIssuer"),
|
|
184
|
+
"class": row.findtext("titleOfClass"),
|
|
185
|
+
"cusip": row.findtext("cusip"),
|
|
186
|
+
"value": numeric(row, "value"),
|
|
187
|
+
"shares": numeric(row, "shrsOrPrnAmt/sshPrnamt"),
|
|
188
|
+
"share_type": row.findtext("shrsOrPrnAmt/sshPrnamtType"),
|
|
189
|
+
"put_call": row.findtext("putCall"),
|
|
190
|
+
"investment_discretion": row.findtext("investmentDiscretion"),
|
|
191
|
+
}
|
|
192
|
+
for row in root.findall("infoTable")
|
|
193
|
+
]
|
|
194
|
+
raise ValueError("13F information table XML unavailable")
|
|
195
|
+
|
|
196
|
+
rows = cached(f"sec-13f:{cik}:{filing['accessionNumber']}", WEEK, fetch_table)
|
|
197
|
+
result = {
|
|
198
|
+
"cik": cik,
|
|
199
|
+
"manager": name,
|
|
200
|
+
"query_type": "manager",
|
|
201
|
+
"source": "sec_edgar",
|
|
202
|
+
"report_date": filing.get("reportDate"),
|
|
203
|
+
"filing_date": filing["filingDate"],
|
|
204
|
+
"accession": filing["accessionNumber"],
|
|
205
|
+
"filing_url": f"{base}/{filing['accessionNumber']}-index.html",
|
|
206
|
+
"holdings": rows,
|
|
207
|
+
"value_unit": "USD" if filing["filingDate"] >= "2023-01-03" else "USD thousands",
|
|
208
|
+
"disclosure_lag_days": (
|
|
209
|
+
date.fromisoformat(filing["filingDate"]) - date.fromisoformat(filing["reportDate"])
|
|
210
|
+
).days,
|
|
211
|
+
"filing_deadline_days": 45,
|
|
212
|
+
"data_age_days": (date.today() - date.fromisoformat(filing["reportDate"])).days,
|
|
213
|
+
"realtime": False,
|
|
214
|
+
"history_truncated": incomplete,
|
|
215
|
+
"coverage": "13F reportable securities only; not a complete portfolio or issuer-to-managers index",
|
|
216
|
+
"amendments": [
|
|
217
|
+
{
|
|
218
|
+
"accession": r["accessionNumber"],
|
|
219
|
+
"filing_date": r["filingDate"],
|
|
220
|
+
"filing_url": f"{archive_base(cik, r)}/{r['accessionNumber']}-index.html",
|
|
221
|
+
}
|
|
222
|
+
for r in amendments
|
|
223
|
+
],
|
|
224
|
+
}
|
|
225
|
+
if incomplete:
|
|
226
|
+
result["warnings"] = ["Older submission history not fully searched; latest report period may be incomplete"]
|
|
227
|
+
if amendments:
|
|
228
|
+
result.setdefault("warnings", []).append(
|
|
229
|
+
"Amendments exist; original holdings returned, review amendment links before comparison"
|
|
230
|
+
)
|
|
231
|
+
return result
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def form4(cik, filing):
|
|
235
|
+
base = archive_base(cik, filing)
|
|
236
|
+
document = filing.get("primaryDocument", "").rsplit("/", 1)[-1]
|
|
237
|
+
if not re.fullmatch(r"[\w.-]+\.xml", document, re.I):
|
|
238
|
+
raise ValueError("Form 4 primary XML unavailable")
|
|
239
|
+
root = xml_root(sec_get(f"{base}/{document}", as_json=False))
|
|
240
|
+
if root.tag != "ownershipDocument" or int(root.findtext("issuer/issuerCik") or 0) != int(cik):
|
|
241
|
+
raise ValueError("Form 4 issuer mismatch")
|
|
242
|
+
owners = [
|
|
243
|
+
{
|
|
244
|
+
"name": r.findtext("reportingOwnerId/rptOwnerName"),
|
|
245
|
+
"cik": r.findtext("reportingOwnerId/rptOwnerCik"),
|
|
246
|
+
"officer_title": r.findtext("reportingOwnerRelationship/officerTitle"),
|
|
247
|
+
"is_director": r.findtext("reportingOwnerRelationship/isDirector") in ("1", "true"),
|
|
248
|
+
"is_officer": r.findtext("reportingOwnerRelationship/isOfficer") in ("1", "true"),
|
|
249
|
+
"is_ten_percent_owner": r.findtext("reportingOwnerRelationship/isTenPercentOwner") in ("1", "true"),
|
|
250
|
+
}
|
|
251
|
+
for r in root.findall("reportingOwner")
|
|
252
|
+
]
|
|
253
|
+
footnotes = {r.get("id"): "".join(r.itertext()) for r in root.findall("footnotes/footnote")}
|
|
254
|
+
rows = []
|
|
255
|
+
for table, tag, derivative in [
|
|
256
|
+
("nonDerivativeTable", "nonDerivativeTransaction", False),
|
|
257
|
+
("derivativeTable", "derivativeTransaction", True),
|
|
258
|
+
]:
|
|
259
|
+
for row in root.findall(f"{table}/{tag}"):
|
|
260
|
+
code = row.findtext("transactionCoding/transactionCode")
|
|
261
|
+
rows.append(
|
|
262
|
+
{
|
|
263
|
+
"transaction_date": row.findtext("transactionDate/value"),
|
|
264
|
+
"owners": owners,
|
|
265
|
+
"security": row.findtext("securityTitle/value"),
|
|
266
|
+
"derivative": derivative,
|
|
267
|
+
"transaction_code": code,
|
|
268
|
+
"direction": {"P": "buy", "S": "sell"}.get(code, "other"),
|
|
269
|
+
"acquired_disposed": row.findtext("transactionAmounts/transactionAcquiredDisposedCode/value"),
|
|
270
|
+
"shares": numeric(row, "transactionAmounts/transactionShares/value"),
|
|
271
|
+
"price": numeric(row, "transactionAmounts/transactionPricePerShare/value"),
|
|
272
|
+
"shares_after": numeric(row, "postTransactionAmounts/sharesOwnedFollowingTransaction/value"),
|
|
273
|
+
"ownership": row.findtext("ownershipNature/directOrIndirectOwnership/value"),
|
|
274
|
+
"footnotes": footnotes,
|
|
275
|
+
"footnote_refs": footnote_refs(row),
|
|
276
|
+
"filing_date": filing["filingDate"],
|
|
277
|
+
"form": filing["form"],
|
|
278
|
+
"accession": filing["accessionNumber"],
|
|
279
|
+
"filing_url": f"{base}/{document}",
|
|
280
|
+
}
|
|
281
|
+
)
|
|
282
|
+
return rows
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def insider_trades(args):
|
|
286
|
+
identity()
|
|
287
|
+
if not 1 <= args.days <= 365 or not 1 <= args.top <= 20:
|
|
288
|
+
raise ValueError("days must be in 1..365 and top (filings) in 1..20")
|
|
289
|
+
cik = resolve_cik(args.symbol)
|
|
290
|
+
cutoff = (date.today() - timedelta(days=args.days)).isoformat()
|
|
291
|
+
_, filings, incomplete = submissions(cik, {"4", "4/A"}, cutoff)
|
|
292
|
+
result = {
|
|
293
|
+
"symbol": args.symbol,
|
|
294
|
+
"cik": cik,
|
|
295
|
+
"source": "sec_edgar",
|
|
296
|
+
"since_filing_date": cutoff,
|
|
297
|
+
"transactions": [],
|
|
298
|
+
"filings_scanned": 0,
|
|
299
|
+
"truncated": incomplete or len(filings) > args.top,
|
|
300
|
+
"coverage": "issuer Form 4 filings in filing-date window; top limits filings, not transactions",
|
|
301
|
+
}
|
|
302
|
+
for filing in filings[: args.top]:
|
|
303
|
+
try:
|
|
304
|
+
rows = cached(
|
|
305
|
+
f"sec-form4:{cik}:{filing['accessionNumber']}", WEEK, lambda filing=filing: form4(cik, filing)
|
|
306
|
+
)
|
|
307
|
+
result["transactions"].extend(rows)
|
|
308
|
+
result["filings_scanned"] += 1
|
|
309
|
+
except Exception as exc:
|
|
310
|
+
result.setdefault("warnings", []).append(f"{filing['accessionNumber']}: unavailable ({type(exc).__name__})")
|
|
311
|
+
if filings and not result["filings_scanned"]:
|
|
312
|
+
raise ValueError("All selected Form 4 XML documents unavailable")
|
|
313
|
+
if any(r["form"] == "4/A" for r in result["transactions"]):
|
|
314
|
+
result.setdefault("warnings", []).append(
|
|
315
|
+
"Includes Form 4 amendments; do not double-count original and amended transactions"
|
|
316
|
+
)
|
|
317
|
+
return result
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def main():
|
|
321
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
322
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
323
|
+
p = sub.add_parser("holdings")
|
|
324
|
+
p.add_argument("cik")
|
|
325
|
+
p.add_argument("--mode", choices=["auto", "manager", "issuer"], default="auto")
|
|
326
|
+
p = sub.add_parser("insider_trades")
|
|
327
|
+
p.add_argument("symbol")
|
|
328
|
+
p.add_argument("--days", type=int, default=90)
|
|
329
|
+
p.add_argument("--top", type=int, default=10)
|
|
330
|
+
try:
|
|
331
|
+
args = parser.parse_args()
|
|
332
|
+
result = holdings(args) if args.command == "holdings" else insider_trades(args)
|
|
333
|
+
print(json.dumps(result, ensure_ascii=False, allow_nan=False))
|
|
334
|
+
except SystemExit as exc:
|
|
335
|
+
if exc.code:
|
|
336
|
+
print(json.dumps({"error": "invalid CLI arguments"}))
|
|
337
|
+
raise
|
|
338
|
+
except Exception as exc:
|
|
339
|
+
message = str(exc) if isinstance(exc, ValueError) else f"SEC data unavailable ({type(exc).__name__})"
|
|
340
|
+
print(json.dumps({"error": message}, ensure_ascii=False))
|
|
341
|
+
sys.exit(1)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
if __name__ == "__main__":
|
|
345
|
+
utf8_stdio()
|
|
346
|
+
main()
|