stock-analysis-plugin 0.2.5 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -10
- package/package.json +1 -1
- package/pi/index.ts +107 -7
- package/schemas/report_schema.json +3 -3
- package/scripts/setup-python.mjs +6 -6
- package/skills/bull-trend/SKILL.md +34 -10
- package/skills/bull-trend/references/fat-tails.md +91 -0
- package/skills/bull-trend/scripts/tail_analysis.py +132 -0
- package/skills/event-driven/SKILL.md +2 -0
- package/skills/growth-quality/SKILL.md +2 -0
- package/skills/stock-analysis/SKILL.md +9 -3
- package/skills/stock-screener/SKILL.md +2 -0
- package/skills/strategy-backtest/SKILL.md +4 -0
- package/skills/wisburg-research/SKILL.md +2 -0
- package/templates/stock_analysis/brief.md.j2 +4 -1
- package/templates/stock_analysis/full.md.j2 +6 -3
- package/tools/alert_rules.py +32 -4
- package/tools/backtest.py +22 -4
- package/tools/capabilities.py +3 -0
- package/tools/gather.py +20 -10
- package/tools/lixinger.py +100 -0
- package/tools/market_regime.py +7 -3
- package/tools/market_review.py +7 -6
- package/tools/position_context.py +20 -2
- package/tools/report_renderer.py +2 -0
- package/tools/risk_screening.py +45 -17
- package/tools/screener.py +17 -9
- package/tools/sec_edgar.py +346 -0
- package/tools/sec_holders.py +263 -0
- package/tools/signal_tracker.py +142 -19
- package/tools/stock_data.py +80 -17
- package/tools/technical.py +16 -2
- package/tools/value_data.py +159 -0
- package/tools/watchlist.py +24 -1
- package/tools/watchlist_context.py +7 -3
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""SEC institutional 13F and issuer Form 4 disclosures, never realtime holdings."""
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import math
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import sys
|
|
10
|
+
import time
|
|
11
|
+
from datetime import date, timedelta
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from xml.etree import ElementTree as ET
|
|
14
|
+
|
|
15
|
+
import requests
|
|
16
|
+
|
|
17
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
18
|
+
from _cache import DAY, WEEK, cached
|
|
19
|
+
from _subproc import utf8_stdio
|
|
20
|
+
|
|
21
|
+
_last_request = 0.0
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def identity():
|
|
25
|
+
agent = os.environ.get("SEC_USER_AGENT", "").strip()
|
|
26
|
+
if "\n" in agent or "\r" in agent or not re.search(r"[^\s@]+@[^\s@]+\.[^\s@]+", agent):
|
|
27
|
+
raise ValueError("Set SEC_USER_AGENT to your application name and real contact email")
|
|
28
|
+
return agent
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def sec_get(url, as_json=True, stream=False):
|
|
32
|
+
global _last_request
|
|
33
|
+
agent = identity()
|
|
34
|
+
time.sleep(max(0, 0.5 - (time.monotonic() - _last_request)))
|
|
35
|
+
_last_request = time.monotonic()
|
|
36
|
+
try:
|
|
37
|
+
response = requests.get(
|
|
38
|
+
url,
|
|
39
|
+
headers={"User-Agent": agent, "Accept-Encoding": "gzip, deflate"},
|
|
40
|
+
timeout=(10, 30) if stream else 20,
|
|
41
|
+
stream=stream,
|
|
42
|
+
)
|
|
43
|
+
response.raise_for_status()
|
|
44
|
+
if stream:
|
|
45
|
+
return response
|
|
46
|
+
return response.json() if as_json else response.text
|
|
47
|
+
except Exception as exc:
|
|
48
|
+
raise ValueError(
|
|
49
|
+
f"SEC data unavailable ({type(exc).__name__}); check identity/access and retry later"
|
|
50
|
+
) from None
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def resolve_cik(symbol):
|
|
54
|
+
if re.fullmatch(r"\d{1,10}", symbol) and int(symbol) > 0:
|
|
55
|
+
return symbol.zfill(10)
|
|
56
|
+
if not re.fullmatch(r"[A-Za-z][A-Za-z0-9.-]{0,14}", symbol):
|
|
57
|
+
raise ValueError("A US ticker or numeric CIK is required")
|
|
58
|
+
mapping = cached("sec-tickers:US", WEEK, lambda: sec_get("https://www.sec.gov/files/company_tickers.json"))
|
|
59
|
+
for row in mapping.values():
|
|
60
|
+
if row["ticker"].upper().replace(".", "-") == symbol.upper().replace(".", "-"):
|
|
61
|
+
return str(row["cik_str"]).zfill(10)
|
|
62
|
+
raise ValueError("Ticker not found in SEC directory; supply issuer CIK")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def filing_rows(data):
|
|
66
|
+
return [
|
|
67
|
+
{key: values[i] for key, values in data.items() if isinstance(values, list) and i < len(values)}
|
|
68
|
+
for i in range(len(data.get("accessionNumber", [])))
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def submissions(cik, forms, cutoff=None):
|
|
73
|
+
data = cached(f"sec-submissions:{cik}", DAY, lambda: sec_get(f"https://data.sec.gov/submissions/CIK{cik}.json"))
|
|
74
|
+
if not isinstance(data.get("filings", {}).get("recent"), dict):
|
|
75
|
+
raise ValueError("SEC submissions metadata unavailable")
|
|
76
|
+
rows = filing_rows(data.get("filings", {}).get("recent", {}))
|
|
77
|
+
files = data.get("filings", {}).get("files", [])
|
|
78
|
+
relevant = [f for f in files if not cutoff or f.get("filingTo", "") >= cutoff]
|
|
79
|
+
relevant.sort(key=lambda f: f.get("filingTo", ""), reverse=True)
|
|
80
|
+
loaded = 0
|
|
81
|
+
# shortcut: at most three older submission shards; expose incomplete coverage when more are needed.
|
|
82
|
+
for file in relevant[:3]:
|
|
83
|
+
latest = max((r.get("reportDate") or "" for r in rows if r.get("form") in forms), default="")
|
|
84
|
+
if (
|
|
85
|
+
not cutoff
|
|
86
|
+
and file.get("filingTo", "") <= latest
|
|
87
|
+
and any(r.get("form") == "13F-HR" and (r.get("reportDate") or "") == latest for r in rows)
|
|
88
|
+
):
|
|
89
|
+
break
|
|
90
|
+
name = file.get("name", "")
|
|
91
|
+
if not re.fullmatch(r"CIK\d{10}-submissions-\d+\.json", name):
|
|
92
|
+
raise ValueError("Unexpected SEC submissions filename")
|
|
93
|
+
extra = cached(
|
|
94
|
+
f"sec-submission-file:{name}", DAY, lambda name=name: sec_get(f"https://data.sec.gov/submissions/{name}")
|
|
95
|
+
)
|
|
96
|
+
rows.extend(filing_rows(extra))
|
|
97
|
+
loaded += 1
|
|
98
|
+
result = [r for r in rows if r.get("form") in forms and (not cutoff or r.get("filingDate", "") >= cutoff)]
|
|
99
|
+
result.sort(key=lambda r: (r.get("filingDate", ""), r.get("accessionNumber", "")), reverse=True)
|
|
100
|
+
earliest_unsearched = cutoff or max((r.get("reportDate") or "" for r in result), default="")
|
|
101
|
+
incomplete = any(f.get("filingTo", "") >= earliest_unsearched for f in relevant[loaded:])
|
|
102
|
+
return data.get("name"), result, incomplete
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def archive_base(cik, filing):
|
|
106
|
+
accession = filing.get("accessionNumber", "")
|
|
107
|
+
if not re.fullmatch(r"\d{10}-\d{2}-\d{6}", accession):
|
|
108
|
+
raise ValueError("Invalid SEC accession number")
|
|
109
|
+
return f"https://www.sec.gov/Archives/edgar/data/{int(cik)}/{accession.replace('-', '')}"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def xml_root(text):
|
|
113
|
+
if "<!DOCTYPE" in text.upper() or "<!ENTITY" in text.upper():
|
|
114
|
+
raise ValueError("Unexpected XML entity declaration")
|
|
115
|
+
root = ET.fromstring(text)
|
|
116
|
+
for element in root.iter():
|
|
117
|
+
element.tag = element.tag.rsplit("}", 1)[-1]
|
|
118
|
+
return root
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def footnote_refs(node, path=""):
|
|
122
|
+
refs = {}
|
|
123
|
+
for child in node:
|
|
124
|
+
child_path = f"{path}/{child.tag}" if path else child.tag
|
|
125
|
+
if child.tag == "footnoteId":
|
|
126
|
+
refs.setdefault(path, []).append(child.get("id"))
|
|
127
|
+
else:
|
|
128
|
+
refs.update(footnote_refs(child, child_path))
|
|
129
|
+
return refs
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def numeric(node, path):
|
|
133
|
+
value = node.findtext(path)
|
|
134
|
+
try:
|
|
135
|
+
result = float(value)
|
|
136
|
+
return result if math.isfinite(result) else None
|
|
137
|
+
except (ValueError, TypeError):
|
|
138
|
+
return None
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def holdings(args):
|
|
142
|
+
identity()
|
|
143
|
+
if not re.fullmatch(r"\d{1,10}", args.cik) or int(args.cik) == 0:
|
|
144
|
+
raise ValueError("13F requires a manager or issuer CIK (1..10 digits)")
|
|
145
|
+
cik = args.cik.zfill(10)
|
|
146
|
+
mode = getattr(args, "mode", "auto")
|
|
147
|
+
if mode not in ("auto", "manager", "issuer"):
|
|
148
|
+
raise ValueError("mode must be auto, manager or issuer")
|
|
149
|
+
if mode != "manager":
|
|
150
|
+
data = cached(f"sec-submissions:{cik}", DAY, lambda: sec_get(f"https://data.sec.gov/submissions/CIK{cik}.json"))
|
|
151
|
+
if not isinstance(data.get("filings", {}).get("recent"), dict):
|
|
152
|
+
raise ValueError("SEC submissions metadata unavailable")
|
|
153
|
+
forms = data["filings"]["recent"].get("form", [])
|
|
154
|
+
is_manager = any(form in ("13F-HR", "13F-HR/A") for form in forms)
|
|
155
|
+
if mode == "auto" and not is_manager:
|
|
156
|
+
_, archived, _ = submissions(cik, {"13F-HR", "13F-HR/A"})
|
|
157
|
+
is_manager = any(row["form"] == "13F-HR" for row in archived)
|
|
158
|
+
if mode == "issuer" or not is_manager:
|
|
159
|
+
from sec_holders import issuer_holders
|
|
160
|
+
|
|
161
|
+
return issuer_holders(cik, data)
|
|
162
|
+
name, filings, incomplete = submissions(cik, {"13F-HR", "13F-HR/A"})
|
|
163
|
+
originals = [r for r in filings if r["form"] == "13F-HR"]
|
|
164
|
+
if not originals:
|
|
165
|
+
raise ValueError("No original 13F-HR in available submission history; issuer CIK is not a holder index")
|
|
166
|
+
filing = max(originals, key=lambda r: (r.get("reportDate", ""), r.get("filingDate", "")))
|
|
167
|
+
if max((r.get("reportDate") or "" for r in filings), default="") > (filing.get("reportDate") or ""):
|
|
168
|
+
raise ValueError("Latest 13F report period has amendments but its original is unavailable in searched history")
|
|
169
|
+
amendments = [r for r in filings if r["form"] == "13F-HR/A" and r.get("reportDate") == filing.get("reportDate")]
|
|
170
|
+
base = archive_base(cik, filing)
|
|
171
|
+
|
|
172
|
+
def fetch_table():
|
|
173
|
+
listing = sec_get(f"{base}/index.json")["directory"]["item"]
|
|
174
|
+
candidates = [r["name"] for r in listing if re.fullmatch(r"[\w.-]+\.xml", r.get("name", ""), re.I)]
|
|
175
|
+
for filename in candidates[:10]:
|
|
176
|
+
try:
|
|
177
|
+
root = xml_root(sec_get(f"{base}/{filename}", as_json=False))
|
|
178
|
+
except ET.ParseError:
|
|
179
|
+
continue
|
|
180
|
+
if root.tag == "informationTable":
|
|
181
|
+
return [
|
|
182
|
+
{
|
|
183
|
+
"issuer": row.findtext("nameOfIssuer"),
|
|
184
|
+
"class": row.findtext("titleOfClass"),
|
|
185
|
+
"cusip": row.findtext("cusip"),
|
|
186
|
+
"value": numeric(row, "value"),
|
|
187
|
+
"shares": numeric(row, "shrsOrPrnAmt/sshPrnamt"),
|
|
188
|
+
"share_type": row.findtext("shrsOrPrnAmt/sshPrnamtType"),
|
|
189
|
+
"put_call": row.findtext("putCall"),
|
|
190
|
+
"investment_discretion": row.findtext("investmentDiscretion"),
|
|
191
|
+
}
|
|
192
|
+
for row in root.findall("infoTable")
|
|
193
|
+
]
|
|
194
|
+
raise ValueError("13F information table XML unavailable")
|
|
195
|
+
|
|
196
|
+
rows = cached(f"sec-13f:{cik}:{filing['accessionNumber']}", WEEK, fetch_table)
|
|
197
|
+
result = {
|
|
198
|
+
"cik": cik,
|
|
199
|
+
"manager": name,
|
|
200
|
+
"query_type": "manager",
|
|
201
|
+
"source": "sec_edgar",
|
|
202
|
+
"report_date": filing.get("reportDate"),
|
|
203
|
+
"filing_date": filing["filingDate"],
|
|
204
|
+
"accession": filing["accessionNumber"],
|
|
205
|
+
"filing_url": f"{base}/{filing['accessionNumber']}-index.html",
|
|
206
|
+
"holdings": rows,
|
|
207
|
+
"value_unit": "USD" if filing["filingDate"] >= "2023-01-03" else "USD thousands",
|
|
208
|
+
"disclosure_lag_days": (
|
|
209
|
+
date.fromisoformat(filing["filingDate"]) - date.fromisoformat(filing["reportDate"])
|
|
210
|
+
).days,
|
|
211
|
+
"filing_deadline_days": 45,
|
|
212
|
+
"data_age_days": (date.today() - date.fromisoformat(filing["reportDate"])).days,
|
|
213
|
+
"realtime": False,
|
|
214
|
+
"history_truncated": incomplete,
|
|
215
|
+
"coverage": "13F reportable securities only; not a complete portfolio or issuer-to-managers index",
|
|
216
|
+
"amendments": [
|
|
217
|
+
{
|
|
218
|
+
"accession": r["accessionNumber"],
|
|
219
|
+
"filing_date": r["filingDate"],
|
|
220
|
+
"filing_url": f"{archive_base(cik, r)}/{r['accessionNumber']}-index.html",
|
|
221
|
+
}
|
|
222
|
+
for r in amendments
|
|
223
|
+
],
|
|
224
|
+
}
|
|
225
|
+
if incomplete:
|
|
226
|
+
result["warnings"] = ["Older submission history not fully searched; latest report period may be incomplete"]
|
|
227
|
+
if amendments:
|
|
228
|
+
result.setdefault("warnings", []).append(
|
|
229
|
+
"Amendments exist; original holdings returned, review amendment links before comparison"
|
|
230
|
+
)
|
|
231
|
+
return result
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def form4(cik, filing):
|
|
235
|
+
base = archive_base(cik, filing)
|
|
236
|
+
document = filing.get("primaryDocument", "").rsplit("/", 1)[-1]
|
|
237
|
+
if not re.fullmatch(r"[\w.-]+\.xml", document, re.I):
|
|
238
|
+
raise ValueError("Form 4 primary XML unavailable")
|
|
239
|
+
root = xml_root(sec_get(f"{base}/{document}", as_json=False))
|
|
240
|
+
if root.tag != "ownershipDocument" or int(root.findtext("issuer/issuerCik") or 0) != int(cik):
|
|
241
|
+
raise ValueError("Form 4 issuer mismatch")
|
|
242
|
+
owners = [
|
|
243
|
+
{
|
|
244
|
+
"name": r.findtext("reportingOwnerId/rptOwnerName"),
|
|
245
|
+
"cik": r.findtext("reportingOwnerId/rptOwnerCik"),
|
|
246
|
+
"officer_title": r.findtext("reportingOwnerRelationship/officerTitle"),
|
|
247
|
+
"is_director": r.findtext("reportingOwnerRelationship/isDirector") in ("1", "true"),
|
|
248
|
+
"is_officer": r.findtext("reportingOwnerRelationship/isOfficer") in ("1", "true"),
|
|
249
|
+
"is_ten_percent_owner": r.findtext("reportingOwnerRelationship/isTenPercentOwner") in ("1", "true"),
|
|
250
|
+
}
|
|
251
|
+
for r in root.findall("reportingOwner")
|
|
252
|
+
]
|
|
253
|
+
footnotes = {r.get("id"): "".join(r.itertext()) for r in root.findall("footnotes/footnote")}
|
|
254
|
+
rows = []
|
|
255
|
+
for table, tag, derivative in [
|
|
256
|
+
("nonDerivativeTable", "nonDerivativeTransaction", False),
|
|
257
|
+
("derivativeTable", "derivativeTransaction", True),
|
|
258
|
+
]:
|
|
259
|
+
for row in root.findall(f"{table}/{tag}"):
|
|
260
|
+
code = row.findtext("transactionCoding/transactionCode")
|
|
261
|
+
rows.append(
|
|
262
|
+
{
|
|
263
|
+
"transaction_date": row.findtext("transactionDate/value"),
|
|
264
|
+
"owners": owners,
|
|
265
|
+
"security": row.findtext("securityTitle/value"),
|
|
266
|
+
"derivative": derivative,
|
|
267
|
+
"transaction_code": code,
|
|
268
|
+
"direction": {"P": "buy", "S": "sell"}.get(code, "other"),
|
|
269
|
+
"acquired_disposed": row.findtext("transactionAmounts/transactionAcquiredDisposedCode/value"),
|
|
270
|
+
"shares": numeric(row, "transactionAmounts/transactionShares/value"),
|
|
271
|
+
"price": numeric(row, "transactionAmounts/transactionPricePerShare/value"),
|
|
272
|
+
"shares_after": numeric(row, "postTransactionAmounts/sharesOwnedFollowingTransaction/value"),
|
|
273
|
+
"ownership": row.findtext("ownershipNature/directOrIndirectOwnership/value"),
|
|
274
|
+
"footnotes": footnotes,
|
|
275
|
+
"footnote_refs": footnote_refs(row),
|
|
276
|
+
"filing_date": filing["filingDate"],
|
|
277
|
+
"form": filing["form"],
|
|
278
|
+
"accession": filing["accessionNumber"],
|
|
279
|
+
"filing_url": f"{base}/{document}",
|
|
280
|
+
}
|
|
281
|
+
)
|
|
282
|
+
return rows
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def insider_trades(args):
|
|
286
|
+
identity()
|
|
287
|
+
if not 1 <= args.days <= 365 or not 1 <= args.top <= 20:
|
|
288
|
+
raise ValueError("days must be in 1..365 and top (filings) in 1..20")
|
|
289
|
+
cik = resolve_cik(args.symbol)
|
|
290
|
+
cutoff = (date.today() - timedelta(days=args.days)).isoformat()
|
|
291
|
+
_, filings, incomplete = submissions(cik, {"4", "4/A"}, cutoff)
|
|
292
|
+
result = {
|
|
293
|
+
"symbol": args.symbol,
|
|
294
|
+
"cik": cik,
|
|
295
|
+
"source": "sec_edgar",
|
|
296
|
+
"since_filing_date": cutoff,
|
|
297
|
+
"transactions": [],
|
|
298
|
+
"filings_scanned": 0,
|
|
299
|
+
"truncated": incomplete or len(filings) > args.top,
|
|
300
|
+
"coverage": "issuer Form 4 filings in filing-date window; top limits filings, not transactions",
|
|
301
|
+
}
|
|
302
|
+
for filing in filings[: args.top]:
|
|
303
|
+
try:
|
|
304
|
+
rows = cached(
|
|
305
|
+
f"sec-form4:{cik}:{filing['accessionNumber']}", WEEK, lambda filing=filing: form4(cik, filing)
|
|
306
|
+
)
|
|
307
|
+
result["transactions"].extend(rows)
|
|
308
|
+
result["filings_scanned"] += 1
|
|
309
|
+
except Exception as exc:
|
|
310
|
+
result.setdefault("warnings", []).append(f"{filing['accessionNumber']}: unavailable ({type(exc).__name__})")
|
|
311
|
+
if filings and not result["filings_scanned"]:
|
|
312
|
+
raise ValueError("All selected Form 4 XML documents unavailable")
|
|
313
|
+
if any(r["form"] == "4/A" for r in result["transactions"]):
|
|
314
|
+
result.setdefault("warnings", []).append(
|
|
315
|
+
"Includes Form 4 amendments; do not double-count original and amended transactions"
|
|
316
|
+
)
|
|
317
|
+
return result
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def main():
|
|
321
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
322
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
323
|
+
p = sub.add_parser("holdings")
|
|
324
|
+
p.add_argument("cik")
|
|
325
|
+
p.add_argument("--mode", choices=["auto", "manager", "issuer"], default="auto")
|
|
326
|
+
p = sub.add_parser("insider_trades")
|
|
327
|
+
p.add_argument("symbol")
|
|
328
|
+
p.add_argument("--days", type=int, default=90)
|
|
329
|
+
p.add_argument("--top", type=int, default=10)
|
|
330
|
+
try:
|
|
331
|
+
args = parser.parse_args()
|
|
332
|
+
result = holdings(args) if args.command == "holdings" else insider_trades(args)
|
|
333
|
+
print(json.dumps(result, ensure_ascii=False, allow_nan=False))
|
|
334
|
+
except SystemExit as exc:
|
|
335
|
+
if exc.code:
|
|
336
|
+
print(json.dumps({"error": "invalid CLI arguments"}))
|
|
337
|
+
raise
|
|
338
|
+
except Exception as exc:
|
|
339
|
+
message = str(exc) if isinstance(exc, ValueError) else f"SEC data unavailable ({type(exc).__name__})"
|
|
340
|
+
print(json.dumps({"error": message}, ensure_ascii=False))
|
|
341
|
+
sys.exit(1)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
if __name__ == "__main__":
|
|
345
|
+
utf8_stdio()
|
|
346
|
+
main()
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
"""Exact issuer CUSIPs joined to the latest published SEC 13F bulk window."""
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
import hashlib
|
|
5
|
+
import io
|
|
6
|
+
import math
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import tempfile
|
|
10
|
+
import time
|
|
11
|
+
import zipfile
|
|
12
|
+
from collections import Counter
|
|
13
|
+
from datetime import date
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from xml.etree import ElementTree
|
|
16
|
+
|
|
17
|
+
import _cache
|
|
18
|
+
from sec_edgar import archive_base, sec_get, xml_root
|
|
19
|
+
|
|
20
|
+
CATALOG = "https://www.sec.gov/data-research/sec-markets-data/form-13f-data-sets"
|
|
21
|
+
MONTHS = ("jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def sec_date(value):
|
|
25
|
+
value = value.strip()
|
|
26
|
+
if re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
|
|
27
|
+
return date.fromisoformat(value).isoformat()
|
|
28
|
+
day, month, year = value.lower().split("-")
|
|
29
|
+
return date(int(year), MONTHS.index(month) + 1, int(day)).isoformat()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def issuer_cusips(cik, data):
|
|
33
|
+
recent = data["filings"]["recent"]
|
|
34
|
+
forms = recent.get("form", [])
|
|
35
|
+
candidates = [
|
|
36
|
+
i
|
|
37
|
+
for i, form in enumerate(forms)
|
|
38
|
+
if form
|
|
39
|
+
in (
|
|
40
|
+
"SCHEDULE 13G",
|
|
41
|
+
"SCHEDULE 13G/A",
|
|
42
|
+
"SCHEDULE 13D",
|
|
43
|
+
"SCHEDULE 13D/A",
|
|
44
|
+
"SC 13G",
|
|
45
|
+
"SC 13G/A",
|
|
46
|
+
"SC 13D",
|
|
47
|
+
"SC 13D/A",
|
|
48
|
+
)
|
|
49
|
+
]
|
|
50
|
+
evidence = []
|
|
51
|
+
for i in candidates[:10]:
|
|
52
|
+
document = recent["primaryDocument"][i].rsplit("/", 1)[-1]
|
|
53
|
+
if not re.fullmatch(r"[\w.-]+\.xml", document, re.I):
|
|
54
|
+
continue
|
|
55
|
+
filing = {"accessionNumber": recent["accessionNumber"][i]}
|
|
56
|
+
url = f"{archive_base(cik, filing)}/{document}"
|
|
57
|
+
try:
|
|
58
|
+
root = xml_root(sec_get(url, as_json=False))
|
|
59
|
+
except (ValueError, ElementTree.ParseError):
|
|
60
|
+
continue
|
|
61
|
+
issuer = root.find(".//issuerInfo")
|
|
62
|
+
if issuer is None:
|
|
63
|
+
continue
|
|
64
|
+
identifier = (issuer.findtext("issuerCik") or issuer.findtext("issuerCIK") or "").strip()
|
|
65
|
+
if not identifier.isdigit() or int(identifier) != int(cik):
|
|
66
|
+
continue
|
|
67
|
+
for node in issuer.findall(".//issuerCusipNumber"):
|
|
68
|
+
cusip = (node.text or "").strip().upper()
|
|
69
|
+
if re.fullmatch(r"[A-Z0-9*@#]{8}\d", cusip):
|
|
70
|
+
evidence.append(
|
|
71
|
+
{
|
|
72
|
+
"cusip": cusip,
|
|
73
|
+
"source_url": url,
|
|
74
|
+
"filing_date": recent["filingDate"][i],
|
|
75
|
+
"security_class": root.findtext(".//securitiesClassTitle"),
|
|
76
|
+
}
|
|
77
|
+
)
|
|
78
|
+
if not evidence:
|
|
79
|
+
raise ValueError("No verified issuer CIK/CUSIP mapping in recent structured SEC Schedule 13D/G filings")
|
|
80
|
+
return evidence
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def latest_bulk():
|
|
84
|
+
html = sec_get(CATALOG, as_json=False)
|
|
85
|
+
windows = []
|
|
86
|
+
for href in re.findall(r'href=["\']([^"\']+\.zip)["\']', html, re.I):
|
|
87
|
+
match = re.search(r"(\d{2}[a-z]{3}\d{4})-(\d{2}[a-z]{3}\d{4})_form13f\.zip$", href, re.I)
|
|
88
|
+
if not match:
|
|
89
|
+
continue
|
|
90
|
+
|
|
91
|
+
start, end = (sec_date(f"{value[:2]}-{value[2:5]}-{value[5:]}") for value in match.groups())
|
|
92
|
+
if href.startswith("/files/"):
|
|
93
|
+
url = "https://www.sec.gov" + href
|
|
94
|
+
elif href.startswith("https://www.sec.gov/files/"):
|
|
95
|
+
url = href
|
|
96
|
+
else:
|
|
97
|
+
continue
|
|
98
|
+
if end <= date.today().isoformat():
|
|
99
|
+
windows.append({"url": url, "start": start, "end": end})
|
|
100
|
+
if not windows:
|
|
101
|
+
raise ValueError("SEC 13F bulk catalog returned no published data window")
|
|
102
|
+
return max(windows, key=lambda w: w["end"])
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def bulk_file(window):
|
|
106
|
+
path = _cache.CACHE_DIR / f"sec-13f-{hashlib.sha256(window['url'].encode()).hexdigest()[:16]}.zip"
|
|
107
|
+
if path.exists() and 0 <= time.time() - path.stat().st_mtime < _cache.WEEK and zipfile.is_zipfile(path):
|
|
108
|
+
return path
|
|
109
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
110
|
+
tmp = None
|
|
111
|
+
try:
|
|
112
|
+
response = sec_get(window["url"], stream=True)
|
|
113
|
+
try:
|
|
114
|
+
with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as out:
|
|
115
|
+
tmp = Path(out.name)
|
|
116
|
+
size = 0
|
|
117
|
+
for chunk in response.iter_content(chunk_size=1024 * 1024):
|
|
118
|
+
size += len(chunk)
|
|
119
|
+
if size > 256 * 1024 * 1024:
|
|
120
|
+
raise ValueError("SEC bulk archive exceeds the 256 MiB download limit")
|
|
121
|
+
out.write(chunk)
|
|
122
|
+
finally:
|
|
123
|
+
response.close()
|
|
124
|
+
if not zipfile.is_zipfile(tmp):
|
|
125
|
+
raise ValueError("SEC bulk download is not a valid ZIP archive")
|
|
126
|
+
os.replace(tmp, path)
|
|
127
|
+
return path
|
|
128
|
+
except OSError as exc:
|
|
129
|
+
raise ValueError(f"SEC bulk archive unavailable ({type(exc).__name__})") from None
|
|
130
|
+
finally:
|
|
131
|
+
if tmp:
|
|
132
|
+
tmp.unlink(missing_ok=True)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def table_rows(archive, table):
|
|
136
|
+
names = [n for n in archive.namelist() if n.rsplit("/", 1)[-1].upper() == table + ".TSV"]
|
|
137
|
+
if len(names) != 1 or archive.getinfo(names[0]).file_size > 2 * 1024**3:
|
|
138
|
+
raise ValueError(f"SEC bulk {table} table missing or too large")
|
|
139
|
+
with archive.open(names[0]) as raw, io.TextIOWrapper(raw, encoding="utf-8-sig") as text:
|
|
140
|
+
reader = csv.DictReader(text, delimiter="\t")
|
|
141
|
+
if not reader.fieldnames or "ACCESSION_NUMBER" not in reader.fieldnames:
|
|
142
|
+
raise ValueError(f"SEC bulk {table} schema unavailable")
|
|
143
|
+
yield from reader
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def scan_holders(path, cusips):
|
|
147
|
+
with zipfile.ZipFile(path) as archive:
|
|
148
|
+
submissions = {r["ACCESSION_NUMBER"]: r for r in table_rows(archive, "SUBMISSION")}
|
|
149
|
+
covers = {r["ACCESSION_NUMBER"]: r for r in table_rows(archive, "COVERPAGE")}
|
|
150
|
+
periods = Counter(
|
|
151
|
+
sec_date(r["PERIODOFREPORT"]) for r in submissions.values() if r["SUBMISSIONTYPE"] == "13F-HR"
|
|
152
|
+
)
|
|
153
|
+
if not periods:
|
|
154
|
+
raise ValueError("SEC bulk contains no original 13F-HR reports")
|
|
155
|
+
# The modal report period is the complete quarterly cohort; late older reports occur in every archive.
|
|
156
|
+
period = max(periods, key=lambda p: (periods[p], p))
|
|
157
|
+
managers = {}
|
|
158
|
+
for accession, row in submissions.items():
|
|
159
|
+
if row["SUBMISSIONTYPE"] not in ("13F-HR", "13F-HR/A") or sec_date(row["PERIODOFREPORT"]) != period:
|
|
160
|
+
continue
|
|
161
|
+
row = {**row, "date": sec_date(row["FILING_DATE"]), "accession": accession, **covers.get(accession, {})}
|
|
162
|
+
managers.setdefault(row["CIK"].zfill(10), []).append(row)
|
|
163
|
+
by_accession = {row["accession"]: (cik, row) for cik, filings in managers.items() for row in filings}
|
|
164
|
+
selected = set()
|
|
165
|
+
incomplete = set()
|
|
166
|
+
for cik, filings in managers.items():
|
|
167
|
+
bases = [r for r in filings if r["SUBMISSIONTYPE"] == "13F-HR" or r.get("AMENDMENTTYPE") == "RESTATEMENT"]
|
|
168
|
+
base = max(bases, key=lambda r: (r["date"], r["accession"])) if bases else None
|
|
169
|
+
if base:
|
|
170
|
+
selected.add(base["accession"])
|
|
171
|
+
else:
|
|
172
|
+
incomplete.add(cik)
|
|
173
|
+
for r in filings:
|
|
174
|
+
if r["SUBMISSIONTYPE"] == "13F-HR/A" and (
|
|
175
|
+
not base or (r["date"], r["accession"]) > (base["date"], base["accession"])
|
|
176
|
+
):
|
|
177
|
+
if r.get("AMENDMENTTYPE") == "NEW HOLDINGS":
|
|
178
|
+
selected.add(r["accession"])
|
|
179
|
+
else:
|
|
180
|
+
incomplete.add(cik)
|
|
181
|
+
result = {}
|
|
182
|
+
# shortcut: stream one published archive per uncached CUSIP set; use a persistent DB if query volume warrants it.
|
|
183
|
+
for row in table_rows(archive, "INFOTABLE"):
|
|
184
|
+
accession = row["ACCESSION_NUMBER"]
|
|
185
|
+
if row["CUSIP"].strip().upper() not in cusips or accession not in by_accession:
|
|
186
|
+
continue
|
|
187
|
+
cik, filing = by_accession[accession]
|
|
188
|
+
if accession not in selected and cik not in incomplete:
|
|
189
|
+
continue
|
|
190
|
+
holder = result.setdefault(
|
|
191
|
+
cik,
|
|
192
|
+
{
|
|
193
|
+
"cik": cik,
|
|
194
|
+
"name": filing.get("FILINGMANAGER_NAME"),
|
|
195
|
+
"positions": [],
|
|
196
|
+
"holdings_complete": cik not in incomplete,
|
|
197
|
+
},
|
|
198
|
+
)
|
|
199
|
+
shares, value = float(row["SSHPRNAMT"]), float(row["VALUE"])
|
|
200
|
+
if not math.isfinite(shares) or not math.isfinite(value):
|
|
201
|
+
raise ValueError("SEC bulk contains nonfinite holding amounts")
|
|
202
|
+
multiplier = 1 if filing["date"] >= "2023-01-03" else 1000
|
|
203
|
+
holder["positions"].append(
|
|
204
|
+
{
|
|
205
|
+
"cusip": row["CUSIP"].strip().upper(),
|
|
206
|
+
"security_class": row.get("TITLEOFCLASS"),
|
|
207
|
+
"shares": shares,
|
|
208
|
+
"share_type": row["SSHPRNAMTTYPE"],
|
|
209
|
+
"put_call": row.get("PUTCALL") or None,
|
|
210
|
+
"reported_value_usd": value * multiplier,
|
|
211
|
+
"filing_date": filing["date"],
|
|
212
|
+
"accession": accession,
|
|
213
|
+
"form": filing["SUBMISSIONTYPE"],
|
|
214
|
+
"amendment_applied": accession in selected,
|
|
215
|
+
"filing_url": f"{archive_base(cik, {'accessionNumber': accession})}/{accession}-index.html",
|
|
216
|
+
}
|
|
217
|
+
)
|
|
218
|
+
for holder in result.values():
|
|
219
|
+
positions = holder["positions"]
|
|
220
|
+
holder["shares"] = (
|
|
221
|
+
sum(r["shares"] for r in positions if r["share_type"] == "SH" and not r["put_call"])
|
|
222
|
+
if holder["holdings_complete"]
|
|
223
|
+
else None
|
|
224
|
+
)
|
|
225
|
+
holder["reported_value_usd"] = (
|
|
226
|
+
sum(r["reported_value_usd"] for r in positions) if holder["holdings_complete"] else None
|
|
227
|
+
)
|
|
228
|
+
return (
|
|
229
|
+
period,
|
|
230
|
+
len(incomplete),
|
|
231
|
+
sorted(result.values(), key=lambda r: (-(r["reported_value_usd"] or 0), r["cik"])),
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def issuer_holders(cik, data):
|
|
236
|
+
evidence = issuer_cusips(cik, data)
|
|
237
|
+
cusips = sorted({r["cusip"] for r in evidence})
|
|
238
|
+
window = latest_bulk()
|
|
239
|
+
|
|
240
|
+
def fetch():
|
|
241
|
+
period, unresolved, holders = scan_holders(bulk_file(window), set(cusips))
|
|
242
|
+
return {"report_date": period, "institutions": holders, "unresolved_manager_count": unresolved}
|
|
243
|
+
|
|
244
|
+
result = _cache.cached(f"sec-issuer-holders:v2:{window['url']}:{','.join(cusips)}", _cache.DAY, fetch)
|
|
245
|
+
return {
|
|
246
|
+
"cik": cik,
|
|
247
|
+
"issuer": data.get("name"),
|
|
248
|
+
"query_type": "issuer",
|
|
249
|
+
"source": "sec_edgar",
|
|
250
|
+
"cusips": cusips,
|
|
251
|
+
"cusip_evidence": evidence,
|
|
252
|
+
**result,
|
|
253
|
+
"institution_count": len(result["institutions"]),
|
|
254
|
+
"bulk_source_url": window["url"],
|
|
255
|
+
"filing_window_start": window["start"],
|
|
256
|
+
"filing_window_end": window["end"],
|
|
257
|
+
"mapping_complete": False,
|
|
258
|
+
"complete_for_mapped_cusips": result["unresolved_manager_count"] == 0,
|
|
259
|
+
"realtime": False,
|
|
260
|
+
"value_unit": "USD",
|
|
261
|
+
"coverage": "Published SEC 13F window for verified CUSIPs; mapping may omit other security classes/CUSIP changes; late filings, confidential positions and subsequent amendments are not included",
|
|
262
|
+
"shares_basis": "SH equity positions only; options and principal amounts remain separate in positions",
|
|
263
|
+
}
|