fetchkit-agents 0.5.4__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fetchkit_agents-0.5.4/src/fetchkit_agents.egg-info → fetchkit_agents-0.7.0}/PKG-INFO +37 -10
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/README.md +36 -9
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/__init__.py +1 -1
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/data/AGENT_GUIDE.md +2 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/edgar.py +279 -3
- fetchkit_agents-0.7.0/src/fetchkit/fetchers/github.py +477 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/schemas/fetcher.py +156 -4
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0/src/fetchkit_agents.egg-info}/PKG-INFO +37 -10
- fetchkit_agents-0.5.4/src/fetchkit/fetchers/github.py +0 -204
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/LICENSE +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/pyproject.toml +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/setup.cfg +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/__main__.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/cache.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/cli.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/collector.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/config_loader.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/__init__.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/autodiscover.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/catalog.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/core.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/data/CATALOG.md +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/data/catalog.json +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/embedding.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/errors.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/external.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/ranking.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/schemas.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/examples.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/__init__.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/arxiv.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/base.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/bluesky.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/gdelt.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/googlenews.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/hackernews.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/lobsters.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/manifold.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/mastodon.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/polymarket.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/reddit.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/registry.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/rss.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/stackexchange.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/suggest_registry.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/http/__init__.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/http/client.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/http/rate_limit.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/http/ssrf.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/py.typed +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/schemas/__init__.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/schemas/base.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/schemas/collector.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/schemas/config.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/schemas/describe.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/schemas/post.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/utils/__init__.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/utils/article.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/utils/env.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/utils/html.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/utils/time.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/utils/urls.py +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit_agents.egg-info/SOURCES.txt +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit_agents.egg-info/dependency_links.txt +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit_agents.egg-info/entry_points.txt +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit_agents.egg-info/requires.txt +0 -0
- {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit_agents.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fetchkit-agents
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: A YAML-configured data fetching library for agentic applications.
|
|
5
5
|
Author-email: mete morris <morrismete@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -604,20 +604,47 @@ pulls recent filings per company from `data.sec.gov/submissions` (acceptance
|
|
|
604
604
|
timestamps are near-real-time — new 8-Ks appear within minutes); `resource: search`
|
|
605
605
|
runs EDGAR full-text search across all filers (date granularity only; the run
|
|
606
606
|
window is passed to the API as a date range). Form type, CIK, accession number,
|
|
607
|
-
and 8-K item codes are kept in `post.metadata
|
|
607
|
+
and 8-K item codes are kept in `post.metadata`; with `fetch_bodies` a Form 4 also
|
|
608
|
+
carries `transaction_code`, `transaction_shares`, `transaction_price`,
|
|
609
|
+
`transaction_value`, `owner` and `shares_held_after`.
|
|
608
610
|
|
|
609
611
|
The SEC's [fair-access policy](https://www.sec.gov/os/accessing-edgar-data) asks
|
|
610
612
|
clients to identify themselves with contact info and stay under 10 req/s — set
|
|
611
613
|
`user_agent` and consider `http.rate_limit_per_host`.
|
|
612
614
|
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
615
|
+
Without `fetch_bodies` a filing's `text` is `primaryDocDescription` — a string
|
|
616
|
+
like `"FORM 8-K"` — so a consumer is judging a title. `fetch_bodies: true`
|
|
617
|
+
reads what the filing actually says, and does something different per form
|
|
618
|
+
because the forms are different:
|
|
619
|
+
|
|
620
|
+
- **8-K item codes become a sentence, at no request cost.** The submissions
|
|
621
|
+
response already carries `items`, and the codes *are* the substance: `5.02` is
|
|
622
|
+
an officer leaving and `4.02` is a restatement.
|
|
623
|
+
- **Narrative forms** (8-K, 6-K, SC 13D, 425, DEFA14A) get their primary
|
|
624
|
+
document fetched and stripped, with the inline-XBRL preamble trimmed off.
|
|
625
|
+
- **Form 4** is parsed into transaction fields — who, what code, how many
|
|
626
|
+
shares, at what price — rather than scraped as text.
|
|
627
|
+
- **Periodic reports (10-Q, 10-K) are excluded on purpose**: their primary
|
|
628
|
+
documents run to megabytes.
|
|
629
|
+
|
|
630
|
+
`transaction_codes` and `min_transaction_value` then filter Form 4s by what
|
|
631
|
+
happened. Most Form 4 volume is code `A` (grants) and `M` (option exercises),
|
|
632
|
+
which mean something quite different from an insider buying with their own
|
|
633
|
+
money — see [`docs/edgar_filing_bodies.md`](docs/edgar_filing_bodies.md) for the
|
|
634
|
+
measurements.
|
|
635
|
+
|
|
636
|
+
| Field | Default | Notes |
|
|
637
|
+
|--------------------------|----------|--------------------------------------------------------------|
|
|
638
|
+
| `resource` | filings | `filings` (per-company recent) or `search` (full-text) |
|
|
639
|
+
| `companies` | `[]` | Tickers or CIKs — required for `resource: filings` |
|
|
640
|
+
| `forms` | `[]` | Form types to include, e.g. `["8-K", "10-Q"]` (empty = all) |
|
|
641
|
+
| `query` | null | Full-text query — required for `resource: search` |
|
|
642
|
+
| `fetch_bodies` | false | Read each filing's substance, per form (above) |
|
|
643
|
+
| `max_body_chars` | 20000 | Cap on an extracted body, after stripping |
|
|
644
|
+
| `transaction_codes` | `[]` | Keep only these Form 4 codes, e.g. `["P","S"]`. Needs `fetch_bodies` |
|
|
645
|
+
| `min_transaction_value` | 0 | Keep only Form 4s worth at least this (shares × price). Needs `fetch_bodies` |
|
|
646
|
+
| `user_agent` | null | Descriptive UA with contact info, e.g. `myapp me@example.com`|
|
|
647
|
+
| `max_items` | 50 | 1–500 |
|
|
621
648
|
|
|
622
649
|
### Reddit (`type: reddit`, `type: reddit-auth`)
|
|
623
650
|
|
|
@@ -559,20 +559,47 @@ pulls recent filings per company from `data.sec.gov/submissions` (acceptance
|
|
|
559
559
|
timestamps are near-real-time — new 8-Ks appear within minutes); `resource: search`
|
|
560
560
|
runs EDGAR full-text search across all filers (date granularity only; the run
|
|
561
561
|
window is passed to the API as a date range). Form type, CIK, accession number,
|
|
562
|
-
and 8-K item codes are kept in `post.metadata
|
|
562
|
+
and 8-K item codes are kept in `post.metadata`; with `fetch_bodies` a Form 4 also
|
|
563
|
+
carries `transaction_code`, `transaction_shares`, `transaction_price`,
|
|
564
|
+
`transaction_value`, `owner` and `shares_held_after`.
|
|
563
565
|
|
|
564
566
|
The SEC's [fair-access policy](https://www.sec.gov/os/accessing-edgar-data) asks
|
|
565
567
|
clients to identify themselves with contact info and stay under 10 req/s — set
|
|
566
568
|
`user_agent` and consider `http.rate_limit_per_host`.
|
|
567
569
|
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
570
|
+
Without `fetch_bodies` a filing's `text` is `primaryDocDescription` — a string
|
|
571
|
+
like `"FORM 8-K"` — so a consumer is judging a title. `fetch_bodies: true`
|
|
572
|
+
reads what the filing actually says, and does something different per form
|
|
573
|
+
because the forms are different:
|
|
574
|
+
|
|
575
|
+
- **8-K item codes become a sentence, at no request cost.** The submissions
|
|
576
|
+
response already carries `items`, and the codes *are* the substance: `5.02` is
|
|
577
|
+
an officer leaving and `4.02` is a restatement.
|
|
578
|
+
- **Narrative forms** (8-K, 6-K, SC 13D, 425, DEFA14A) get their primary
|
|
579
|
+
document fetched and stripped, with the inline-XBRL preamble trimmed off.
|
|
580
|
+
- **Form 4** is parsed into transaction fields — who, what code, how many
|
|
581
|
+
shares, at what price — rather than scraped as text.
|
|
582
|
+
- **Periodic reports (10-Q, 10-K) are excluded on purpose**: their primary
|
|
583
|
+
documents run to megabytes.
|
|
584
|
+
|
|
585
|
+
`transaction_codes` and `min_transaction_value` then filter Form 4s by what
|
|
586
|
+
happened. Most Form 4 volume is code `A` (grants) and `M` (option exercises),
|
|
587
|
+
which mean something quite different from an insider buying with their own
|
|
588
|
+
money — see [`docs/edgar_filing_bodies.md`](docs/edgar_filing_bodies.md) for the
|
|
589
|
+
measurements.
|
|
590
|
+
|
|
591
|
+
| Field | Default | Notes |
|
|
592
|
+
|--------------------------|----------|--------------------------------------------------------------|
|
|
593
|
+
| `resource` | filings | `filings` (per-company recent) or `search` (full-text) |
|
|
594
|
+
| `companies` | `[]` | Tickers or CIKs — required for `resource: filings` |
|
|
595
|
+
| `forms` | `[]` | Form types to include, e.g. `["8-K", "10-Q"]` (empty = all) |
|
|
596
|
+
| `query` | null | Full-text query — required for `resource: search` |
|
|
597
|
+
| `fetch_bodies` | false | Read each filing's substance, per form (above) |
|
|
598
|
+
| `max_body_chars` | 20000 | Cap on an extracted body, after stripping |
|
|
599
|
+
| `transaction_codes` | `[]` | Keep only these Form 4 codes, e.g. `["P","S"]`. Needs `fetch_bodies` |
|
|
600
|
+
| `min_transaction_value` | 0 | Keep only Form 4s worth at least this (shares × price). Needs `fetch_bodies` |
|
|
601
|
+
| `user_agent` | null | Descriptive UA with contact info, e.g. `myapp me@example.com`|
|
|
602
|
+
| `max_items` | 50 | 1–500 |
|
|
576
603
|
|
|
577
604
|
### Reddit (`type: reddit`, `type: reddit-auth`)
|
|
578
605
|
|
|
@@ -48,6 +48,7 @@ fetchkit skipped it.
|
|
|
48
48
|
| `hackernews` | Self-post body (Ask/Show HN). Link posts have no body → null |
|
|
49
49
|
| `github` (search_repos) | Repo description; the README with `include_readme: true` |
|
|
50
50
|
| `github` (releases) | Release notes |
|
|
51
|
+
| `github` (pull_requests) | The pull-request body; changed file paths and diff size in `metadata` |
|
|
51
52
|
| `lobsters` | Story description (often empty for pure links) |
|
|
52
53
|
| `stackexchange` | Question body (HTML) |
|
|
53
54
|
| `reddit` | Selftext (link posts → null) |
|
|
@@ -69,6 +70,7 @@ but each source's clock means something different:
|
|
|
69
70
|
| `arxiv` | Paper submission date (daily announce batches — a 1-day window can be legitimately empty) | Client-side |
|
|
70
71
|
| `github` search_repos | Repo creation date; window maps to a `created:>=`/`pushed:>=` search qualifier per `window_qualifier` (**use `pushed` for "active this week"**) | Server-side |
|
|
71
72
|
| `github` releases | Release publish time | Client-side |
|
|
73
|
+
| `github` pull_requests | **Merge** time, not open time — the window is a window of work that landed | Client-side, paging stops past the window |
|
|
72
74
|
| `lobsters` | Story submission time | Client-side |
|
|
73
75
|
| `stackexchange` | Question creation time | Server-side |
|
|
74
76
|
| `bluesky` / `mastodon` / `reddit` | Post time | Client-side |
|
|
@@ -22,6 +22,8 @@ detected here and reported as an actionable error; the fix is network-level
|
|
|
22
22
|
(run from a residential IP, or route through a proxy via ``HTTPS_PROXY``).
|
|
23
23
|
"""
|
|
24
24
|
|
|
25
|
+
import html
|
|
26
|
+
import logging
|
|
25
27
|
import re
|
|
26
28
|
from datetime import datetime, timezone
|
|
27
29
|
from typing import Any, Optional
|
|
@@ -38,6 +40,8 @@ FULL_TEXT_SEARCH_URL = "https://efts.sec.gov/LATEST/search-index"
|
|
|
38
40
|
COMPANY_TICKERS_URL = "https://www.sec.gov/files/company_tickers.json"
|
|
39
41
|
ARCHIVES_BASE = "https://www.sec.gov/Archives/edgar/data"
|
|
40
42
|
|
|
43
|
+
logger = logging.getLogger(__name__)
|
|
44
|
+
|
|
41
45
|
SOURCE_NAME = Source.EDGAR
|
|
42
46
|
DEFAULT_TIMEOUT_S = 15
|
|
43
47
|
# efts.sec.gov returns fixed pages of 10 hits, paginated via a `from` offset.
|
|
@@ -89,6 +93,225 @@ def _resolve_cik(company: str, ticker_map: dict[str, int]) -> Optional[int]:
|
|
|
89
93
|
return ticker_map.get(ticker.replace(".", "-"))
|
|
90
94
|
|
|
91
95
|
|
|
96
|
+
# --- Tier 1: what an 8-K is actually about, for free -------------------------
|
|
97
|
+
#
|
|
98
|
+
# The submissions API already returns an `items` string like "5.02" or
|
|
99
|
+
# "7.01,9.01" on every 8-K. Without it a filing reaches a consumer as
|
|
100
|
+
# `primaryDocDescription` — "FORM 8-K" — which says nothing, and a downstream
|
|
101
|
+
# LLM stage is left judging a title. Mapping the codes costs no request.
|
|
102
|
+
_EIGHT_K_ITEMS: dict[str, str] = {
|
|
103
|
+
"1.01": "entry into a material definitive agreement",
|
|
104
|
+
"1.02": "termination of a material definitive agreement",
|
|
105
|
+
"1.03": "bankruptcy or receivership",
|
|
106
|
+
"1.04": "mine safety reporting",
|
|
107
|
+
"1.05": "a material cybersecurity incident",
|
|
108
|
+
"2.01": "completion of an acquisition or disposition of assets",
|
|
109
|
+
"2.02": "results of operations and financial condition",
|
|
110
|
+
"2.03": "creation of a direct financial obligation",
|
|
111
|
+
"2.04": "triggering events that accelerate a financial obligation",
|
|
112
|
+
"2.05": "costs associated with exit or disposal activities",
|
|
113
|
+
"2.06": "a material impairment",
|
|
114
|
+
"3.01": "notice of delisting or failure to satisfy a listing rule",
|
|
115
|
+
"3.02": "unregistered sales of equity securities",
|
|
116
|
+
"3.03": "material modification to the rights of security holders",
|
|
117
|
+
"4.01": "a change in the registrant's certifying accountant",
|
|
118
|
+
"4.02": "non-reliance on previously issued financial statements",
|
|
119
|
+
"5.01": "a change in control of the registrant",
|
|
120
|
+
"5.02": "departure or election of directors or principal officers",
|
|
121
|
+
"5.03": "amendments to articles of incorporation or bylaws",
|
|
122
|
+
"5.04": "suspension of trading under employee benefit plans",
|
|
123
|
+
"5.05": "amendment to the code of ethics",
|
|
124
|
+
"5.07": "submission of matters to a vote of security holders",
|
|
125
|
+
"5.08": "shareholder director nominations",
|
|
126
|
+
"6.01": "ABS informational and computational material",
|
|
127
|
+
"7.01": "Regulation FD disclosure",
|
|
128
|
+
"8.01": "other events the registrant deems material",
|
|
129
|
+
"9.01": "financial statements and exhibits",
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
#: Form 4 transaction codes. `P` is the one a reader usually means by "insider
|
|
133
|
+
#: buying": an open-market purchase with the insider's own money. `A` and `M`
|
|
134
|
+
#: dominate the volume and mean something quite different — a grant, and an
|
|
135
|
+
#: option exercise — so reporting them as buying is the most common false
|
|
136
|
+
#: signal in this data.
|
|
137
|
+
_TRANSACTION_CODES: dict[str, str] = {
|
|
138
|
+
"P": "open-market purchase",
|
|
139
|
+
"S": "open-market sale",
|
|
140
|
+
"A": "grant or award",
|
|
141
|
+
"M": "option or derivative exercise",
|
|
142
|
+
"F": "shares withheld for tax",
|
|
143
|
+
"G": "gift",
|
|
144
|
+
"D": "disposition to the issuer",
|
|
145
|
+
"C": "conversion of a derivative",
|
|
146
|
+
"X": "in-the-money derivative exercise",
|
|
147
|
+
"V": "transaction voluntarily reported early",
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _describe_filing(form: Optional[str], items: Optional[str],
|
|
152
|
+
description: Optional[str]) -> Optional[str]:
|
|
153
|
+
"""A sentence saying what this filing is, from data already in hand.
|
|
154
|
+
|
|
155
|
+
For an 8-K the item codes are the substance: "5.02" is an officer leaving
|
|
156
|
+
and "4.02" is a restatement, and those are not remotely the same event. For
|
|
157
|
+
every other form the SEC's own description is better than nothing.
|
|
158
|
+
"""
|
|
159
|
+
codes = [c.strip() for c in (items or "").split(",") if c.strip()]
|
|
160
|
+
known = [f"Item {c} — {_EIGHT_K_ITEMS[c]}" for c in codes if c in _EIGHT_K_ITEMS]
|
|
161
|
+
unknown = [f"Item {c}" for c in codes if c not in _EIGHT_K_ITEMS]
|
|
162
|
+
if known or unknown:
|
|
163
|
+
return f"{form or 'Filing'} reporting: " + "; ".join(known + unknown) + "."
|
|
164
|
+
return description or None
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
# --- Tier 2: the narrative body of a filing that has one ---------------------
|
|
168
|
+
|
|
169
|
+
#: Forms whose primary document is prose worth reading. Deliberately a short
|
|
170
|
+
#: allowlist rather than "everything that is not a Form 4": a 10-Q's primary
|
|
171
|
+
#: document is 1.5-7.7 MB of inline-XBRL HTML (measured on MSFT, AMZN and NVDA,
|
|
172
|
+
#: 185k-320k characters after stripping), which no prompt budget can hold and
|
|
173
|
+
#: which would be truncated into uselessness anyway.
|
|
174
|
+
_NARRATIVE_FORMS = {"8-K", "8-K/A", "6-K", "SC 13D", "SC 13D/A", "425", "DEFA14A"}
|
|
175
|
+
|
|
176
|
+
_BODY_ANCHORS = (
|
|
177
|
+
"UNITED STATES SECURITIES AND EXCHANGE COMMISSION",
|
|
178
|
+
"SECURITIES AND EXCHANGE COMMISSION",
|
|
179
|
+
)
|
|
180
|
+
#: Inline-XBRL leaves a run of machine tokens ahead of the readable document —
|
|
181
|
+
#: namespace-qualified concept names, the CIK repeated, bare dates and
|
|
182
|
+
#: `true`/`false` flags. Measured on MSFT's 8-K the first ~200 characters were
|
|
183
|
+
#: entirely this. They are dropped token-wise when no anchor is found.
|
|
184
|
+
_XBRL_TOKEN = re.compile(r"^(?:[A-Za-z][\w-]*:[\w.-]+|\d{4}-\d{2}-\d{2}|\d{6,}|true|false)$")
|
|
185
|
+
_TAG = re.compile(r"<[^>]+>")
|
|
186
|
+
_SCRIPT = re.compile(r"<(script|style)[^>]*>.*?</\1>", re.S | re.I)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _strip_filing_html(document: str) -> str:
|
|
190
|
+
"""Plain text from a filing's primary document, minus the XBRL preamble."""
|
|
191
|
+
text = _SCRIPT.sub(" ", document)
|
|
192
|
+
text = _TAG.sub(" ", text)
|
|
193
|
+
text = html.unescape(text)
|
|
194
|
+
text = re.sub(r"\s+", " ", text).strip()
|
|
195
|
+
|
|
196
|
+
for anchor in _BODY_ANCHORS:
|
|
197
|
+
position = text.find(anchor)
|
|
198
|
+
if 0 <= position < 4000:
|
|
199
|
+
return text[position:]
|
|
200
|
+
|
|
201
|
+
tokens = text.split(" ")
|
|
202
|
+
cut = 0
|
|
203
|
+
for index, token in enumerate(tokens[:200]):
|
|
204
|
+
if _XBRL_TOKEN.match(token):
|
|
205
|
+
cut = index + 1
|
|
206
|
+
return " ".join(tokens[cut:]) if cut else text
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _fetch_filing_body(url: str, config: EdgarFetchConfig) -> Optional[str]:
|
|
210
|
+
"""Best-effort body for one filing. Never raises: a filing without a
|
|
211
|
+
readable body keeps whatever text it already had."""
|
|
212
|
+
try:
|
|
213
|
+
client = get_default_client()
|
|
214
|
+
resp = client.get(url, headers=_headers(config), timeout=DEFAULT_TIMEOUT_S)
|
|
215
|
+
_sec_raise_for_status(resp)
|
|
216
|
+
except Exception as exc: # noqa: BLE001 - a missing body is routine
|
|
217
|
+
logger.debug("edgar: no body for %s (%s)", url, type(exc).__name__)
|
|
218
|
+
return None
|
|
219
|
+
body = _strip_filing_html(resp.text)
|
|
220
|
+
if len(body) < _MIN_BODY_CHARS:
|
|
221
|
+
return None
|
|
222
|
+
return body[: config.max_body_chars]
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
_MIN_BODY_CHARS = 200
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
# --- Tier 3: a Form 4 as fields, not as text ---------------------------------
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _form4_field(xml: str, tag: str) -> Optional[str]:
|
|
232
|
+
"""First value of ``tag``, unwrapping the <value> element Form 4 uses."""
|
|
233
|
+
match = re.search(rf"<{tag}>\s*(?:<value>)?\s*([^<]+)", xml)
|
|
234
|
+
return match.group(1).strip() if match else None
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _fetch_form4(cik: int, accession: str, config: EdgarFetchConfig
|
|
238
|
+
) -> Optional[dict[str, Any]]:
|
|
239
|
+
"""Parse one Form 4 into fields. None when it cannot be read.
|
|
240
|
+
|
|
241
|
+
Two requests: the accession's `index.json`, because the XML's filename is
|
|
242
|
+
not derivable (`wk-form4_1789160684.xml` on a live NVDA filing), then the
|
|
243
|
+
document itself. The XSL-rendered view named as `primaryDocument` is a
|
|
244
|
+
styled table — stripping it yields labels and numbers, not prose — so the
|
|
245
|
+
raw XML beside it is what carries meaning.
|
|
246
|
+
"""
|
|
247
|
+
base = f"{ARCHIVES_BASE}/{cik}/{accession.replace('-', '')}"
|
|
248
|
+
try:
|
|
249
|
+
client = get_default_client()
|
|
250
|
+
listing = client.get(f"{base}/index.json", headers=_headers(config),
|
|
251
|
+
timeout=DEFAULT_TIMEOUT_S)
|
|
252
|
+
_sec_raise_for_status(listing)
|
|
253
|
+
items = (listing.json().get("directory") or {}).get("item") or []
|
|
254
|
+
names = [i.get("name", "") if isinstance(i, dict) else str(i) for i in items]
|
|
255
|
+
raw = [n for n in names if n.endswith(".xml") and "xsl" not in n.lower()]
|
|
256
|
+
if not raw:
|
|
257
|
+
return None
|
|
258
|
+
document = client.get(f"{base}/{raw[0]}", headers=_headers(config),
|
|
259
|
+
timeout=DEFAULT_TIMEOUT_S)
|
|
260
|
+
_sec_raise_for_status(document)
|
|
261
|
+
xml = document.text
|
|
262
|
+
except Exception as exc: # noqa: BLE001 - routine, never fails the fetch
|
|
263
|
+
logger.debug("edgar: could not read Form 4 %s (%s)", accession,
|
|
264
|
+
type(exc).__name__)
|
|
265
|
+
return None
|
|
266
|
+
|
|
267
|
+
code = (_form4_field(xml, "transactionCode") or "").upper() or None
|
|
268
|
+
shares = _form4_field(xml, "transactionShares")
|
|
269
|
+
price = _form4_field(xml, "transactionPricePerShare")
|
|
270
|
+
owner = _form4_field(xml, "rptOwnerName")
|
|
271
|
+
title = (_form4_field(xml, "officerTitle")
|
|
272
|
+
or ("director" if _form4_field(xml, "isDirector") in ("1", "true")
|
|
273
|
+
else None))
|
|
274
|
+
|
|
275
|
+
def number(value: Optional[str]) -> Optional[float]:
|
|
276
|
+
try:
|
|
277
|
+
return float(value) if value is not None else None
|
|
278
|
+
except ValueError:
|
|
279
|
+
return None
|
|
280
|
+
|
|
281
|
+
share_count, unit_price = number(shares), number(price)
|
|
282
|
+
parts = [owner or "An insider"]
|
|
283
|
+
if title:
|
|
284
|
+
parts.append(f"({title})")
|
|
285
|
+
parts.append(_TRANSACTION_CODES.get(code, "transaction") if code
|
|
286
|
+
else "transaction")
|
|
287
|
+
if share_count:
|
|
288
|
+
parts.append(f"of {share_count:,.0f} share"
|
|
289
|
+
+ ("" if share_count == 1 else "s"))
|
|
290
|
+
if unit_price is not None:
|
|
291
|
+
parts.append(f"at ${unit_price:,.2f}" if unit_price else "at no cost")
|
|
292
|
+
held = number(_form4_field(xml, "sharesOwnedFollowingTransaction"))
|
|
293
|
+
summary = " ".join(parts) + "."
|
|
294
|
+
if held is not None:
|
|
295
|
+
summary += (f" Holding afterwards: {held:,.0f} share"
|
|
296
|
+
+ ("" if held == 1 else "s") + ".")
|
|
297
|
+
|
|
298
|
+
value = (share_count * unit_price
|
|
299
|
+
if share_count is not None and unit_price is not None else None)
|
|
300
|
+
if value:
|
|
301
|
+
summary += f" Transaction value: ${value:,.0f}."
|
|
302
|
+
|
|
303
|
+
return {
|
|
304
|
+
"summary": summary,
|
|
305
|
+
"transaction_value": value,
|
|
306
|
+
"transaction_code": code,
|
|
307
|
+
"transaction_shares": share_count,
|
|
308
|
+
"transaction_price": unit_price,
|
|
309
|
+
"owner": owner,
|
|
310
|
+
"owner_title": title,
|
|
311
|
+
"shares_held_after": held,
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
|
|
92
315
|
def _load_ticker_map(headers: dict[str, str]) -> dict[str, int]:
|
|
93
316
|
"""Fetch the SEC's ticker→CIK mapping (one request, covers all companies)."""
|
|
94
317
|
client = get_default_client()
|
|
@@ -146,8 +369,9 @@ def _recent_filings_to_posts(cik: int, submissions: dict[str, Any],
|
|
|
146
369
|
posts.append(Post(
|
|
147
370
|
id=str(accession),
|
|
148
371
|
source=SOURCE_NAME,
|
|
149
|
-
title=f"{form}: {company}" if form else company,
|
|
150
|
-
text=
|
|
372
|
+
title=f"Form {form}: {company}" if form else company,
|
|
373
|
+
text=_describe_filing(form, items[i] if i < len(items) else None,
|
|
374
|
+
description),
|
|
151
375
|
url=_filing_url(cik, accession, document),
|
|
152
376
|
author=company,
|
|
153
377
|
created_at=(acceptance[i] if i < len(acceptance) else None)
|
|
@@ -186,7 +410,59 @@ def _fetch_company_filings(config: EdgarFetchConfig) -> list[Post]:
|
|
|
186
410
|
timeout=DEFAULT_TIMEOUT_S)
|
|
187
411
|
_sec_raise_for_status(resp)
|
|
188
412
|
posts.extend(_recent_filings_to_posts(cik, resp.json(), config))
|
|
189
|
-
|
|
413
|
+
kept = [p for p in posts if _in_window(p, config)]
|
|
414
|
+
return _enrich(kept, config) if config.fetch_bodies else kept
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _enrich(posts: list[Post], config: EdgarFetchConfig) -> list[Post]:
|
|
418
|
+
"""Read each filing's substance, and drop Form 4s the caller filtered out.
|
|
419
|
+
|
|
420
|
+
Ordered so nothing is paid for twice: the window filter has already run, so
|
|
421
|
+
only filings a consumer will actually see are fetched, and a Form 4 excluded
|
|
422
|
+
by `transaction_codes` is dropped the moment its code is known.
|
|
423
|
+
"""
|
|
424
|
+
wanted = {c.upper() for c in config.transaction_codes}
|
|
425
|
+
floor = config.min_transaction_value
|
|
426
|
+
out: list[Post] = []
|
|
427
|
+
for post in posts:
|
|
428
|
+
form = (post.metadata or {}).get("form") or ""
|
|
429
|
+
cik = (post.metadata or {}).get("cik")
|
|
430
|
+
accession = (post.metadata or {}).get("accession")
|
|
431
|
+
|
|
432
|
+
if form.upper().startswith("4") and cik and accession:
|
|
433
|
+
parsed = _fetch_form4(int(cik), str(accession), config)
|
|
434
|
+
if parsed:
|
|
435
|
+
if wanted and (parsed.get("transaction_code") or "") not in wanted:
|
|
436
|
+
continue
|
|
437
|
+
if floor:
|
|
438
|
+
value = parsed.get("transaction_value")
|
|
439
|
+
# An unpriced filing cannot clear a value floor. A grant is
|
|
440
|
+
# priced at $0 and is genuinely worth nothing to a reader
|
|
441
|
+
# watching for conviction; an unreadable one is dropped for
|
|
442
|
+
# the same reason the code filter drops it.
|
|
443
|
+
if not value or value < floor:
|
|
444
|
+
continue
|
|
445
|
+
post = post.model_copy(update={
|
|
446
|
+
"text": parsed["summary"],
|
|
447
|
+
"metadata": {**(post.metadata or {}),
|
|
448
|
+
**{k: v for k, v in parsed.items() if k != "summary"}},
|
|
449
|
+
})
|
|
450
|
+
elif wanted or floor:
|
|
451
|
+
# The filter could not be evaluated. Dropping is the honest
|
|
452
|
+
# choice: a caller asking for open-market purchases must not be
|
|
453
|
+
# handed an unreadable filing that might be a grant.
|
|
454
|
+
continue
|
|
455
|
+
elif form.upper() in _NARRATIVE_FORMS and post.url:
|
|
456
|
+
body = _fetch_filing_body(post.url, config)
|
|
457
|
+
if body:
|
|
458
|
+
# The item-code sentence still leads: it names what the filing
|
|
459
|
+
# is *about*, which the document's own preamble does not.
|
|
460
|
+
post = post.model_copy(update={
|
|
461
|
+
"text": f"{post.text} {body}".strip() if post.text else body,
|
|
462
|
+
"metadata": {**(post.metadata or {}), "body_source": "filing"},
|
|
463
|
+
})
|
|
464
|
+
out.append(post)
|
|
465
|
+
return out
|
|
190
466
|
|
|
191
467
|
|
|
192
468
|
def _clean_display_name(name: str) -> str:
|