fetchkit-agents 0.5.4__tar.gz → 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. {fetchkit_agents-0.5.4/src/fetchkit_agents.egg-info → fetchkit_agents-0.7.0}/PKG-INFO +37 -10
  2. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/README.md +36 -9
  3. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/__init__.py +1 -1
  4. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/data/AGENT_GUIDE.md +2 -0
  5. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/edgar.py +279 -3
  6. fetchkit_agents-0.7.0/src/fetchkit/fetchers/github.py +477 -0
  7. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/schemas/fetcher.py +156 -4
  8. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0/src/fetchkit_agents.egg-info}/PKG-INFO +37 -10
  9. fetchkit_agents-0.5.4/src/fetchkit/fetchers/github.py +0 -204
  10. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/LICENSE +0 -0
  11. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/pyproject.toml +0 -0
  12. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/setup.cfg +0 -0
  13. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/__main__.py +0 -0
  14. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/cache.py +0 -0
  15. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/cli.py +0 -0
  16. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/collector.py +0 -0
  17. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/config_loader.py +0 -0
  18. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/__init__.py +0 -0
  19. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/autodiscover.py +0 -0
  20. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/catalog.py +0 -0
  21. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/core.py +0 -0
  22. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/data/CATALOG.md +0 -0
  23. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/data/catalog.json +0 -0
  24. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/embedding.py +0 -0
  25. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/errors.py +0 -0
  26. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/external.py +0 -0
  27. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/ranking.py +0 -0
  28. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/discovery/schemas.py +0 -0
  29. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/examples.py +0 -0
  30. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/__init__.py +0 -0
  31. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/arxiv.py +0 -0
  32. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/base.py +0 -0
  33. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/bluesky.py +0 -0
  34. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/gdelt.py +0 -0
  35. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/googlenews.py +0 -0
  36. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/hackernews.py +0 -0
  37. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/lobsters.py +0 -0
  38. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/manifold.py +0 -0
  39. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/mastodon.py +0 -0
  40. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/polymarket.py +0 -0
  41. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/reddit.py +0 -0
  42. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/registry.py +0 -0
  43. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/rss.py +0 -0
  44. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/stackexchange.py +0 -0
  45. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/fetchers/suggest_registry.py +0 -0
  46. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/http/__init__.py +0 -0
  47. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/http/client.py +0 -0
  48. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/http/rate_limit.py +0 -0
  49. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/http/ssrf.py +0 -0
  50. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/py.typed +0 -0
  51. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/schemas/__init__.py +0 -0
  52. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/schemas/base.py +0 -0
  53. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/schemas/collector.py +0 -0
  54. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/schemas/config.py +0 -0
  55. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/schemas/describe.py +0 -0
  56. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/schemas/post.py +0 -0
  57. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/utils/__init__.py +0 -0
  58. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/utils/article.py +0 -0
  59. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/utils/env.py +0 -0
  60. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/utils/html.py +0 -0
  61. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/utils/time.py +0 -0
  62. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit/utils/urls.py +0 -0
  63. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit_agents.egg-info/SOURCES.txt +0 -0
  64. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit_agents.egg-info/dependency_links.txt +0 -0
  65. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit_agents.egg-info/entry_points.txt +0 -0
  66. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit_agents.egg-info/requires.txt +0 -0
  67. {fetchkit_agents-0.5.4 → fetchkit_agents-0.7.0}/src/fetchkit_agents.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fetchkit-agents
3
- Version: 0.5.4
3
+ Version: 0.7.0
4
4
  Summary: A YAML-configured data fetching library for agentic applications.
5
5
  Author-email: mete morris <morrismete@gmail.com>
6
6
  License: MIT
@@ -604,20 +604,47 @@ pulls recent filings per company from `data.sec.gov/submissions` (acceptance
604
604
  timestamps are near-real-time — new 8-Ks appear within minutes); `resource: search`
605
605
  runs EDGAR full-text search across all filers (date granularity only; the run
606
606
  window is passed to the API as a date range). Form type, CIK, accession number,
607
- and 8-K item codes are kept in `post.metadata`.
607
+ and 8-K item codes are kept in `post.metadata`; with `fetch_bodies` a Form 4 also
608
+ carries `transaction_code`, `transaction_shares`, `transaction_price`,
609
+ `transaction_value`, `owner` and `shares_held_after`.
608
610
 
609
611
  The SEC's [fair-access policy](https://www.sec.gov/os/accessing-edgar-data) asks
610
612
  clients to identify themselves with contact info and stay under 10 req/s — set
611
613
  `user_agent` and consider `http.rate_limit_per_host`.
612
614
 
613
- | Field | Default | Notes |
614
- |--------------|----------|--------------------------------------------------------------|
615
- | `resource` | filings | `filings` (per-company recent) or `search` (full-text) |
616
- | `companies` | `[]` | Tickers or CIKs — required for `resource: filings` |
617
- | `forms` | `[]` | Form types to include, e.g. `["8-K", "10-Q"]` (empty = all) |
618
- | `query` | null | Full-text query — required for `resource: search` |
619
- | `user_agent` | null | Descriptive UA with contact info, e.g. `myapp me@example.com`|
620
- | `max_items` | 50 | 1–500 |
615
+ Without `fetch_bodies` a filing's `text` is `primaryDocDescription` — a string
616
+ like `"FORM 8-K"` — so a consumer is judging a title. `fetch_bodies: true`
617
+ reads what the filing actually says, and does something different per form
618
+ because the forms are different:
619
+
620
+ - **8-K item codes become a sentence, at no request cost.** The submissions
621
+ response already carries `items`, and the codes *are* the substance: `5.02` is
622
+ an officer leaving and `4.02` is a restatement.
623
+ - **Narrative forms** (8-K, 6-K, SC 13D, 425, DEFA14A) get their primary
624
+ document fetched and stripped, with the inline-XBRL preamble trimmed off.
625
+ - **Form 4** is parsed into transaction fields — who, what code, how many
626
+ shares, at what price — rather than scraped as text.
627
+ - **Periodic reports (10-Q, 10-K) are excluded on purpose**: their primary
628
+ documents run to megabytes.
629
+
630
+ `transaction_codes` and `min_transaction_value` then filter Form 4s by what
631
+ happened. Most Form 4 volume is code `A` (grants) and `M` (option exercises),
632
+ which mean something quite different from an insider buying with their own
633
+ money — see [`docs/edgar_filing_bodies.md`](docs/edgar_filing_bodies.md) for the
634
+ measurements.
635
+
636
+ | Field | Default | Notes |
637
+ |--------------------------|----------|--------------------------------------------------------------|
638
+ | `resource` | filings | `filings` (per-company recent) or `search` (full-text) |
639
+ | `companies` | `[]` | Tickers or CIKs — required for `resource: filings` |
640
+ | `forms` | `[]` | Form types to include, e.g. `["8-K", "10-Q"]` (empty = all) |
641
+ | `query` | null | Full-text query — required for `resource: search` |
642
+ | `fetch_bodies` | false | Read each filing's substance, per form (above) |
643
+ | `max_body_chars` | 20000 | Cap on an extracted body, after stripping |
644
+ | `transaction_codes` | `[]` | Keep only these Form 4 codes, e.g. `["P","S"]`. Needs `fetch_bodies` |
645
+ | `min_transaction_value` | 0 | Keep only Form 4s worth at least this (shares × price). Needs `fetch_bodies` |
646
+ | `user_agent` | null | Descriptive UA with contact info, e.g. `myapp me@example.com`|
647
+ | `max_items` | 50 | 1–500 |
621
648
 
622
649
  ### Reddit (`type: reddit`, `type: reddit-auth`)
623
650
 
@@ -559,20 +559,47 @@ pulls recent filings per company from `data.sec.gov/submissions` (acceptance
559
559
  timestamps are near-real-time — new 8-Ks appear within minutes); `resource: search`
560
560
  runs EDGAR full-text search across all filers (date granularity only; the run
561
561
  window is passed to the API as a date range). Form type, CIK, accession number,
562
- and 8-K item codes are kept in `post.metadata`.
562
+ and 8-K item codes are kept in `post.metadata`; with `fetch_bodies` a Form 4 also
563
+ carries `transaction_code`, `transaction_shares`, `transaction_price`,
564
+ `transaction_value`, `owner` and `shares_held_after`.
563
565
 
564
566
  The SEC's [fair-access policy](https://www.sec.gov/os/accessing-edgar-data) asks
565
567
  clients to identify themselves with contact info and stay under 10 req/s — set
566
568
  `user_agent` and consider `http.rate_limit_per_host`.
567
569
 
568
- | Field | Default | Notes |
569
- |--------------|----------|--------------------------------------------------------------|
570
- | `resource` | filings | `filings` (per-company recent) or `search` (full-text) |
571
- | `companies` | `[]` | Tickers or CIKs — required for `resource: filings` |
572
- | `forms` | `[]` | Form types to include, e.g. `["8-K", "10-Q"]` (empty = all) |
573
- | `query` | null | Full-text query — required for `resource: search` |
574
- | `user_agent` | null | Descriptive UA with contact info, e.g. `myapp me@example.com`|
575
- | `max_items` | 50 | 1–500 |
570
+ Without `fetch_bodies` a filing's `text` is `primaryDocDescription` — a string
571
+ like `"FORM 8-K"` — so a consumer is judging a title. `fetch_bodies: true`
572
+ reads what the filing actually says, and does something different per form
573
+ because the forms are different:
574
+
575
+ - **8-K item codes become a sentence, at no request cost.** The submissions
576
+ response already carries `items`, and the codes *are* the substance: `5.02` is
577
+ an officer leaving and `4.02` is a restatement.
578
+ - **Narrative forms** (8-K, 6-K, SC 13D, 425, DEFA14A) get their primary
579
+ document fetched and stripped, with the inline-XBRL preamble trimmed off.
580
+ - **Form 4** is parsed into transaction fields — who, what code, how many
581
+ shares, at what price — rather than scraped as text.
582
+ - **Periodic reports (10-Q, 10-K) are excluded on purpose**: their primary
583
+ documents run to megabytes.
584
+
585
+ `transaction_codes` and `min_transaction_value` then filter Form 4s by what
586
+ happened. Most Form 4 volume is code `A` (grants) and `M` (option exercises),
587
+ which mean something quite different from an insider buying with their own
588
+ money — see [`docs/edgar_filing_bodies.md`](docs/edgar_filing_bodies.md) for the
589
+ measurements.
590
+
591
+ | Field | Default | Notes |
592
+ |--------------------------|----------|--------------------------------------------------------------|
593
+ | `resource` | filings | `filings` (per-company recent) or `search` (full-text) |
594
+ | `companies` | `[]` | Tickers or CIKs — required for `resource: filings` |
595
+ | `forms` | `[]` | Form types to include, e.g. `["8-K", "10-Q"]` (empty = all) |
596
+ | `query` | null | Full-text query — required for `resource: search` |
597
+ | `fetch_bodies` | false | Read each filing's substance, per form (above) |
598
+ | `max_body_chars` | 20000 | Cap on an extracted body, after stripping |
599
+ | `transaction_codes` | `[]` | Keep only these Form 4 codes, e.g. `["P","S"]`. Needs `fetch_bodies` |
600
+ | `min_transaction_value` | 0 | Keep only Form 4s worth at least this (shares × price). Needs `fetch_bodies` |
601
+ | `user_agent` | null | Descriptive UA with contact info, e.g. `myapp me@example.com`|
602
+ | `max_items` | 50 | 1–500 |
576
603
 
577
604
  ### Reddit (`type: reddit`, `type: reddit-auth`)
578
605
 
@@ -27,7 +27,7 @@ See the README for the full configuration reference and the guide to adding
27
27
  fetchers.
28
28
  """
29
29
 
30
- __version__ = "0.5.4"
30
+ __version__ = "0.7.0"
31
31
 
32
32
  from typing import TYPE_CHECKING, Any
33
33
 
@@ -48,6 +48,7 @@ fetchkit skipped it.
48
48
  | `hackernews` | Self-post body (Ask/Show HN). Link posts have no body → null |
49
49
  | `github` (search_repos) | Repo description; the README with `include_readme: true` |
50
50
  | `github` (releases) | Release notes |
51
+ | `github` (pull_requests) | The pull-request body; changed file paths and diff size in `metadata` |
51
52
  | `lobsters` | Story description (often empty for pure links) |
52
53
  | `stackexchange` | Question body (HTML) |
53
54
  | `reddit` | Selftext (link posts → null) |
@@ -69,6 +70,7 @@ but each source's clock means something different:
69
70
  | `arxiv` | Paper submission date (daily announce batches — a 1-day window can be legitimately empty) | Client-side |
70
71
  | `github` search_repos | Repo creation date; window maps to a `created:>=`/`pushed:>=` search qualifier per `window_qualifier` (**use `pushed` for "active this week"**) | Server-side |
71
72
  | `github` releases | Release publish time | Client-side |
73
+ | `github` pull_requests | **Merge** time, not open time — the window is a window of work that landed | Client-side, paging stops past the window |
72
74
  | `lobsters` | Story submission time | Client-side |
73
75
  | `stackexchange` | Question creation time | Server-side |
74
76
  | `bluesky` / `mastodon` / `reddit` | Post time | Client-side |
@@ -22,6 +22,8 @@ detected here and reported as an actionable error; the fix is network-level
22
22
  (run from a residential IP, or route through a proxy via ``HTTPS_PROXY``).
23
23
  """
24
24
 
25
+ import html
26
+ import logging
25
27
  import re
26
28
  from datetime import datetime, timezone
27
29
  from typing import Any, Optional
@@ -38,6 +40,8 @@ FULL_TEXT_SEARCH_URL = "https://efts.sec.gov/LATEST/search-index"
38
40
  COMPANY_TICKERS_URL = "https://www.sec.gov/files/company_tickers.json"
39
41
  ARCHIVES_BASE = "https://www.sec.gov/Archives/edgar/data"
40
42
 
43
+ logger = logging.getLogger(__name__)
44
+
41
45
  SOURCE_NAME = Source.EDGAR
42
46
  DEFAULT_TIMEOUT_S = 15
43
47
  # efts.sec.gov returns fixed pages of 10 hits, paginated via a `from` offset.
@@ -89,6 +93,225 @@ def _resolve_cik(company: str, ticker_map: dict[str, int]) -> Optional[int]:
89
93
  return ticker_map.get(ticker.replace(".", "-"))
90
94
 
91
95
 
96
+ # --- Tier 1: what an 8-K is actually about, for free -------------------------
97
+ #
98
+ # The submissions API already returns an `items` string like "5.02" or
99
+ # "7.01,9.01" on every 8-K. Without it a filing reaches a consumer as
100
+ # `primaryDocDescription` — "FORM 8-K" — which says nothing, and a downstream
101
+ # LLM stage is left judging a title. Mapping the codes costs no request.
102
+ _EIGHT_K_ITEMS: dict[str, str] = {
103
+ "1.01": "entry into a material definitive agreement",
104
+ "1.02": "termination of a material definitive agreement",
105
+ "1.03": "bankruptcy or receivership",
106
+ "1.04": "mine safety reporting",
107
+ "1.05": "a material cybersecurity incident",
108
+ "2.01": "completion of an acquisition or disposition of assets",
109
+ "2.02": "results of operations and financial condition",
110
+ "2.03": "creation of a direct financial obligation",
111
+ "2.04": "triggering events that accelerate a financial obligation",
112
+ "2.05": "costs associated with exit or disposal activities",
113
+ "2.06": "a material impairment",
114
+ "3.01": "notice of delisting or failure to satisfy a listing rule",
115
+ "3.02": "unregistered sales of equity securities",
116
+ "3.03": "material modification to the rights of security holders",
117
+ "4.01": "a change in the registrant's certifying accountant",
118
+ "4.02": "non-reliance on previously issued financial statements",
119
+ "5.01": "a change in control of the registrant",
120
+ "5.02": "departure or election of directors or principal officers",
121
+ "5.03": "amendments to articles of incorporation or bylaws",
122
+ "5.04": "suspension of trading under employee benefit plans",
123
+ "5.05": "amendment to the code of ethics",
124
+ "5.07": "submission of matters to a vote of security holders",
125
+ "5.08": "shareholder director nominations",
126
+ "6.01": "ABS informational and computational material",
127
+ "7.01": "Regulation FD disclosure",
128
+ "8.01": "other events the registrant deems material",
129
+ "9.01": "financial statements and exhibits",
130
+ }
131
+
132
+ #: Form 4 transaction codes. `P` is the one a reader usually means by "insider
133
+ #: buying": an open-market purchase with the insider's own money. `A` and `M`
134
+ #: dominate the volume and mean something quite different — a grant, and an
135
+ #: option exercise — so reporting them as buying is the most common false
136
+ #: signal in this data.
137
+ _TRANSACTION_CODES: dict[str, str] = {
138
+ "P": "open-market purchase",
139
+ "S": "open-market sale",
140
+ "A": "grant or award",
141
+ "M": "option or derivative exercise",
142
+ "F": "shares withheld for tax",
143
+ "G": "gift",
144
+ "D": "disposition to the issuer",
145
+ "C": "conversion of a derivative",
146
+ "X": "in-the-money derivative exercise",
147
+ "V": "transaction voluntarily reported early",
148
+ }
149
+
150
+
151
+ def _describe_filing(form: Optional[str], items: Optional[str],
152
+ description: Optional[str]) -> Optional[str]:
153
+ """A sentence saying what this filing is, from data already in hand.
154
+
155
+ For an 8-K the item codes are the substance: "5.02" is an officer leaving
156
+ and "4.02" is a restatement, and those are not remotely the same event. For
157
+ every other form the SEC's own description is better than nothing.
158
+ """
159
+ codes = [c.strip() for c in (items or "").split(",") if c.strip()]
160
+ known = [f"Item {c} — {_EIGHT_K_ITEMS[c]}" for c in codes if c in _EIGHT_K_ITEMS]
161
+ unknown = [f"Item {c}" for c in codes if c not in _EIGHT_K_ITEMS]
162
+ if known or unknown:
163
+ return f"{form or 'Filing'} reporting: " + "; ".join(known + unknown) + "."
164
+ return description or None
165
+
166
+
167
+ # --- Tier 2: the narrative body of a filing that has one ---------------------
168
+
169
+ #: Forms whose primary document is prose worth reading. Deliberately a short
170
+ #: allowlist rather than "everything that is not a Form 4": a 10-Q's primary
171
+ #: document is 1.5-7.7 MB of inline-XBRL HTML (measured on MSFT, AMZN and NVDA,
172
+ #: 185k-320k characters after stripping), which no prompt budget can hold and
173
+ #: which would be truncated into uselessness anyway.
174
+ _NARRATIVE_FORMS = {"8-K", "8-K/A", "6-K", "SC 13D", "SC 13D/A", "425", "DEFA14A"}
175
+
176
+ _BODY_ANCHORS = (
177
+ "UNITED STATES SECURITIES AND EXCHANGE COMMISSION",
178
+ "SECURITIES AND EXCHANGE COMMISSION",
179
+ )
180
+ #: Inline-XBRL leaves a run of machine tokens ahead of the readable document —
181
+ #: namespace-qualified concept names, the CIK repeated, bare dates and
182
+ #: `true`/`false` flags. Measured on MSFT's 8-K the first ~200 characters were
183
+ #: entirely this. They are dropped token-wise when no anchor is found.
184
+ _XBRL_TOKEN = re.compile(r"^(?:[A-Za-z][\w-]*:[\w.-]+|\d{4}-\d{2}-\d{2}|\d{6,}|true|false)$")
185
+ _TAG = re.compile(r"<[^>]+>")
186
+ _SCRIPT = re.compile(r"<(script|style)[^>]*>.*?</\1>", re.S | re.I)
187
+
188
+
189
+ def _strip_filing_html(document: str) -> str:
190
+ """Plain text from a filing's primary document, minus the XBRL preamble."""
191
+ text = _SCRIPT.sub(" ", document)
192
+ text = _TAG.sub(" ", text)
193
+ text = html.unescape(text)
194
+ text = re.sub(r"\s+", " ", text).strip()
195
+
196
+ for anchor in _BODY_ANCHORS:
197
+ position = text.find(anchor)
198
+ if 0 <= position < 4000:
199
+ return text[position:]
200
+
201
+ tokens = text.split(" ")
202
+ cut = 0
203
+ for index, token in enumerate(tokens[:200]):
204
+ if _XBRL_TOKEN.match(token):
205
+ cut = index + 1
206
+ return " ".join(tokens[cut:]) if cut else text
207
+
208
+
209
+ def _fetch_filing_body(url: str, config: EdgarFetchConfig) -> Optional[str]:
210
+ """Best-effort body for one filing. Never raises: a filing without a
211
+ readable body keeps whatever text it already had."""
212
+ try:
213
+ client = get_default_client()
214
+ resp = client.get(url, headers=_headers(config), timeout=DEFAULT_TIMEOUT_S)
215
+ _sec_raise_for_status(resp)
216
+ except Exception as exc: # noqa: BLE001 - a missing body is routine
217
+ logger.debug("edgar: no body for %s (%s)", url, type(exc).__name__)
218
+ return None
219
+ body = _strip_filing_html(resp.text)
220
+ if len(body) < _MIN_BODY_CHARS:
221
+ return None
222
+ return body[: config.max_body_chars]
223
+
224
+
225
+ _MIN_BODY_CHARS = 200
226
+
227
+
228
+ # --- Tier 3: a Form 4 as fields, not as text ---------------------------------
229
+
230
+
231
+ def _form4_field(xml: str, tag: str) -> Optional[str]:
232
+ """First value of ``tag``, unwrapping the <value> element Form 4 uses."""
233
+ match = re.search(rf"<{tag}>\s*(?:<value>)?\s*([^<]+)", xml)
234
+ return match.group(1).strip() if match else None
235
+
236
+
237
+ def _fetch_form4(cik: int, accession: str, config: EdgarFetchConfig
238
+ ) -> Optional[dict[str, Any]]:
239
+ """Parse one Form 4 into fields. None when it cannot be read.
240
+
241
+ Two requests: the accession's `index.json`, because the XML's filename is
242
+ not derivable (`wk-form4_1789160684.xml` on a live NVDA filing), then the
243
+ document itself. The XSL-rendered view named as `primaryDocument` is a
244
+ styled table — stripping it yields labels and numbers, not prose — so the
245
+ raw XML beside it is what carries meaning.
246
+ """
247
+ base = f"{ARCHIVES_BASE}/{cik}/{accession.replace('-', '')}"
248
+ try:
249
+ client = get_default_client()
250
+ listing = client.get(f"{base}/index.json", headers=_headers(config),
251
+ timeout=DEFAULT_TIMEOUT_S)
252
+ _sec_raise_for_status(listing)
253
+ items = (listing.json().get("directory") or {}).get("item") or []
254
+ names = [i.get("name", "") if isinstance(i, dict) else str(i) for i in items]
255
+ raw = [n for n in names if n.endswith(".xml") and "xsl" not in n.lower()]
256
+ if not raw:
257
+ return None
258
+ document = client.get(f"{base}/{raw[0]}", headers=_headers(config),
259
+ timeout=DEFAULT_TIMEOUT_S)
260
+ _sec_raise_for_status(document)
261
+ xml = document.text
262
+ except Exception as exc: # noqa: BLE001 - routine, never fails the fetch
263
+ logger.debug("edgar: could not read Form 4 %s (%s)", accession,
264
+ type(exc).__name__)
265
+ return None
266
+
267
+ code = (_form4_field(xml, "transactionCode") or "").upper() or None
268
+ shares = _form4_field(xml, "transactionShares")
269
+ price = _form4_field(xml, "transactionPricePerShare")
270
+ owner = _form4_field(xml, "rptOwnerName")
271
+ title = (_form4_field(xml, "officerTitle")
272
+ or ("director" if _form4_field(xml, "isDirector") in ("1", "true")
273
+ else None))
274
+
275
+ def number(value: Optional[str]) -> Optional[float]:
276
+ try:
277
+ return float(value) if value is not None else None
278
+ except ValueError:
279
+ return None
280
+
281
+ share_count, unit_price = number(shares), number(price)
282
+ parts = [owner or "An insider"]
283
+ if title:
284
+ parts.append(f"({title})")
285
+ parts.append(_TRANSACTION_CODES.get(code, "transaction") if code
286
+ else "transaction")
287
+ if share_count:
288
+ parts.append(f"of {share_count:,.0f} share"
289
+ + ("" if share_count == 1 else "s"))
290
+ if unit_price is not None:
291
+ parts.append(f"at ${unit_price:,.2f}" if unit_price else "at no cost")
292
+ held = number(_form4_field(xml, "sharesOwnedFollowingTransaction"))
293
+ summary = " ".join(parts) + "."
294
+ if held is not None:
295
+ summary += (f" Holding afterwards: {held:,.0f} share"
296
+ + ("" if held == 1 else "s") + ".")
297
+
298
+ value = (share_count * unit_price
299
+ if share_count is not None and unit_price is not None else None)
300
+ if value:
301
+ summary += f" Transaction value: ${value:,.0f}."
302
+
303
+ return {
304
+ "summary": summary,
305
+ "transaction_value": value,
306
+ "transaction_code": code,
307
+ "transaction_shares": share_count,
308
+ "transaction_price": unit_price,
309
+ "owner": owner,
310
+ "owner_title": title,
311
+ "shares_held_after": held,
312
+ }
313
+
314
+
92
315
  def _load_ticker_map(headers: dict[str, str]) -> dict[str, int]:
93
316
  """Fetch the SEC's ticker→CIK mapping (one request, covers all companies)."""
94
317
  client = get_default_client()
@@ -146,8 +369,9 @@ def _recent_filings_to_posts(cik: int, submissions: dict[str, Any],
146
369
  posts.append(Post(
147
370
  id=str(accession),
148
371
  source=SOURCE_NAME,
149
- title=f"{form}: {company}" if form else company,
150
- text=description or None,
372
+ title=f"Form {form}: {company}" if form else company,
373
+ text=_describe_filing(form, items[i] if i < len(items) else None,
374
+ description),
151
375
  url=_filing_url(cik, accession, document),
152
376
  author=company,
153
377
  created_at=(acceptance[i] if i < len(acceptance) else None)
@@ -186,7 +410,59 @@ def _fetch_company_filings(config: EdgarFetchConfig) -> list[Post]:
186
410
  timeout=DEFAULT_TIMEOUT_S)
187
411
  _sec_raise_for_status(resp)
188
412
  posts.extend(_recent_filings_to_posts(cik, resp.json(), config))
189
- return [p for p in posts if _in_window(p, config)]
413
+ kept = [p for p in posts if _in_window(p, config)]
414
+ return _enrich(kept, config) if config.fetch_bodies else kept
415
+
416
+
417
+ def _enrich(posts: list[Post], config: EdgarFetchConfig) -> list[Post]:
418
+ """Read each filing's substance, and drop Form 4s the caller filtered out.
419
+
420
+ Ordered so nothing is paid for twice: the window filter has already run, so
421
+ only filings a consumer will actually see are fetched, and a Form 4 excluded
422
+ by `transaction_codes` is dropped the moment its code is known.
423
+ """
424
+ wanted = {c.upper() for c in config.transaction_codes}
425
+ floor = config.min_transaction_value
426
+ out: list[Post] = []
427
+ for post in posts:
428
+ form = (post.metadata or {}).get("form") or ""
429
+ cik = (post.metadata or {}).get("cik")
430
+ accession = (post.metadata or {}).get("accession")
431
+
432
+ if form.upper().startswith("4") and cik and accession:
433
+ parsed = _fetch_form4(int(cik), str(accession), config)
434
+ if parsed:
435
+ if wanted and (parsed.get("transaction_code") or "") not in wanted:
436
+ continue
437
+ if floor:
438
+ value = parsed.get("transaction_value")
439
+ # An unpriced filing cannot clear a value floor. A grant is
440
+ # priced at $0 and is genuinely worth nothing to a reader
441
+ # watching for conviction; an unreadable one is dropped for
442
+ # the same reason the code filter drops it.
443
+ if not value or value < floor:
444
+ continue
445
+ post = post.model_copy(update={
446
+ "text": parsed["summary"],
447
+ "metadata": {**(post.metadata or {}),
448
+ **{k: v for k, v in parsed.items() if k != "summary"}},
449
+ })
450
+ elif wanted or floor:
451
+ # The filter could not be evaluated. Dropping is the honest
452
+ # choice: a caller asking for open-market purchases must not be
453
+ # handed an unreadable filing that might be a grant.
454
+ continue
455
+ elif form.upper() in _NARRATIVE_FORMS and post.url:
456
+ body = _fetch_filing_body(post.url, config)
457
+ if body:
458
+ # The item-code sentence still leads: it names what the filing
459
+ # is *about*, which the document's own preamble does not.
460
+ post = post.model_copy(update={
461
+ "text": f"{post.text} {body}".strip() if post.text else body,
462
+ "metadata": {**(post.metadata or {}), "body_source": "filing"},
463
+ })
464
+ out.append(post)
465
+ return out
190
466
 
191
467
 
192
468
  def _clean_display_name(name: str) -> str: