agentdata 0.3.26__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentdata/__init__.py +33 -0
- agentdata/__main__.py +4 -0
- agentdata/cache.py +115 -0
- agentdata/cli.py +173 -0
- agentdata/client.py +158 -0
- agentdata/company/__init__.py +3 -0
- agentdata/company/cf.py +145 -0
- agentdata/company/ch_zefix.py +100 -0
- agentdata/company/cli.py +243 -0
- agentdata/company/co_rues.py +72 -0
- agentdata/company/edgar_cik.py +84 -0
- agentdata/company/edgar_fts.py +101 -0
- agentdata/company/faces.py +746 -0
- agentdata/company/fdic_bankfind.py +107 -0
- agentdata/company/fi_ytj.py +89 -0
- agentdata/company/fr_recherche.py +124 -0
- agentdata/company/fts.py +121 -0
- agentdata/company/gleif.py +178 -0
- agentdata/company/kr_hometax.py +112 -0
- agentdata/company/no_brreg.py +124 -0
- agentdata/company/ror.py +105 -0
- agentdata/company/ted.py +117 -0
- agentdata/company/usaspending.py +124 -0
- agentdata/company/vies.py +66 -0
- agentdata/daterange.py +35 -0
- agentdata/economy/__init__.py +10 -0
- agentdata/economy/bis.py +132 -0
- agentdata/economy/cli.py +199 -0
- agentdata/economy/ecb.py +117 -0
- agentdata/economy/eurostat.py +163 -0
- agentdata/economy/faces.py +573 -0
- agentdata/economy/fred.py +92 -0
- agentdata/economy/ilo_stat.py +99 -0
- agentdata/economy/imf_sdmx.py +133 -0
- agentdata/economy/un_comtrade.py +199 -0
- agentdata/economy/unsd_sdg.py +154 -0
- agentdata/economy/wb.py +208 -0
- agentdata/economy/who_xmart.py +177 -0
- agentdata/entries.py +30 -0
- agentdata/errors.py +38 -0
- agentdata/esg/__init__.py +5 -0
- agentdata/esg/cli.py +180 -0
- agentdata/esg/climatetrace.py +260 -0
- agentdata/esg/eiti.py +187 -0
- agentdata/esg/epa_frs.py +138 -0
- agentdata/esg/euasl.py +75 -0
- agentdata/esg/eutl.py +66 -0
- agentdata/esg/faces.py +498 -0
- agentdata/esg/paris_mou.py +240 -0
- agentdata/esg/thetis_mrv.py +67 -0
- agentdata/healthcare/__init__.py +7 -0
- agentdata/healthcare/au_tga.py +142 -0
- agentdata/healthcare/ca_dpd.py +123 -0
- agentdata/healthcare/ca_mdall.py +155 -0
- agentdata/healthcare/cli.py +375 -0
- agentdata/healthcare/ctgov.py +101 -0
- agentdata/healthcare/dailymed.py +102 -0
- agentdata/healthcare/epmc.py +108 -0
- agentdata/healthcare/eudamed.py +124 -0
- agentdata/healthcare/faces.py +811 -0
- agentdata/healthcare/fda_crl.py +86 -0
- agentdata/healthcare/fda_devices.py +199 -0
- agentdata/healthcare/fda_drugs.py +156 -0
- agentdata/healthcare/fda_recalls.py +137 -0
- agentdata/healthcare/gb_mhra.py +137 -0
- agentdata/healthcare/gudid.py +84 -0
- agentdata/healthcare/mesh.py +89 -0
- agentdata/healthcare/rasff.py +164 -0
- agentdata/healthcare/rxnav.py +67 -0
- agentdata/healthcare/safety_gate.py +175 -0
- agentdata/healthcare/shortages.py +107 -0
- agentdata/legal/__init__.py +4 -0
- agentdata/legal/afdb.py +93 -0
- agentdata/legal/ca_sema.py +143 -0
- agentdata/legal/cli.py +312 -0
- agentdata/legal/co_sic.py +70 -0
- agentdata/legal/courtlistener.py +77 -0
- agentdata/legal/csl.py +139 -0
- agentdata/legal/dgcomp.py +125 -0
- agentdata/legal/ecfr.py +129 -0
- agentdata/legal/faces.py +700 -0
- agentdata/legal/federalregister.py +188 -0
- agentdata/legal/gb_cma.py +117 -0
- agentdata/legal/gb_tna_caselaw.py +245 -0
- agentdata/legal/idb.py +85 -0
- agentdata/legal/lda.py +147 -0
- agentdata/legal/sipri.py +178 -0
- agentdata/legal/ukec.py +148 -0
- agentdata/legal/wb.py +92 -0
- agentdata/procurement/__init__.py +5 -0
- agentdata/procurement/br_pncp.py +126 -0
- agentdata/procurement/cli.py +155 -0
- agentdata/procurement/co_secop.py +88 -0
- agentdata/procurement/faces.py +239 -0
- agentdata/procurement/ny_open_data.py +82 -0
- agentdata/procurement/ungm.py +137 -0
- agentdata/procurement/wb_proc.py +84 -0
- agentdata/research/__init__.py +4 -0
- agentdata/research/biorxiv.py +66 -0
- agentdata/research/cli.py +129 -0
- agentdata/research/crossref.py +121 -0
- agentdata/research/faces.py +349 -0
- agentdata/research/gb_gtr.py +58 -0
- agentdata/research/icite.py +52 -0
- agentdata/research/nih_reporter.py +143 -0
- agentdata/research/nsf.py +68 -0
- agentdata/research/openalex.py +145 -0
- agentdata/research/opencitations.py +50 -0
- agentdata/security/__init__.py +3 -0
- agentdata/security/cisa_kev.py +42 -0
- agentdata/security/cli.py +37 -0
- agentdata/security/faces.py +91 -0
- agentdata/security/osv.py +72 -0
- agentdata/transport.py +541 -0
- agentdata-0.3.26.dist-info/METADATA +266 -0
- agentdata-0.3.26.dist-info/RECORD +120 -0
- agentdata-0.3.26.dist-info/WHEEL +5 -0
- agentdata-0.3.26.dist-info/entry_points.txt +10 -0
- agentdata-0.3.26.dist-info/licenses/LICENSE +201 -0
- agentdata-0.3.26.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""ch_zefix — Swiss Zefix commercial register (Zentraler Firmenindex) provider (same-origin REST front door; keyless).
|
|
2
|
+
|
|
3
|
+
2026-10-29 probed shape: POST /firm/search.json with JSON body
|
|
4
|
+
{languageKey, maxEntries, offset, name} → {list[], hasMoreResults, offset, maxEntries, maxOffset};
|
|
5
|
+
name is a contains-match (nestle hits Nestler), pagination = offset+hasMoreResults (maxOffset ~540k = upstream offset wall);
|
|
6
|
+
single record GET /firm/{ehraid}.json returns the full record: oldNames[] name history + hasTakenOver/wasTakenOverBy M&A edges
|
|
7
|
+
(rows carry ehraid/uid/chid cross-references — closed-loop linking inside the Swiss registry system) + purpose full text.
|
|
8
|
+
Three keys surfaced directly: ehraid (master id) / uid (CHE business id) / chid (register id). Officers/directors are not in the public face.
|
|
9
|
+
TTL: search 7 days (current-value query) / firm 30 days (records change slowly); politeness tier 0.5 rps (no published quota).
|
|
10
|
+
"""
|
|
11
|
+
from ..errors import SdkError
|
|
12
|
+
|
|
13
|
+
provider = {
|
|
14
|
+
"id": "ch_zefix",
|
|
15
|
+
"baseUrl": "https://www.zefix.ch/ZefixREST/api/v1",
|
|
16
|
+
"rate": {"rps": 0.5, "burst": 2},
|
|
17
|
+
"ops": {
|
|
18
|
+
"search": {"path": "/firm/search.json", "method": "POST", "idempotent": True}, # params: name/maxEntries/offset (all in the JSON body)
|
|
19
|
+
"firm": {"path": "/firm/{ehraid}.json", "notFound": [404]}, # params: ehraid (path template)
|
|
20
|
+
},
|
|
21
|
+
"ttl": {"search": 7 * 86400, "firm": 30 * 86400},
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def search_firms(name, *, max_entries=100, offset=0, cache=None, no_cache=False):
|
|
26
|
+
"""→ {firms[], has_more, max_offset, params, _meta}; name → registry candidates (contains-match, not authoritative normalization).
|
|
27
|
+
maxEntries/offset are honored and echoed upstream (probed: maxEntries=3 returned 3 rows); 1-100 is our politeness cap, not an upstream wall."""
|
|
28
|
+
from ..transport import call_provider
|
|
29
|
+
|
|
30
|
+
q = str(name or "").strip()
|
|
31
|
+
if not q:
|
|
32
|
+
raise SdkError("usage_error", provider="ch_zefix", detail="search requires name (company name, contains-match)")
|
|
33
|
+
n = min(max(1, int(max_entries or 100)), 100)
|
|
34
|
+
off = max(0, int(offset or 0))
|
|
35
|
+
r = call_provider(provider, "search",
|
|
36
|
+
{"name": q, "maxEntries": n, "offset": off, "languageKey": "en"},
|
|
37
|
+
cache=cache, no_cache=no_cache)
|
|
38
|
+
d = r["data"] or {}
|
|
39
|
+
rows = [{
|
|
40
|
+
"ehraid": f.get("ehraid"),
|
|
41
|
+
"uid": f.get("uid"),
|
|
42
|
+
"uid_formatted": f.get("uidFormatted"),
|
|
43
|
+
"chid": f.get("chid"),
|
|
44
|
+
"chid_formatted": f.get("chidFormatted"),
|
|
45
|
+
"name": f.get("name"),
|
|
46
|
+
"status": f.get("status"), # EXISTIEREND / GELOESCHT…
|
|
47
|
+
"legal_seat": f.get("legalSeat"),
|
|
48
|
+
"legal_form_id": f.get("legalFormId"),
|
|
49
|
+
"register_office_id": f.get("registerOfficeId"),
|
|
50
|
+
"shab_date": f.get("shabDate"), # latest gazette date
|
|
51
|
+
"delete_date": f.get("deleteDate"), # non-null on GELOESCHT rows
|
|
52
|
+
"cantonal_excerpt_web": f.get("cantonalExcerptWeb"), # cantonal register excerpt page (human-readable)
|
|
53
|
+
} for f in d.get("list") or []]
|
|
54
|
+
return {"firms": rows, "has_more": bool(d.get("hasMoreResults")),
|
|
55
|
+
"max_offset": d.get("maxOffset"),
|
|
56
|
+
"params": {"name": q, "maxEntries": n, "offset": off, "languageKey": "en"},
|
|
57
|
+
"_meta": r}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def get_firm(ehraid, *, cache=None, no_cache=False):
|
|
61
|
+
"""→ {firm, _meta}; full record card by ehraid (purpose/oldNames/M&A edges).
|
|
62
|
+
hasTakenOver = predecessors this firm absorbed / wasTakenOverBy = the successor that absorbed this firm —
|
|
63
|
+
rows carry ehraid/uid/chid so ch_zefix_firm/ch_zefix_firm_search can be called next for a closed loop."""
|
|
64
|
+
from ..transport import call_provider
|
|
65
|
+
|
|
66
|
+
v = str(ehraid or "").strip()
|
|
67
|
+
if not v.isdigit():
|
|
68
|
+
raise SdkError("usage_error", provider="ch_zefix",
|
|
69
|
+
detail=f"ehraid must be numeric (master id, surfaced by search rows), got \"{ehraid}\"")
|
|
70
|
+
r = call_provider(provider, "firm", {"ehraid": v}, cache=cache, no_cache=no_cache)
|
|
71
|
+
d = r["data"] or {}
|
|
72
|
+
firm = {
|
|
73
|
+
"ehraid": d.get("ehraid"),
|
|
74
|
+
"uid": d.get("uid"),
|
|
75
|
+
"uid_formatted": d.get("uidFormatted"),
|
|
76
|
+
"chid": d.get("chid"),
|
|
77
|
+
"chid_formatted": d.get("chidFormatted"),
|
|
78
|
+
"name": d.get("name"),
|
|
79
|
+
"translation": d.get("translation"),
|
|
80
|
+
"status": d.get("status"),
|
|
81
|
+
"legal_seat": d.get("legalSeat"),
|
|
82
|
+
"legal_form_id": d.get("legalFormId"),
|
|
83
|
+
"register_office_id": d.get("registerOfficeId"),
|
|
84
|
+
"shab_date": d.get("shabDate"),
|
|
85
|
+
"delete_date": d.get("deleteDate"),
|
|
86
|
+
"cantonal_excerpt_web": d.get("cantonalExcerptWeb"),
|
|
87
|
+
"purpose": d.get("purpose"), # full statement of purpose
|
|
88
|
+
"address": d.get("address"), # registered address (upstream shape passed through)
|
|
89
|
+
"old_names": [{"name": o.get("name"), "sequence_nr": o.get("sequenceNr"),
|
|
90
|
+
"translation": o.get("translation")} for o in d.get("oldNames") or []],
|
|
91
|
+
"has_taken_over": _edge(d.get("hasTakenOver")), # predecessors: merged into this firm
|
|
92
|
+
"taken_over_by": _edge(d.get("wasTakenOverBy")), # successors: absorbed this firm
|
|
93
|
+
}
|
|
94
|
+
return {"firm": firm, "_meta": r}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _edge(rows):
|
|
98
|
+
"""M&A edge row projection: name + three keys (ehraid/uid/chid) — keys feed the next hop, the rest is dropped."""
|
|
99
|
+
return [{"name": e.get("name"), "ehraid": e.get("ehraid"),
|
|
100
|
+
"uid": e.get("uid"), "chid": e.get("chid")} for e in rows or []]
|
agentdata/company/cli.py
ADDED
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
"""company cli — domain command face registration (split per domain in 0.2.6, moved in from the root cli.py; purely flat since the 0.3.0 verb-layer + sugar removal).
|
|
2
|
+
|
|
3
|
+
register(v, g): registers flat commands on the domain subparser (g = the root _leaf_globals factory,
|
|
4
|
+
passed in as a parameter to avoid a cli ↔ domain-cli cycle); FLAT = the flat dispatch table, merged by the root cli for main to use.
|
|
5
|
+
"""
|
|
6
|
+
import argparse
|
|
7
|
+
|
|
8
|
+
def register(v, g):
|
|
9
|
+
# —— Flat panel commands (company main face: source-qualified names + upstream params under original names + our defaults; the only face family since 0.3.0) ——
|
|
10
|
+
f1 = v.add_parser("ch_zefix_firm", parents=[g()],
|
|
11
|
+
help="Swiss firm full profile by ehraid (Zefix /firm/{ehraid}.json)",
|
|
12
|
+
description="Zefix single record: purpose full text, oldNames history, hasTakenOver/takenOverBy M&A edges with ehraid/uid/chid cross-references (precedessor/successor)")
|
|
13
|
+
f1.add_argument("--ehraid", required=True, help="Registry master id (upstream path param: ehraid; from search rows)")
|
|
14
|
+
# —— 0.3.3 European country registry realtime faces (CH Zefix / NO Brreg / FI YTJ) ——
|
|
15
|
+
f2 = v.add_parser("ch_zefix_firm_search", parents=[g()],
|
|
16
|
+
help="Swiss commercial register name search (Zefix POST search.json)",
|
|
17
|
+
description="Zefix firm search: name contains-match (hits historical names too); three registry keys (ehraid/uid/chid) per row; maxEntries/offset honored upstream (echoed back)")
|
|
18
|
+
f2.add_argument("--name", required=True, help="Company name (upstream param: name; contains-match, non-authoritative)")
|
|
19
|
+
f2.add_argument("--max-entries", dest="max_entries", type=int, default=100,
|
|
20
|
+
help="Rows per page (upstream param: maxEntries; honored upstream; 1-100 our politeness cap)")
|
|
21
|
+
f2.add_argument("--offset", type=int, default=0, help="Offset (upstream param: offset; walk with --offset, cap ~540k upstream maxOffset)")
|
|
22
|
+
# —— 0.3.11 CO RUES + KR hometax dual-country batch ——
|
|
23
|
+
f_co = v.add_parser("co_rues_companies", parents=[g()],
|
|
24
|
+
help="Colombian business registry master (RUES via datos.gov.co Socrata; --where SoQL)",
|
|
25
|
+
description="RUES registro mercantil master, 9.4M rows: matricula registry key x NIT tax id x CIIU industry x legal form x representante_legal name edge; base filter keeps NIT-identified (tax entity) rows only — persona natural rows whose id columns are cedulas stay out of the face, and the RL cedula-number column is stripped (names kept)")
|
|
26
|
+
f_co.add_argument("--where", default=None,
|
|
27
|
+
help="Raw SoQL filter (upstream param: $where; merged with base filter clase_identificacion='NIT' by AND)")
|
|
28
|
+
f_co.add_argument("--limit", type=int, default=20, help="Rows per page (upstream param: $limit; 1-500 our politeness cap)")
|
|
29
|
+
f_co.add_argument("--offset", type=int, default=0, help="Row offset (upstream param: $offset; >=0)")
|
|
30
|
+
f_co.add_argument("--order", default=None, help="Order clause (upstream param: $order; column [ASC|DESC], no NULLS syntax)")
|
|
31
|
+
f3 = v.add_parser("fdic_institutions", parents=[g()],
|
|
32
|
+
help="US insured-bank master record (FDIC BankFind; --search/--filters raw channels)",
|
|
33
|
+
description="FDIC institutions: ES search × structured filters are combinable raw channels (search 'NAME:\"…\"' / filters 'CERT:n'); no upstream limit wall (10,000 verified)")
|
|
34
|
+
f3.add_argument("--search", default=None, help="ES full-text (upstream param: search; e.g. 'NAME:\"First National\" AND ST:TX')")
|
|
35
|
+
f3.add_argument("--filters", default=None, help="Structured filter (upstream param: filters; e.g. 'CERT:628' or 'ST:TX AND CITY:Dallas')")
|
|
36
|
+
f3.add_argument("--limit", type=int, default=5, help="Rows to return (upstream param: limit; no wall verified ≤10000, default 5)")
|
|
37
|
+
f4 = v.add_parser("fi_ytj_companies", parents=[g()],
|
|
38
|
+
help="Finnish business register records (PRH YTJ v3; --name search XOR --business-id direct)",
|
|
39
|
+
description="YTJ v3 companies: name contains-match hits historical/branch names; businessId direct lookup; pagination params are silently ignored upstream — fixed ~63-row pages, totalResults is the real total")
|
|
40
|
+
f4.add_argument("--name", default=None, help="Company name (upstream param: name; contains-match incl. historical names)")
|
|
41
|
+
f4.add_argument("--business-id", dest="business_id", default=None,
|
|
42
|
+
help="Business id, 0194099-3 form (upstream param: businessId; spaces ok)")
|
|
43
|
+
# —— 0.3.4 FR annuaire realtime search face (recherche-entreprises) ——
|
|
44
|
+
# —— 0.3.4 FR annuaire realtime search face (recherche-entreprises) ——
|
|
45
|
+
f5 = v.add_parser("fr_recherche_search", parents=[g()],
|
|
46
|
+
help="French registry aggregate search (annuaire-entreprises; --q name/officer-name/SIREN)",
|
|
47
|
+
description="etalab recherche-entreprises: one q for company name, officer name (cross-company mandate search) or SIREN direct lookup; rich rows with dirigeants (person + corporate slots), finances, tva; etat_administratif A=active/C=ceased true filter")
|
|
48
|
+
f5.add_argument("--q", required=True,
|
|
49
|
+
help="Query: company name / officer name / SIREN (upstream param: q; name search hits historical names, officer names give cross-company mandates)")
|
|
50
|
+
f5.add_argument("--page", type=int, default=None, help="Page, 1-based (upstream param: page; walk while page<=total_pages)")
|
|
51
|
+
f5.add_argument("--per-page", dest="per_page", type=int, default=10,
|
|
52
|
+
help="Rows per page (upstream param: per_page; hard wall 1-25 upstream-verified, default 10)")
|
|
53
|
+
f5.add_argument("--etat-administratif", dest="etat_administratif", default=None,
|
|
54
|
+
help="Lifecycle filter (upstream param: etat_administratif; A=active, C=ceased — true filter, ceased rows carry date_fermeture)")
|
|
55
|
+
f6 = v.add_parser("gb_contracts_finder_published_notices", parents=[g()],
|
|
56
|
+
help="UK below-threshold contracts (Contracts Finder OCDS; bare dates legal here)",
|
|
57
|
+
description="Sub-threshold UK contracts: publishedFrom/publishedTo are bare dates (opposite of FTS); no window → our rolling 7-day default (upstream returns empty without one); --stages filters tender|award (upstream 400 self-validates); --name = client-side contains filter")
|
|
58
|
+
f6.add_argument("--name", default=None, help="Company name — client-side contains filter (no upstream name-search param exists)")
|
|
59
|
+
f6.add_argument("--stages", default=None,
|
|
60
|
+
help="OCDS stage filter (upstream param: stages; tender|award — invalid → upstream 400 self-describing; absent = mixed feed; 0.2.9 audit)")
|
|
61
|
+
f6.add_argument("--published-from", dest="published_from", default=None,
|
|
62
|
+
help="Window start YYYY-MM-DD, bare (upstream param: publishedFrom; default 7 days back)")
|
|
63
|
+
f6.add_argument("--published-to", dest="published_to", default=None,
|
|
64
|
+
help="Window end YYYY-MM-DD, bare (upstream param: publishedTo; default today)")
|
|
65
|
+
f6.add_argument("--limit", type=int, default=10, help="Rows (upstream param: limit; 1-100; with --name = matches cap, fetch stays 100)")
|
|
66
|
+
f6.add_argument("--cursor", default=None, help="Pagination cursor (upstream param: cursor; pass the same window when walking)")
|
|
67
|
+
f7 = v.add_parser("gb_fts_release_packages", parents=[g()],
|
|
68
|
+
help="UK above-threshold tenders (Find a Tender OCDS; upstream has no name-search param)",
|
|
69
|
+
description="WTO-line UK procurement releases: stages/limit/cursor/updatedFrom/updatedTo upstream-verbatim; --name is a client-side contains filter (forces fetch of the full 100-row page, matches buyer or suppliers)")
|
|
70
|
+
f7.add_argument("--name", default=None, help="Company name — client-side contains filter (no upstream name-search param exists)")
|
|
71
|
+
f7.add_argument("--stages", default="award", help="OCDS stage (upstream param: stages; award|tender, default award)")
|
|
72
|
+
f7.add_argument("--limit", type=int, default=10, help="Rows (upstream param: limit; 1-100; with --name = matches cap, fetch stays 100)")
|
|
73
|
+
f7.add_argument("--cursor", default=None, help="Pagination cursor (upstream param: cursor; feed next_cursor)")
|
|
74
|
+
f7.add_argument("--updated-from", dest="updated_from", default=None,
|
|
75
|
+
help="Window start YYYY-MM-DD or full ISO datetime (upstream param: updatedFrom; bare date gets T00:00:00Z)")
|
|
76
|
+
f7.add_argument("--updated-to", dest="updated_to", default=None,
|
|
77
|
+
help="Window end (upstream param: updatedTo; bare date gets T23:59:59Z)")
|
|
78
|
+
f8 = v.add_parser("gleif_direct_parent", parents=[g()],
|
|
79
|
+
help="LEI→direct parent (GLEIF dedicated endpoint)",
|
|
80
|
+
description="Direct parent card; 404 = honestly no parent (not an error)")
|
|
81
|
+
f8.add_argument("--lei", required=True, help="LEI, 20 chars ISO 17442 (upstream path param: lei)")
|
|
82
|
+
f9 = v.add_parser("gleif_lei_records", parents=[g()],
|
|
83
|
+
help="LEI→entity profile (GLEIF single-record face)",
|
|
84
|
+
description="Legal Entity Identifier → entity card (legal name, status, jurisdiction, registration; "
|
|
85
|
+
"+ successor LEIs (M&A edge), expiration, legal form, BIC/MIC)")
|
|
86
|
+
f9.add_argument("--lei", required=True, help="LEI, 20 chars ISO 17442 (upstream path param: lei)")
|
|
87
|
+
f10 = v.add_parser("gleif_lei_records_search", parents=[g()],
|
|
88
|
+
help="Company name→LEI candidates (GLEIF legalName contains; --filter-entity-jurisdiction/--filter-entity-status narrow)",
|
|
89
|
+
description="Name → LEI candidates (non-authoritative; verify with gleif_lei_records before persisting); "
|
|
90
|
+
"jurisdiction/status filters and page[number] walking upstream-verbatim; rows carry registered_as (cross-registry join key)")
|
|
91
|
+
f10.add_argument("--filter-entity-legal-name", dest="filter_entity_legal_name", required=True,
|
|
92
|
+
help="Company name (upstream param: filter[entity.legalName], bracket-transcribed)")
|
|
93
|
+
f10.add_argument("--filter-entity-jurisdiction", dest="filter_entity_jurisdiction", default=None,
|
|
94
|
+
help="Jurisdiction filter (upstream param: filter[entity.jurisdiction]; ISO 3166 country or region code, e.g. CH or US-MO — probed US-MO→1 hit; jurisdictionCountry is whitelist-rejected upstream, not mirrored)")
|
|
95
|
+
f10.add_argument("--filter-entity-status", dest="filter_entity_status", default=None,
|
|
96
|
+
help="Entity status filter (upstream param: filter[entity.status]; ACTIVE/INACTIVE/NULL — entity lifecycle, not registration.status ISSUED/LAPSED/RETIRED)")
|
|
97
|
+
f10.add_argument("--page-number", dest="page_number", type=int, default=None,
|
|
98
|
+
help="Page, 1-based (upstream param: page[number]; walk while page ≤ result page.lastPage)")
|
|
99
|
+
f10.add_argument("--page-size", dest="page_size", type=int, default=10,
|
|
100
|
+
help="Candidates cap (upstream param: page[size]; 1-200 hard wall, default 10)")
|
|
101
|
+
f11 = v.add_parser("gleif_ultimate_parent", parents=[g()],
|
|
102
|
+
help="LEI→ultimate parent (GLEIF dedicated endpoint)",
|
|
103
|
+
description="Ultimate parent card (top of the control chain); 404 = honestly no parent (not an error)")
|
|
104
|
+
f11.add_argument("--lei", required=True, help="LEI, 20 chars ISO 17442 (upstream path param: lei)")
|
|
105
|
+
f_kr = v.add_parser("kr_hometax_biz_status", parents=[g()],
|
|
106
|
+
help="Korean biz registration status, single lookup (hometax 휴폐업/과세유형; --biz-no)",
|
|
107
|
+
description="NTS hometax anonymous single-BRN status: four states all legal answers — active (일반/간이/과세특례 taxpayer), closed (ISO closure date extracted), unregistered (result=S, not an error), error; txprNm is always null upstream (no owner-name leak)")
|
|
108
|
+
f_kr.add_argument("--biz-no", dest="biz_no", required=True,
|
|
109
|
+
help="10-digit business registration number (사업자등록번호; 3-2-5 hyphens optional, e.g. 124-81-00998)")
|
|
110
|
+
f12 = v.add_parser("no_brreg_enheter", parents=[g()],
|
|
111
|
+
help="Norwegian registry master record (Enhetsregisteret; --navn search XOR --organisasjonsnummer single)",
|
|
112
|
+
description="Brønnøysund Enhetsregisteret: navn search with HAL pagination (page/size, 0-based) or orgnr single record; navn is the Norwegian upstream param (?name= 400s)")
|
|
113
|
+
f12.add_argument("--navn", default=None, help="Name search (upstream param: navn — Norwegian, not 'name')")
|
|
114
|
+
f12.add_argument("--organisasjonsnummer", dest="organisasjonsnummer", default=None,
|
|
115
|
+
help="9-digit org number (upstream path param; spaces ok)")
|
|
116
|
+
f12.add_argument("--page", type=int, default=0, help="Page, 0-based (upstream param: page; search face only)")
|
|
117
|
+
f12.add_argument("--size", type=int, default=20, help="Rows per page (upstream param: size; 1-100 our politeness cap)")
|
|
118
|
+
f13 = v.add_parser("no_brreg_roller", parents=[g()],
|
|
119
|
+
help="Norwegian registry roles by org number (Enhetsregisteret /roller)",
|
|
120
|
+
description="Role groups (DAGL CEO / STYR board / REVI auditor): persons carry name+birth date (no national id in the public face), corporate roles (auditors) carry org number; avregistrert=true = retired")
|
|
121
|
+
f13.add_argument("--organisasjonsnummer", dest="organisasjonsnummer", required=True,
|
|
122
|
+
help="9-digit org number (upstream path param; spaces ok)")
|
|
123
|
+
f14 = v.add_parser("ror_organizations", parents=[g()],
|
|
124
|
+
help="Research-org record (ROR v2; --query search XOR --ror-id single record)",
|
|
125
|
+
description="Research Organization Registry: query → candidates (fixed 20/page, --page walks), or ror_id → authoritative single record")
|
|
126
|
+
f14.add_argument("--query", default=None, help="Org name search (upstream param: query)")
|
|
127
|
+
f14.add_argument("--ror-id", dest="ror_id", default=None, help="ROR id, 0+8 chars (upstream path: /organizations/{id}; bare or full URL)")
|
|
128
|
+
f14.add_argument("--page", type=int, default=None, help="Page number for --query (upstream param: page; fixed 20 rows/page)")
|
|
129
|
+
f14.add_argument("--all-status", dest="all_status", action="store_true", default=None,
|
|
130
|
+
help="Include inactive/withdrawn orgs (upstream param: all_status; default upstream=false hides them — 0.2.9 audit)")
|
|
131
|
+
f15 = v.add_parser("sec_fulltext_search", parents=[g()],
|
|
132
|
+
help="SEC filing full-text search 2001+ (EDGAR FTS)",
|
|
133
|
+
description="EDGAR full-text search: q/forms/startdt/enddt upstream-verbatim; a single date end is auto-completed (silently ignored upstream), dateRange=custom always two-ended; total_relation gte = capped at 10k")
|
|
134
|
+
f15.add_argument("--q", required=True, help="Query term (upstream param: q; EDGAR FTS syntax)")
|
|
135
|
+
f15.add_argument("--forms", default=None, help="Form-type filter (upstream param: forms; e.g. 8-K,10-K)")
|
|
136
|
+
f15.add_argument("--startdt", default=None, help="Start date YYYY-MM-DD (upstream param: startdt; single end auto-completed to 2001-01-01)")
|
|
137
|
+
f15.add_argument("--enddt", default=None, help="End date YYYY-MM-DD (upstream param: enddt; single end auto-completed to today)")
|
|
138
|
+
f15.add_argument("--limit", type=int, default=10, help="Rows to return (client-side slice of the 100-hit page; ≤100)")
|
|
139
|
+
f16 = v.add_parser("sec_submissions", parents=[g()],
|
|
140
|
+
help="SEC CIK→registrant card (data.sec.gov submissions)",
|
|
141
|
+
description="Registrant identity card: legal/former names, tickers·exchanges, EIN, SIC, filing totals (accepts bare/zero-padded/CIK-prefixed input)")
|
|
142
|
+
f16.add_argument("--cik", required=True, help="CIK, 1-10 digits (upstream path param: CIK{cik10}.json)")
|
|
143
|
+
f17 = v.add_parser("ted_notices_search", parents=[g()],
|
|
144
|
+
help="EU procurement notices (TED v3 expert search; --query expert syntax)",
|
|
145
|
+
description="Tenders Electronic Daily: expert-syntax query via --query (e.g. 'organisation-name-buyer=Siemens* AND organisation-country-buyer=DEU' / classification-cpv=33100000); limit 1-250 is an upstream hard wall")
|
|
146
|
+
f17.add_argument("--query", required=True, help="Expert syntax (upstream param: query; trailing wildcard = prefix match; buyer facets organisation-name-buyer / organisation-country-buyer, CPV via classification-cpv)")
|
|
147
|
+
f17.add_argument("--fields", default=None, help="Comma list of eForms fields (upstream param: fields; wrong names 400 with the full 1,833-field dump)")
|
|
148
|
+
f17.add_argument("--limit", type=int, default=10, help="Notices per page (upstream param: limit; 1-250 hard wall, default 10)")
|
|
149
|
+
f17.add_argument("--page", type=int, default=1, help="Page number (upstream param: page)")
|
|
150
|
+
f18 = v.add_parser("usaspending_spending_by_award", parents=[g()],
|
|
151
|
+
help="US federal awards by recipient (USAspending spending_by_award POST)",
|
|
152
|
+
description="Federal contracts/grants: recipient name (array upstream, scalar auto-wrapped); award-type codes, time window (single end auto-completed), fields all upstream-verbatim")
|
|
153
|
+
f18.add_argument("--recipient-search-text", dest="recipient_search_text", required=True,
|
|
154
|
+
help="Recipient name (upstream param: recipient_search_text; array semantics — scalar auto-wrapped)")
|
|
155
|
+
f18.add_argument("--award-type-codes", dest="award_type_codes", default=None,
|
|
156
|
+
help="Comma list (upstream param: award_type_codes; default A,B,C,D contracts — grants 02/03 etc.)")
|
|
157
|
+
f18.add_argument("--start-date", dest="start_date", default=None,
|
|
158
|
+
help="Window start YYYY-MM-DD (upstream param: time_period[].start_date; single end auto-completed)")
|
|
159
|
+
f18.add_argument("--end-date", dest="end_date", default=None,
|
|
160
|
+
help="Window end YYYY-MM-DD (upstream param: time_period[].end_date; single end auto-completed)")
|
|
161
|
+
f18.add_argument("--fields", default=None, help="Comma list of columns (upstream param: fields; default 6 columns)")
|
|
162
|
+
f18.add_argument("--limit", type=int, default=10, help="Rows per page (upstream param: limit; 1-100, default 10)")
|
|
163
|
+
f18.add_argument("--page", type=int, default=1, help="Page number (upstream param: page; walk while has_more)")
|
|
164
|
+
f18.add_argument("--subawards", action=argparse.BooleanOptionalAction, default=False,
|
|
165
|
+
help="Include subawards (upstream param: subawards; default off)")
|
|
166
|
+
f19 = v.add_parser("vies_vat", parents=[g()],
|
|
167
|
+
help="EU VAT number check (VIES; INVALID is a valid answer)",
|
|
168
|
+
description="Official EU VAT validation: member-state prefixed number; INVALID → ok:true with valid=false; '---' masking → null (not missing data)")
|
|
169
|
+
f19.add_argument("--vat", required=True, help="VAT with member-state prefix (upstream path: /ms/{cc}/vat/{number}; e.g. DE811907980)")
|
|
170
|
+
# —— 0.3.3 European country registry realtime faces (CH Zefix / NO Brreg / FI YTJ) ——
|
|
171
|
+
|
|
172
|
+
FLAT = {
|
|
173
|
+
# company 0.2.0 flat panel (upstream params under original names + our defaults; the lambda only ferries names across namespaces)
|
|
174
|
+
"ch_zefix_firm": lambda a: {"ehraid": a.ehraid},
|
|
175
|
+
# 0.3.3 European country registry realtime faces (five)
|
|
176
|
+
"ch_zefix_firm_search": lambda a: {"name": a.name, "max_entries": a.max_entries, "offset": a.offset},
|
|
177
|
+
# 0.3.11 CO RUES + KR hometax dual-country batch
|
|
178
|
+
"co_rues_companies": lambda a: {"where": a.where, "limit": a.limit, "offset": a.offset,
|
|
179
|
+
"order": a.order},
|
|
180
|
+
"fdic_institutions": lambda a: {"search": a.search, "filters": a.filters,
|
|
181
|
+
"limit": a.limit},
|
|
182
|
+
"fi_ytj_companies": lambda a: {"name": a.name, "business_id": a.business_id},
|
|
183
|
+
"fr_recherche_search": lambda a: {"q": a.q, "page": a.page, "per_page": a.per_page,
|
|
184
|
+
"etat_administratif": a.etat_administratif},
|
|
185
|
+
"gb_contracts_finder_published_notices": lambda a: {"name": a.name, "published_from": a.published_from,
|
|
186
|
+
"published_to": a.published_to, "stages": a.stages,
|
|
187
|
+
"limit": a.limit, "cursor": a.cursor},
|
|
188
|
+
"gb_fts_release_packages": lambda a: {"name": a.name, "stages": a.stages, "limit": a.limit,
|
|
189
|
+
"cursor": a.cursor, "updated_from": a.updated_from,
|
|
190
|
+
"updated_to": a.updated_to},
|
|
191
|
+
"gleif_direct_parent": lambda a: {"lei": a.lei},
|
|
192
|
+
"gleif_lei_records": lambda a: {"lei": a.lei},
|
|
193
|
+
"gleif_lei_records_search": lambda a: {"filter_entity_legal_name": a.filter_entity_legal_name,
|
|
194
|
+
"page_size": a.page_size,
|
|
195
|
+
"filter_entity_jurisdiction": a.filter_entity_jurisdiction,
|
|
196
|
+
"filter_entity_status": a.filter_entity_status,
|
|
197
|
+
"page_number": a.page_number},
|
|
198
|
+
"gleif_ultimate_parent": lambda a: {"lei": a.lei},
|
|
199
|
+
"kr_hometax_biz_status": lambda a: {"biz_no": a.biz_no},
|
|
200
|
+
"no_brreg_enheter": lambda a: {"navn": a.navn, "organisasjonsnummer": a.organisasjonsnummer,
|
|
201
|
+
"page": a.page, "size": a.size},
|
|
202
|
+
"no_brreg_roller": lambda a: {"organisasjonsnummer": a.organisasjonsnummer},
|
|
203
|
+
"ror_organizations": lambda a: {"query": a.query, "ror_id": a.ror_id, "page": a.page,
|
|
204
|
+
"all_status": a.all_status},
|
|
205
|
+
"sec_fulltext_search": lambda a: {"q": a.q, "forms": a.forms, "startdt": a.startdt,
|
|
206
|
+
"enddt": a.enddt, "limit": a.limit},
|
|
207
|
+
"sec_submissions": lambda a: {"cik": a.cik},
|
|
208
|
+
"ted_notices_search": lambda a: {"query": a.query,
|
|
209
|
+
"fields": a.fields, "limit": a.limit, "page": a.page},
|
|
210
|
+
"usaspending_spending_by_award": lambda a: {"recipient_search_text": a.recipient_search_text,
|
|
211
|
+
"award_type_codes": a.award_type_codes,
|
|
212
|
+
"start_date": a.start_date, "end_date": a.end_date,
|
|
213
|
+
"fields": a.fields, "limit": a.limit, "page": a.page,
|
|
214
|
+
"subawards": a.subawards},
|
|
215
|
+
"vies_vat": lambda a: {"vat": a.vat},
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
# 0.3.10 jurisdiction chapter: face → jurisdiction (default = the issuing authority's jurisdiction; bodies
|
|
220
|
+
# covering global phenomena = GLOBAL; vocabulary = ISO-3166 alpha-2 + EU + GLOBAL) — merged into _JURISDICTION by the root cli.py; success envelopes carry the jurisdiction chapter
|
|
221
|
+
COUNTRY = {
|
|
222
|
+
"ch_zefix_firm": "CH",
|
|
223
|
+
"ch_zefix_firm_search": "CH",
|
|
224
|
+
"co_rues_companies": "CO",
|
|
225
|
+
"fdic_institutions": "US",
|
|
226
|
+
"fi_ytj_companies": "FI",
|
|
227
|
+
"fr_recherche_search": "FR",
|
|
228
|
+
"gb_contracts_finder_published_notices": "GB",
|
|
229
|
+
"gb_fts_release_packages": "GB",
|
|
230
|
+
"gleif_direct_parent": "GLOBAL",
|
|
231
|
+
"gleif_lei_records": "GLOBAL",
|
|
232
|
+
"gleif_lei_records_search": "GLOBAL",
|
|
233
|
+
"gleif_ultimate_parent": "GLOBAL",
|
|
234
|
+
"kr_hometax_biz_status": "KR",
|
|
235
|
+
"no_brreg_enheter": "NO",
|
|
236
|
+
"no_brreg_roller": "NO",
|
|
237
|
+
"ror_organizations": "GLOBAL",
|
|
238
|
+
"sec_fulltext_search": "US",
|
|
239
|
+
"sec_submissions": "US",
|
|
240
|
+
"ted_notices_search": "EU",
|
|
241
|
+
"usaspending_spending_by_award": "US",
|
|
242
|
+
"vies_vat": "EU",
|
|
243
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""co_rues — Colombia RUES commercial-registry master provider (datos.gov.co Socrata; keyless).
|
|
2
|
+
|
|
3
|
+
company roster (admitted in 0.3.11, probe tmp/probe-co-kr/; harvest-side recipe in the co-rues-socrata volume).
|
|
4
|
+
Shape verified 2026-10-30: GET /resource/c82u-588k.json?$limit= → array straight out (9,436,330-row corpus,
|
|
5
|
+
36 columns — Socrata omits null fields per row: natural-person rows carry no NIT/RL columns, legal-entity
|
|
6
|
+
rows surface the three-column RL family).
|
|
7
|
+
Company-row key and profile: matricula (camara x matricula registry key) + numero_identificacion
|
|
8
|
+
(rows with clase_identificacion='NIT' = corporate tax id NIT) + digito_verificacion check digit +
|
|
9
|
+
razon_social + CIIU primary/secondary industries (cod_ciiu_act_econ_pri/sec) + tipo_sociedad/
|
|
10
|
+
organizacion_juridica two-level legal form + estado_matricula/fecha_cancelacion status family +
|
|
11
|
+
representante_legal (officer-name edge — names in a public registry are collectable, no_brreg roller precedent).
|
|
12
|
+
|
|
13
|
+
**PII double gate (red line, pinned by the 2026-10-30 probe)**: (1) default base filter
|
|
14
|
+
clase_identificacion='NIT' — on persona natural rows numero_identificacion equals the nit column
|
|
15
|
+
= cedula personal id (observed 7174354); the company-domain face admits tax-entity rows only
|
|
16
|
+
(the user where is AND-merged with the base filter, parentheses preserve the user expression);
|
|
17
|
+
(2) row shaping strips num_identificacion_representante_legal (the RL's cedula number) —
|
|
18
|
+
the name is kept, the number is not collected. Sole-proprietor bulk needs go through the harvest
|
|
19
|
+
bulk line (api/views rows.csv has no Range), not this face.
|
|
20
|
+
Licence CC-BY-SA (contagious — site redistribution on its own rail; the SDK query face is not restricted).
|
|
21
|
+
|
|
22
|
+
Socrata $-param verbatim channel (same contract as ny_open_data/secop): $where raw SoQL passthrough /
|
|
23
|
+
$limit 1-500 / $offset>=0 / $order rejects NULLS; no total echo, none faked (corpus count probed
|
|
24
|
+
separately via $select=count).
|
|
25
|
+
TTL: registry master changes slowly — 7 days; politeness 1 rps (conservative tier for anonymous Socrata).
|
|
26
|
+
"""
|
|
27
|
+
from ..errors import SdkError
|
|
28
|
+
|
|
29
|
+
provider = {
|
|
30
|
+
"id": "co_rues",
|
|
31
|
+
"baseUrl": "https://www.datos.gov.co",
|
|
32
|
+
"rate": {"rps": 1, "burst": 2},
|
|
33
|
+
"ops": {
|
|
34
|
+
"companies": {"path": "/resource/c82u-588k.json"}, # Socrata $limit/$offset/$where/$order
|
|
35
|
+
},
|
|
36
|
+
"ttl": {"companies": 7 * 86400},
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
# PII red-line columns: RL (legal representative) cedula number — name columns kept, number columns always stripped
|
|
40
|
+
_RL_ID_COLUMNS = ("num_identificacion_representante_legal",)
|
|
41
|
+
|
|
42
|
+
# Default base filter: tax-entity rows only (clase_identificacion='NIT') — on persona natural rows
|
|
43
|
+
# numero_identificacion/nit = cedula personal id; the company-domain face does not admit them (PII gate one of two)
|
|
44
|
+
_BASE_FILTER = "clase_identificacion='NIT'"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _strip_pii(row):
|
|
48
|
+
for col in _RL_ID_COLUMNS:
|
|
49
|
+
row.pop(col, None)
|
|
50
|
+
return row
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def companies_raw(where=None, *, limit=20, offset=0, order=None, cache=None, no_cache=False):
|
|
54
|
+
"""→ {rows[], count, limit_applied, params, _meta} (Socrata echoes no total; RL number column stripped from rows)."""
|
|
55
|
+
from ..transport import call_provider
|
|
56
|
+
|
|
57
|
+
size = int(limit) if limit is not None else 20 # explicit None check — 0 swallowed by or is the 0.2.10-style pit
|
|
58
|
+
if not 1 <= size <= 500:
|
|
59
|
+
raise SdkError("usage_error", provider="co_rues",
|
|
60
|
+
detail=f"limit must be 1-500 (politeness cap), got {limit!r}")
|
|
61
|
+
off = int(offset) if offset is not None else 0
|
|
62
|
+
if off < 0:
|
|
63
|
+
raise SdkError("usage_error", provider="co_rues", detail=f"offset must be >=0, got {offset!r}")
|
|
64
|
+
w = str(where or "").strip() or None
|
|
65
|
+
merged = f"({w}) and {_BASE_FILTER}" if w else _BASE_FILTER # base filter AND-merged (parentheses preserve the user expression)
|
|
66
|
+
params = {"$limit": size, "$offset": off, "$where": merged}
|
|
67
|
+
o = str(order or "").strip() or None
|
|
68
|
+
if o:
|
|
69
|
+
params["$order"] = o
|
|
70
|
+
r = call_provider(provider, "companies", params, cache=cache, no_cache=no_cache)
|
|
71
|
+
data = [_strip_pii(x) for x in (r["data"] or [])]
|
|
72
|
+
return {"rows": data, "count": len(data), "limit_applied": size, "params": params, "_meta": r}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""edgar_cik — SEC EDGAR submissions (CIK → registrant identity card; keyless).
|
|
2
|
+
|
|
3
|
+
data.sec.gov/submissions/CIK##########.json — same source as edgar_fts, different face: FTS handles search,
|
|
4
|
+
this file handles normalization (fulltext hits resolve via ciks[0] here; ticker/LEI→CIK closes at the same door).
|
|
5
|
+
Official fair-access policy caps at 10 req/s and requires a UA — we set 5 rps/burst 5 (0.2.7 realignment: the old
|
|
6
|
+
5/10 worst case was 15>10 in a 1s window, an lda-type violation; quota semantics 5+5=10≤10 stay inside; as with
|
|
7
|
+
edgar_fts, the UA is attached uniformly by the transport layer).
|
|
8
|
+
Field-value findings confirmed by probing (2026-10-22): business state key = stateOrCountry, not state (the old
|
|
9
|
+
read of state made the US state permanently null — Apple=CA was the confirmed live failure); foreign-address state
|
|
10
|
+
semantics lives in foreignStateTerritory (Shell plc=DC); ein '000000000' = foreign issuer without EIN sentinel;
|
|
11
|
+
person-face empty-string values (sic/category/stateOfIncorporation '') normalize to null; entityType has only the
|
|
12
|
+
two values {operating, other} — natural persons (Musk) are also other, not a distinct value.
|
|
13
|
+
P1-1 port (2026-10-22): full business key table (three live entities) = street1/street2/city/stateOrCountry/zipCode/
|
|
14
|
+
stateOrCountryDescription/isForeignLocation/foreignStateTerritory/country/countryCode — state code and readable name
|
|
15
|
+
are two parallel slots (ASML P7→Netherlands; Apple CA→CA; foreign rows fall back to the readable country name
|
|
16
|
+
country=United Kingdom; the country key is kept, not always null); filings_total = recent + files[].filingCount
|
|
17
|
+
(recent is always truncated at the top).
|
|
18
|
+
CIK input tolerance: bare digits / leading zeros / the "CIK0000320193" prefix are all accepted (non-digits stripped,
|
|
19
|
+
then padded to 10).
|
|
20
|
+
"""
|
|
21
|
+
import re
|
|
22
|
+
|
|
23
|
+
provider = {
|
|
24
|
+
"id": "edgar-cik",
|
|
25
|
+
"baseUrl": "https://data.sec.gov",
|
|
26
|
+
"rate": {"rps": 5, "burst": 5}, # quota semantics at the line: 5+5×1s=10≤SEC 10/s (0.2.7 realignment)
|
|
27
|
+
"ops": {
|
|
28
|
+
"submissions": {"path": "/submissions/CIK{cik10}.json", "notFound": [404]},
|
|
29
|
+
},
|
|
30
|
+
"ttl": {"submissions": 7 * 86400},
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def normalize_cik(value) -> str:
|
|
35
|
+
"""→ 10-digit zero-padded CIK (parity with Node normalizeCik; non 1-10 digit number → usage_error)."""
|
|
36
|
+
from ..errors import SdkError
|
|
37
|
+
|
|
38
|
+
digits = re.sub(r"\D+", "", str(value if value is not None else ""))
|
|
39
|
+
if not digits or len(digits) > 10:
|
|
40
|
+
raise SdkError("usage_error", provider="edgar-cik",
|
|
41
|
+
detail=f"CIK must be 1-10 digits (got \"{value}\")")
|
|
42
|
+
return digits.zfill(10)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def card_from_upstream(d, cik10):
|
|
46
|
+
"""Identity card pure mapping (split out for offline testing; key-name fixes and sentinel normalization live in this layer)."""
|
|
47
|
+
recent = (d.get("filings") or {}).get("recent") or {}
|
|
48
|
+
biz = (d.get("addresses") or {}).get("business")
|
|
49
|
+
return {
|
|
50
|
+
"cik": d.get("cik") or cik10,
|
|
51
|
+
"name": d.get("name"),
|
|
52
|
+
"entity_type": d.get("entityType"),
|
|
53
|
+
"tickers": d.get("tickers") or [],
|
|
54
|
+
"exchanges": d.get("exchanges") or [],
|
|
55
|
+
"ein": d.get("ein") if d.get("ein") and d.get("ein") != "000000000" else None, # foreign sentinel→null
|
|
56
|
+
"sic": ({"code": d["sic"], "description": d.get("sicDescription") or None}
|
|
57
|
+
if d.get("sic") else None), # person face sic='' is falsy → None
|
|
58
|
+
"filer_category": d.get("category") or None,
|
|
59
|
+
"state_of_incorporation": d.get("stateOfIncorporation") or None, # empty string → None
|
|
60
|
+
"fiscal_year_end": d.get("fiscalYearEnd"), # MMDD format (Apple '0926') kept upstream-verbatim
|
|
61
|
+
"former_names": [{"name": f.get("name"),
|
|
62
|
+
"from": (f.get("from") or "")[:10], "to": (f.get("to") or "")[:10]}
|
|
63
|
+
for f in (d.get("formerNames") or [])],
|
|
64
|
+
"business_address": ({"city": biz.get("city"),
|
|
65
|
+
"state_or_country": biz.get("stateOrCountry") or biz.get("foreignStateTerritory"), # code slot: CA/P7; foreign state semantics = foreignStateTerritory (Shell=DC)
|
|
66
|
+
"state_or_country_name": biz.get("stateOrCountryDescription") or biz.get("country"), # readable-name slot: CA/Netherlands; foreign rows fall back to the country name (United Kingdom)
|
|
67
|
+
"country": biz.get("country")} if biz else None), # only present for foreign addresses (2026-10-22 live check: Shell carries a real value, do not remove)
|
|
68
|
+
"recent_filings_count": len(recent.get("form") or []),
|
|
69
|
+
"filings_total": (len(recent.get("form") or [])
|
|
70
|
+
+ sum(int(f.get("filingCount") or 0)
|
|
71
|
+
for f in ((d.get("filings") or {}).get("files") or []))), # recent is always truncated at ~1000, history pagination lives in files[].filingCount — the two numbers must not be mixed
|
|
72
|
+
"last_filing_date": (recent.get("filingDate") or [None])[0], # recent arrays are newest-first
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def lookup_cik(value, *, cache=None, no_cache=False):
|
|
77
|
+
"""→ registrant identity card (legal/former names, tickers·exchanges, EIN, SIC industry, filer category, address)."""
|
|
78
|
+
from ..transport import call_provider
|
|
79
|
+
|
|
80
|
+
cik10 = normalize_cik(value)
|
|
81
|
+
r = call_provider(provider, "submissions", {"cik10": cik10}, cache=cache, no_cache=no_cache)
|
|
82
|
+
out = card_from_upstream(r["data"] or {}, cik10)
|
|
83
|
+
out["_meta"] = r
|
|
84
|
+
return out
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""edgar_fts — SEC EDGAR Full-Text Search (2001+ full-text filing search; keyless).
|
|
2
|
+
|
|
3
|
+
Official fair-access policy caps at 10 req/s and requires a UA — we set 5 rps/burst 5 (0.2.7 realignment: the old
|
|
4
|
+
5/10 worst case was 15>10 in a 1s window, an lda-type violation; quota semantics 5+5=10≤10; the UA is attached
|
|
5
|
+
uniformly by the transport layer).
|
|
6
|
+
First case of the "full-text search exception": we do not build a local terabyte-scale inverted index and instead
|
|
7
|
+
lean on the official compute as an external read-only FTS face.
|
|
8
|
+
Live-shape check:
|
|
9
|
+
hits.hits[]{_id="adsh:file", _source{adsh,ciks[],display_names[],root_forms[],file_date,form}}.
|
|
10
|
+
Date-range pitfall (probed live 2026-10-20): upstream applies dateRange=custom only when startdt AND enddt are
|
|
11
|
+
both given — a single end is silently ignored (tell-tale signs: total pinned at 10000 + relation:"gte" + out-of-range
|
|
12
|
+
file_date with ok:true) — so if either end is given the other is filled in (missing startdt → corpus start 2001-01-01,
|
|
13
|
+
missing enddt → today); date_range echoes the window actually in effect.
|
|
14
|
+
"""
|
|
15
|
+
import re
|
|
16
|
+
|
|
17
|
+
provider = {
|
|
18
|
+
"id": "edgar-fts",
|
|
19
|
+
"baseUrl": "https://efts.sec.gov",
|
|
20
|
+
"rate": {"rps": 5, "burst": 5}, # quota semantics at the line: 5+5×1s=10≤SEC 10/s (0.2.7 realignment)
|
|
21
|
+
"ops": {
|
|
22
|
+
"search": {"path": "/LATEST/search-index"}, # params: q, forms, startdt, enddt, dateRange (assembled query)
|
|
23
|
+
},
|
|
24
|
+
"ttl": {"search": 24 * 3600}, # new filings keep landing, a search page stays valid for a day
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
_TAIL_PARENS = re.compile(r"(\s*\([^()]*\))+\s*$")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def clean_display_name(v):
|
|
31
|
+
"""Strip the parenthesized tail noise from upstream display_names[0] ('NAME (TICKER) (CIK ##########)' → 'NAME').
|
|
32
|
+
|
|
33
|
+
Double spaces + a CIK tail that duplicates the cik field already in the card; mid-string parentheses such as
|
|
34
|
+
'X (formerly Y) Inc.' are left untouched."""
|
|
35
|
+
|
|
36
|
+
if not isinstance(v, str):
|
|
37
|
+
return v if v is not None else None
|
|
38
|
+
cleaned = _TAIL_PARENS.sub("", v).strip()
|
|
39
|
+
return cleaned or v
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def search_fulltext(q, *, forms=None, startdt=None, enddt=None, limit=10, cache=None, no_cache=False):
|
|
43
|
+
"""→ {total, total_relation, hits[], date_range, _meta}; hit.adsh directly composes the accession key shared with the SEC corpus line, doc_url points at the human-readable original.
|
|
44
|
+
total_relation:"gte" = upstream truncation (≥ ten thousand, not an exact value), "eq" = exact; date_range=None means no date window was enabled."""
|
|
45
|
+
from ..transport import call_provider
|
|
46
|
+
from ..errors import SdkError
|
|
47
|
+
|
|
48
|
+
query = str(q or "").strip()
|
|
49
|
+
if not query:
|
|
50
|
+
raise SdkError("usage_error", provider="edgar-fts", detail="fulltext requires --q")
|
|
51
|
+
params = {"q": query}
|
|
52
|
+
if forms:
|
|
53
|
+
params["forms"] = str(forms) # form-type filter is decoupled from the date window (forms no longer triggers dateRange)
|
|
54
|
+
date_range = None
|
|
55
|
+
if startdt or enddt: # fill the missing end: a single end is silently ignored upstream, both ends are required
|
|
56
|
+
from ..daterange import resolve as _range
|
|
57
|
+
_start, _end, date_range = _range(startdt, enddt, default_start="2001-01-01",
|
|
58
|
+
provider=provider["id"])
|
|
59
|
+
params["startdt"], params["enddt"] = _start, _end
|
|
60
|
+
params["dateRange"] = "custom"
|
|
61
|
+
r = call_provider(provider, "search", params, cache=cache, no_cache=no_cache)
|
|
62
|
+
hits = ((r["data"] or {}).get("hits") or {}).get("hits") or []
|
|
63
|
+
cap = min(max(1, int(limit) or 10), 100)
|
|
64
|
+
rows = []
|
|
65
|
+
for h in hits[:cap]:
|
|
66
|
+
s = h.get("_source") or {}
|
|
67
|
+
cik = (s.get("ciks") or [None])[0]
|
|
68
|
+
_id = h.get("_id") or ""
|
|
69
|
+
file = _id.split(":")[1] if ":" in _id else None
|
|
70
|
+
adsh = s.get("adsh") or (_id.split(":")[0] if _id else None)
|
|
71
|
+
rows.append({
|
|
72
|
+
"adsh": adsh, # accession number — same key as the SEC corpus line
|
|
73
|
+
"cik": cik,
|
|
74
|
+
"company": clean_display_name((s.get("display_names") or [None])[0]), # strip the (TICKER)(CIK) parenthesized tail
|
|
75
|
+
"form_type": s.get("form") or (s.get("root_forms") or [None])[0],
|
|
76
|
+
"file_date": s.get("file_date"),
|
|
77
|
+
"period_ending": s.get("period_ending"),
|
|
78
|
+
"doc_url": (f"https://www.sec.gov/Archives/edgar/data/{int(cik)}/{adsh.replace('-', '')}/{file}"
|
|
79
|
+
if cik and adsh and file else None),
|
|
80
|
+
})
|
|
81
|
+
total_obj = (((r["data"] or {}).get("hits") or {}).get("total") or {})
|
|
82
|
+
total = total_obj.get("value")
|
|
83
|
+
return {"total": total if total is not None else len(rows),
|
|
84
|
+
"total_relation": total_obj.get("relation"), # gte=truncated (≥ ten thousand), not an exact value
|
|
85
|
+
"hits": rows, "limit_applied": cap, "date_range": date_range, "_meta": r} # cap = client-side slice size, truncation must stay visible
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def search_raw(q, *, forms=None, startdt=None, enddt=None, limit=10, cache=None, no_cache=False):
|
|
89
|
+
"""→ {total, total_relation, hits[], limit_applied, date_range, params, _meta}; the four params pass through verbatim.
|
|
90
|
+
params = the actually sent query-string keys (q/forms/startdt/enddt[/dateRange=custom]) — after end completion both ends are always present (a single end is
|
|
91
|
+
silently ignored upstream, measured 2026-10-20), date_range echoes the window actually in effect; the echo is replayable."""
|
|
92
|
+
r = search_fulltext(q, forms=forms, startdt=startdt, enddt=enddt,
|
|
93
|
+
limit=limit, cache=cache, no_cache=no_cache)
|
|
94
|
+
params = {"q": str(q).strip()}
|
|
95
|
+
if forms:
|
|
96
|
+
params["forms"] = str(forms)
|
|
97
|
+
if r["date_range"]:
|
|
98
|
+
params["startdt"] = r["date_range"]["startdt"]
|
|
99
|
+
params["enddt"] = r["date_range"]["enddt"]
|
|
100
|
+
params["dateRange"] = "custom"
|
|
101
|
+
return {**r, "params": params}
|