agentdata 0.3.26__py3-none-any.whl

agentdata/procurement/ny_open_data.py

@@ -0,0 +1,82 @@
+"""ny_open_data — New York State open data Socrata three-face provider (data.ny.gov; keyless).
+
+procurement register (0.3.6 twin faces + 0.3.12 OGS third face; shapes verified by first-hand live probing). Probed live 2026-10-29/30:
+GET /resource/ehig-g5x3.json?$limit=2 → [rows…25 cols] (ABO state-authority contracts **275,763 rows**)
+GET /resource/twsw-2mqa.json?$limit=2 → [rows…27 cols] (MTA procurements **107,503 rows**)
+GET /resource/rb9h-9fit.json?$limit=1 → [rows…7 cols] (OGS design & construction payments **69,221 rows**)
+  —— Socrata emits a bare array with no total echo (same contract as co_secop: page via $offset, probe the
+  full corpus separately with $select=count).
+ABO row (Authorities Budget Office state-authority universe): authority_name (buyer authority) × vendor_name+
+vendor_address_1/city/state/postal_code/country (**name+full address — no UEI/DUNS, name-walk identification**) +
+contract_amount/amount_expended_for_fiscal_year/amount_expended_to_date/
+current_or_outstanding_balance four value columns + award_date/begin_date/renewal_date/
+fiscal_year_end_date date family + status/type_of_procurement/award_process/
+number_of_bids_or_proposals_received + solicited_mwbe/vendor_is_a_mwbe/
+nys_or_foreign_business_enterprise procurement-preference flag family.
+MTA row (Metropolitan Transportation Authority): same skeleton + end_date/transaction_number/number_of_mwbe_proposals/
+vendor_is_nys_or_fbe (no authority_name — buyer is always the MTA).
+OGS row (Office of General Services design & construction payments, probed live 2026-10-30): contractnumber/
+vendor/typeofservice/paymentamount/fiscalyear (a "2014-2015"-form string)/quarter/county — seven columns,
+buyer always OGS, **no address columns** (name-walk single key; contractnumber+vendor+quarter pin a payment row).
+Socrata $-param verbatim channel shares the co_secop contract: $where raw SoQL verbatim pass-through/$limit/$offset/$order;
+$order does not take NULLS syntax. Socrata float-coercion pitfall on record: without a $select projection
+transaction_number comes back in scientific-notation form — the row face returns it as upstream does; when exact
+numeric keys are needed, narrow with --order/--where and probe separately.
+TTL: contract/procurement rows drift slowly with status — 24h; payment rows are historical fact — 7d;
+polite tier 1 rps (conservative pick for anonymous Socrata).
+"""
+from ..errors import SdkError
+
+provider = {
+    "id": "ny_open_data",
+    "baseUrl": "https://data.ny.gov",
+    "rate": {"rps": 1, "burst": 2},
+    "ops": {
+        "abo_contracts": {"path": "/resource/ehig-g5x3.json"},      # Socrata $limit/$offset/$where/$order
+        "mta_procurements": {"path": "/resource/twsw-2mqa.json"},
+        "ogs_dc_payments": {"path": "/resource/rb9h-9fit.json"},
+    },
+    "ttl": {"abo_contracts": 24 * 3600, "mta_procurements": 24 * 3600,
+            "ogs_dc_payments": 7 * 24 * 3600},
+}
+
+
+def _socrata_raw(op, where=None, *, limit=20, offset=0, order=None, cache=None, no_cache=False):
+    """Socrata $-channel, same contract (co_secop-shaped): where=$where raw SoQL/limit 1-500/offset≥0/order=column [ASC|DESC]."""
+    from ..transport import call_provider
+
+    size = int(limit) if limit is not None else 20  # test None explicitly — 0 swallowed by `or` into the default is the same-shaped 0.2.10 pitfall
+    if not 1 <= size <= 500:
+        raise SdkError("usage_error", provider="ny_open_data",
+                       detail=f"limit must be 1-500 (polite tier), got '{limit}'")
+    off = int(offset) if offset is not None else 0
+    if off < 0:
+        raise SdkError("usage_error", provider="ny_open_data", detail=f"offset must be ≥0, got '{offset}'")
+    params = {"$limit": size, "$offset": off}
+    w = str(where or "").strip() or None
+    if w:
+        params["$where"] = w
+    o = str(order or "").strip() or None
+    if o:
+        params["$order"] = o
+    r = call_provider(provider, op, params, cache=cache, no_cache=no_cache)
+    data = r["data"] or []
+    return {"rows": data, "count": len(data), "limit_applied": size, "params": params, "_meta": r}
+
+
+def abo_contracts_raw(where=None, *, limit=20, offset=0, order=None, cache=None, no_cache=False):
+    """→ {rows[], count, limit_applied, params, _meta} (Socrata sends no total echo)."""
+    return _socrata_raw("abo_contracts", where, limit=limit, offset=offset, order=order,
+                        cache=cache, no_cache=no_cache)
+
+
+def mta_procurements_raw(where=None, *, limit=20, offset=0, order=None, cache=None, no_cache=False):
+    """→ same as above (buyer is always the MTA — rows carrying no authority_name column is the upstream shape, not missing data)."""
+    return _socrata_raw("mta_procurements", where, limit=limit, offset=offset, order=order,
+                        cache=cache, no_cache=no_cache)
+
+
+def ogs_dc_payments_raw(where=None, *, limit=20, offset=0, order=None, cache=None, no_cache=False):
+    """→ same as above (OGS design & construction payments, 69,221 rows — buyer is always OGS; rows carry no address columns, name-walk identity)."""
+    return _socrata_raw("ogs_dc_payments", where, limit=limit, offset=offset, order=order,
+                        cache=cache, no_cache=no_cache)

agentdata/procurement/ungm.py

@@ -0,0 +1,137 @@
+"""ungm — UN Global Marketplace contract-award search provider (www.ungm.org/Public/ContractAward; keyless).
+
+procurement register (admitted in 0.3.12; shapes verified by first-hand live probing). Probed live 2026-10-30
+(the XHR contract reverse-engineered from bundles/ungmcommon; the two-step pairing walked end to end):
+GET /Public/ContractAward (**a browser UA is mandatory** — the default agentdata UA is refused with a 403
+(a 134-byte 403 stub; same UA-fingerprint gate family as FAA; any full browser UA passes)) → two Set-Cookie
+pieces (the __RequestVerificationToken session cookie + GCLB) + the same-named token in a hidden field on
+the page (the pairing key).
+POST /Public/ContractAward/PublicSearch (JSON body = the browser buildOptions() **full 15-field shape**;
+carries the Cookie + RequestVerificationToken header pair) → HTML row family back (each row keyed by
+data-contractawardid + six slots: title/vendor/award_date/agency/reference/supplier_country) + a trailing
+script `#searchTotal").html(N)` total echo. A bare POST without the pairing = 400; **a null SortField = 500
+server crash** (pinned live 2026-10-30 — the request body must be the full shape and SortField never empty).
+Full corpus ~50,035 awards (Supplier/AwardFrom/AwardTo/Countries/Agencies/UNSPSCs filters;
+PageSize 1-100 with the browser at 15/page, PageIndex 1-based paging).
+The anti-forgery pairing (cookie × token) is cached 15 minutes and reused; 400 = the pairing expired —
+re-collected once automatically.
+TTL: published awards are immutable — 7d; polite tier 0.5 rps (conservative pick; the UN portal publishes no quota).
+"""
+import html as _html
+import re
+
+from ..errors import SdkError
+
+provider = {
+    "id": "ungm",
+    "baseUrl": "https://www.ungm.org",
+    "rate": {"rps": 0.5, "burst": 1},
+    "ops": {
+        "search_page": {"path": "/Public/ContractAward", "accept": "text"},
+        "search": {"path": "/Public/ContractAward/PublicSearch", "method": "POST",
+                   "idempotent": True, "accept": "text"},
+    },
+    "ttl": {"search": 7 * 24 * 3600},
+}
+
+# UA fingerprint gate (pinned live 2026-10-30): curl/python default UAs get 403; full browser UAs get 200 — per-request header override (ext 8)
+_BROWSER_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+               "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
+
+_SESSION_TTL = 15 * 60  # anti-forgery pairing reuse window (the same pairing is reusable within the ASP.NET session lifetime)
+
+_TOKEN_FIELD = re.compile(r'name="__RequestVerificationToken"[^>]*value="([^"]+)"')
+_ROW_SPLIT = re.compile(r'<div role="row"[^>]*data-contractawardid="(\d+)"')
+_SPAN = re.compile(r"<span[^>]*>(.*?)</span>", re.S)
+_TAGWS = re.compile(r"<[^>]+>|\s+")
+_TOTAL = re.compile(r'#searchTotal"\)\.html\((\d+)\)')
+
+
+def _cell_text(chunk):
+    """cell block -> the first span's text (tags stripped + entities unescaped + whitespace collapsed; an empty slot = None — e.g. rows carrying no reference)."""
+    m = _SPAN.search(chunk)
+    if not m:
+        return None
+    return _html.unescape(_TAGWS.sub(" ", m.group(1))).strip() or None
+
+
+def awards_from_upstream(text):
+    """search-page HTML -> {rows[], total, count} (a pure function, testable offline).
+
+    The row's six slots (stable role=cell order): title/vendor/award_date/agency/reference/supplier_country."""
+    rows = []
+    parts = _ROW_SPLIT.split(text or "")
+    for rid, block in zip(parts[1::2], parts[2::2]):
+        cells = [_cell_text(c) for c in block.split('<div role="cell"')[1:]]
+        c = (cells + [None] * 6)[:6]
+        rows.append({"id": rid, "title": c[0], "vendor": c[1], "award_date": c[2],
+                     "agency": c[3], "reference": c[4], "supplier_country": c[5]})
+    m = _TOTAL.search(text or "")
+    return {"rows": rows, "total": int(m.group(1)) if m else None, "count": len(rows)}
+
+
+def _session(cache, no_cache=False):
+    """-> (cookie_hdr, token): the anti-forgery pairing — cached 15 minutes and reused; on miss, collected by GETting the search page."""
+    if not no_cache and cache is not None:
+        hit = cache.get("ungm", "antiforgery_session", _SESSION_TTL)
+        if hit:
+            return hit["payload"]["cookie"], hit["payload"]["token"]
+    from ..transport import call_provider
+    r = call_provider(provider, "search_page", {}, cache=cache, no_cache=True,
+                      extra_headers={"User-Agent": _BROWSER_UA,
+                                     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"})
+    m = _TOKEN_FIELD.search(r["data"] or "")
+    if not m or not r.get("set_cookie"):
+        raise SdkError("upstream_unavailable", provider="ungm",
+                       detail="bootstrap did not obtain the anti-forgery pairing (page token/session cookie missing) — the upstream page shape may have changed")
+    pair = {"cookie": "; ".join(sc.split(";", 1)[0].strip() for sc in r["set_cookie"]),
+            "token": m.group(1)}
+    if cache is not None:
+        cache.set("ungm", "antiforgery_session", pair, _SESSION_TTL)
+    return pair["cookie"], pair["token"]
+
+
+def awards_raw(supplier=None, *, title=None, reference=None, award_from=None, award_to=None,
+               countries=None, agencies=None, unspscs=None, page_index=1, page_size=15,
+               sort_field="AwardDate", sort_ascending=False, cache=None, no_cache=False):
+    """-> {rows[], total, count, params, _meta} (params = the JSON body actually sent — replayable).
+
+    Filter params mirror the upstream names (Supplier/Title/Reference/AwardFrom/AwardTo/Countries/Agencies/UNSPSCs);
+    page_index is 1-based / page_size 1-100 (the browser uses 15/page); sort_field defaults to AwardDate
+    (null = upstream 500, never empty)."""
+    from ..transport import call_provider
+
+    idx = int(page_index) if page_index is not None else 1
+    if idx < 1:
+        raise SdkError("usage_error", provider="ungm",
+                       detail=f"page_index must be >=1 (1-based), got {page_index!r}")
+    size = int(page_size) if page_size is not None else 15
+    if not 1 <= size <= 100:
+        raise SdkError("usage_error", provider="ungm",
+                       detail=f"page_size must be 1-100 (politeness cap; the browser uses 15/page), got {page_size!r}")
+    # request body = the browser buildOptions() full shape: missing keys/empty values can each trigger a
+    # server-side exception (pinned: a null SortField = 500)
+    params = {
+        "PageIndex": idx, "PageSize": size,
+        "Title": title or "", "Description": "", "Reference": reference or "",
+        "Supplier": supplier or "", "UngmNumber": "",
+        "AwardFrom": award_from or "", "AwardTo": award_to or "",
+        "Countries": list(countries or []), "SupplierCountries": [],
+        "Agencies": list(agencies or []), "UNSPSCs": list(unspscs or []),
+        "SortField": sort_field or "AwardDate", "SortAscending": bool(sort_ascending),
+    }
+    cookie, token = _session(cache, no_cache)
+    hdrs = {"User-Agent": _BROWSER_UA, "Cookie": cookie, "RequestVerificationToken": token}
+    try:
+        r = call_provider(provider, "search", params, cache=cache, no_cache=no_cache,
+                          extra_headers=hdrs)
+    except SdkError as e:
+        # 400 = the anti-forgery pairing expired (session cookie invalidated) — drop the cached pairing,
+        # re-collect once and resend; still 400 → raise as-is
+        if e.status != 400 or no_cache:
+            raise
+        cookie, token = _session(None)  # force re-collection (bypassing the cached pairing)
+        r = call_provider(provider, "search", params, cache=cache, no_cache=no_cache,
+                          extra_headers={"User-Agent": _BROWSER_UA, "Cookie": cookie,
+                                         "RequestVerificationToken": token})
+    return {**awards_from_upstream(r["data"]), "params": params, "_meta": r}

agentdata/procurement/wb_proc.py

@@ -0,0 +1,84 @@
+"""wb_proc — World Bank procurement notice stream face provider (search.worldbank.org /api/v2/procnotices; keyless).
+
+procurement register (admitted in 0.3.6; shapes verified by first-hand live probing). Probed live 2026-10-29:
+GET /api/v2/procnotices?format=json&notice_type=Contract+Award&rows=2&os=0
+  → {rows (echoed), os (echoed), page (echoed), total ("313071" — a string, not a number), procnotices [rows…15 keys]}
+  —— **313,071 award notices in full** (MDB buyer universe: IBRD/IDA development projects in 100+ countries).
+Row keys: id (OP004xxxxx stable key)/notice_type/noticedate ("28-Oct-2026" form)/notice_status/
+project_id (P five-digit key)/project_name/project_ctry_name/bid_reference_no/bid_description/
+procurement_group/procurement_method_code/_name/submission_date/notice_text (full HTML text).
+Supplier keys hide in the embedded notice_text section: `Awarded Firm(s): NAME (SUPP_ID) address…` — regex-extracted into
+{name, supp_id} (**firm name + WB supplier id, both emitted**, probed live 2026-10-29: EASYMANAGEMENT (1137478) etc.);
+notice_text itself is not returned on the row face (embedded contact addresses/phones stay out of the face — face values = name + key + project axis).
+Params mirrored verbatim: notice_type (default "Contract Award" — the value domain of other notice types is upstream's own, passed through with no local wall)/
+rows (page size)/os (offset); filter params in quoted form (%22…%22) give a fake zero-hit — upstream quirk on record, not compensated.
+TTL: immutable once notice_status=Published — 7 days; polite tier 1 rps (no published quota, conservative pick).
+"""
+import re
+
+from ..errors import SdkError
+
+provider = {
+    "id": "wb_proc",
+    "baseUrl": "https://search.worldbank.org",
+    "rate": {"rps": 1, "burst": 2},
+    "ops": {
+        "procnotices": {"path": "/api/v2/procnotices"},
+    },
+    "ttl": {"procnotices": 7 * 24 * 3600},
+}
+
+# Row narrowing (15 keys → skeleton minus notice_text/notice_lang_name; upstream key names verbatim)
+_FIELDS = ("id", "notice_type", "noticedate", "notice_status", "project_ctry_name",
+           "project_id", "project_name", "bid_reference_no", "bid_description",
+           "procurement_group", "procurement_method_code", "procurement_method_name",
+           "submission_date")
+
+# Embedded supplier section of notice_text (after tag stripping): starts at `Awarded Firm(s):`, stops at the Scores/Minimum Qualifying section markers
+_AWARD_SEG = re.compile(r"Awarded Firm\(s\)\s*:?\s*(.*?)(?=\s{2,}(?:Scores|Minimum Qualifying|Date Notification)|$)",
+                        re.S)
+# Pair-by-pair `NAME (SUPP_ID)` inside the section — supplier id is 4-10 digits
+_FIRM_PAIR = re.compile(r"([^\(\n]{2,120}?)\s*\((\d{4,10})\)")
+_TAG = re.compile(r"<[^>]+>")
+
+
+def _awarded_firms(notice_text):
+    """notice_text HTML → [{name, supp_id}…] (pure function, offline-testable; address/phone sections are not paired in)."""
+    seg = _AWARD_SEG.search(_TAG.sub(" ", notice_text or ""))
+    if not seg:
+        return []
+    return [{"name": m.group(1).strip(" ,;:"), "supp_id": m.group(2)}
+            for m in _FIRM_PAIR.finditer(seg.group(1))]
+
+
+def procnotices_from_upstream(data):
+    """Upstream response → {rows[], total, count} (pure function, offline-testable)."""
+    rows = []
+    for r in (data or {}).get("procnotices") or []:
+        row = {k: r.get(k) for k in _FIELDS}
+        row["awarded_firms"] = _awarded_firms(r.get("notice_text"))
+        rows.append(row)
+    try:
+        total = int(data.get("total")) if data and data.get("total") is not None else None
+    except (TypeError, ValueError):
+        total = None  # total is echoed as a string — a non-numeric shape is not passed off as a number
+    return {"rows": rows, "total": total, "count": len(rows)}
+
+
+def procnotices_raw(notice_type="Contract Award", *, rows=10, os_=0, cache=None, no_cache=False):
+    """→ {rows[], total, count, params, _meta} (os = offset under its upstream name — takes os_ to avoid shadowing the Python built-in).
+
+    notice_type passed through under its upstream name (default Contract Award); rows 1-100 polite tier; os_ ≥0."""
+    from ..transport import call_provider
+
+    size = int(rows) if rows is not None else 10
+    if not 1 <= size <= 100:
+        raise SdkError("usage_error", provider="wb_proc",
+                       detail=f"rows must be 1-100 (polite tier), got '{rows}'")
+    off = int(os_ if os_ is not None else 0)
+    if off < 0:
+        raise SdkError("usage_error", provider="wb_proc", detail=f"os must be ≥0, got '{os_}'")
+    params = {"format": "json", "notice_type": notice_type or "Contract Award",
+              "rows": size, "os": off}
+    r = call_provider(provider, "procnotices", params, cache=cache, no_cache=no_cache)
+    return {**procnotices_from_upstream(r["data"]), "params": params, "_meta": r}

agentdata/research/__init__.py

@@ -0,0 +1,4 @@
+"""research domain package — flat provider adapters (one file per source) + faces/cli (0.2.6 domain reorg).
+"""
+from . import (biorxiv, crossref, gb_gtr, icite, nih_reporter, nsf,  # noqa: F401
+               openalex, opencitations)

agentdata/research/biorxiv.py

@@ -0,0 +1,66 @@
+"""biorxiv — bioRxiv/medRxiv details API (preprint metadata stream; keyless).
+
+No key needed (api.biorxiv.org; no stated quota — self-imposed polite 1 rps tier; for heavy
+loads use the upstream S3 bulk files). details face, interval stream (probed 2026-10-26:
+2026-10-01→20 total 4,619): GET /details/biorxiv/{from}/{to}/{cursor}; response
+{messages:[{status,count,total,…}], collection:[…]}.
+Three traps: count/total inside messages are strings ('30'/'4619') and must be normalized
+to int; the per-row authors field is a semicolon-joined string ("Balch, J. K.; McIntosh,
+T. L.; …"), not an array; jatsxml is a URL to the full-text XML (not inline) — full-text
+bulk belongs on the jatsxml links, this face is metadata only.
+"""
+provider = {
+    "id": "biorxiv",
+    "baseUrl": "https://api.biorxiv.org",
+    "rate": {"rps": 1, "burst": 5},
+    "ops": {
+        "details": {"path": "/details/biorxiv/{from_date}/{to_date}/{cursor}", "method": "GET", "idempotent": True},
+    },
+    "ttl": {"details": 24 * 3600},
+}
+
+
+def _int_or_none(v):
+    try:
+        return int(str(v))
+    except (TypeError, ValueError):
+        return None
+
+
+def _detail_row(c):
+    return {
+        "doi": c.get("doi"), "title": c.get("title"), "category": c.get("category"),
+        "date": c.get("date"), "version": c.get("version"),
+        "authors": c.get("authors"),  # semicolon-joined string (upstream verbatim)
+        "author_corresponding": c.get("author_corresponding"),
+        "author_corresponding_institution": c.get("author_corresponding_institution"),
+        "jatsxml_url": c.get("jatsxml"),  # full-text XML link (URL string, not inline)
+    }
+
+
+def details_raw(from_date, to_date, *, cursor=0, cache=None, no_cache=False):
+    """→ {total, count, rows, has_more, params, _meta}; /details/biorxiv/{from}/{to}/{cursor} direct.
+    from_date/to_date required (YYYY-MM-DD interval); cursor=page index (0-based, upstream also
+    accepts 1 in practice); total=total papers in the interval (string normalized to int); upstream
+    does not echo the page size — has_more judges the boundary conservatively (a non-empty page may
+    not be the end; an empty page certainly is)."""
+    from ..transport import call_provider
+    from ..errors import SdkError
+
+    fd = str(from_date or "").strip()
+    td = str(to_date or "").strip()
+    if not fd or not td:
+        raise SdkError("usage_error", provider="biorxiv",
+                       detail="biorxiv_details requires from_date and to_date (YYYY-MM-DD interval)")
+    cur = int(cursor)
+    if cur < 0:
+        raise SdkError("usage_error", provider="biorxiv", detail="cursor must be ≥0 (page index)")
+    params = {"from_date": fd, "to_date": td, "cursor": cur}
+    r = call_provider(provider, "details", params, cache=cache, no_cache=no_cache)
+    data = r["data"] or {}
+    m0 = (data.get("messages") or [{}])[0] or {}
+    rows = [_detail_row(c) for c in data.get("collection") or []]
+    total = _int_or_none(m0.get("total"))
+    count = _int_or_none(m0.get("count"))
+    return {"total": total, "count": count if count is not None else len(rows),
+            "rows": rows, "has_more": bool(rows), "params": params, "_meta": r}

agentdata/research/cli.py

@@ -0,0 +1,129 @@
+"""research cli — domain command-face registration (split per domain in 0.2.6, migrated from the root cli.py).
+
+register(v, g): registers verb-compat faces + flat commands onto the domain subparser (g = root _leaf_globals
+factory, passed in as a parameter to avoid a cli <-> domain-cli cycle); FLAT = flat dispatch table, merged by
+the root cli for main to use.
+"""
+import argparse
+
+def register(v, g):
+    f1 = v.add_parser("biorxiv_details", parents=[g()],
+                      description="bioRxiv preprint metadata window stream (realtime, local execution; /details/biorxiv/{from}/{to}/{cursor}; authors arrive as a semicolon-joined string and jatsxml is a full-text XML URL, not inline — full-text bulk belongs to the jatsxml_url links; message counts are strings upstream, normalized here)")
+    f1.add_argument("--from-date", dest="from_date", required=True, help="Window start YYYY-MM-DD")
+    f1.add_argument("--to-date", dest="to_date", required=True, help="Window end YYYY-MM-DD")
+    f1.add_argument("--cursor", type=int, default=0, help="Page index (0-based)")
+
+    f2 = v.add_parser("crossref_works", parents=[g()],
+                      description="Crossref works search with funder bridge (realtime, local execution; query full-text × funder DOI (filter=funder:) × award_number (filter=award.number:, requires --funder) × from/to publication window combine AND; award numbers live only inside rows' funders[].award — select cannot fetch them (probe: select-not-available); offset+rows ≤ 10,000 documented upstream wall — deep paging switches to --cursor, the official deep-pager channel (0.2.9 audit)")
+    f2.add_argument("--query", default=None, help="Full-text query")
+    f2.add_argument("--funder", default=None, help="Funder DOI (e.g. 10.13039/100000001 = NSF)")
+    f2.add_argument("--award-number", dest="award_number", default=None,
+                    help="Award/grant number (requires --funder)")
+    f2.add_argument("--from-pub-date", dest="from_pub_date", default=None,
+                    help="Publication window start YYYY-MM-DD (upstream filter: from-pub-date; 0.2.9 audit)")
+    f2.add_argument("--to-pub-date", dest="to_pub_date", default=None,
+                    help="Publication window end YYYY-MM-DD (upstream filter: until-pub-date; 0.2.9 audit)")
+    f2.add_argument("--rows", type=int, default=25, help="Rows (upstream param: rows; 1-100 politeness, upstream doc cap 1000)")
+    f2.add_argument("--offset", type=int, default=0, help="Result offset (offset+rows ≤ 10,000; mutually exclusive with --cursor)")
+    f2.add_argument("--cursor", default=None,
+                    help="Deep-pager cursor ('*' for first page, then feed next_cursor; bypasses the 10,000 offset wall; 0.2.9 audit)")
+
+    f3 = v.add_parser("gb_gtr_projects", parents=[g()],
+                      description="UK UKRI Gateway to Research projects (realtime, local execution; /projects?q=&s=&p=; s ≥ 10 is an upstream 400 wall; upstream sends no total — full page signals more via has_more; organisation edges live one hop deeper in detail links.link[] rel=LEAD_ORG, this face is the list only)")
+    f3.add_argument("--q", required=True, help="Search term")
+    f3.add_argument("--s", type=int, default=20, help="Page size (upstream param: s; ≥10)")
+    f3.add_argument("--p", type=int, default=1, help="Page number (upstream param: p)")
+
+    f4 = v.add_parser("icite_pubs", parents=[g()],
+                      description="NIH iCite citation-impact card for a PubMed paper (realtime, local execution; /api/pubs?pmids= single record; RCR/FCR/citation counts — the impact hop after nih_reporter_publications hands you a pmid)")
+    f4.add_argument("--pmid", required=True, help="PubMed id (digits only, e.g. 26017442)")
+
+    f5 = v.add_parser("nih_reporter_projects", parents=[g()],
+                      description="NIH RePORTER project search (realtime, local execution; POST /v2/projects/search; criteria=snake_case and include_fields=camelCase are separately hard-bound upstream — wrong shapes are silently ignored, so bare passthrough is not exposed; org match is exact, not fuzzy; offset+limit cap 15,000)")
+    f5.add_argument("--org-name", dest="org_name", default=None,
+                    help="Receiving organization name (exact match upstream; scalar auto-wrapped into org_names array)")
+    f5.add_argument("--fiscal-year", dest="fiscal_year", type=int, default=None,
+                    help="Fiscal year (goes into fiscal_years array)")
+    f5.add_argument("--include-fields", dest="include_fields", default=None,
+                    help="Comma list of camelCase fields (upstream include_fields; wrong names are silently ignored upstream)")
+    f5.add_argument("--limit", type=int, default=25, help="Page size (upstream param: limit; 1-500)")
+    f5.add_argument("--offset", type=int, default=0, help="Result offset (offset+limit ≤ 15,000)")
+
+    f6 = v.add_parser("nih_reporter_publications", parents=[g()],
+                      description="NIH funding↔publication bridge (realtime, local execution; POST /v2/publications/search; criteria recognizes ONLY pmids/appl_ids — every other key is silently ignored upstream and dumps the whole 8.3M table, so the local gate requires one of the two and no other criteria is passed through; rows are a 3-key bridge {coreproject, pmid, applid}; zero rows = no linkage on record, sparseness is a real answer)")
+    f6.add_argument("--pmids", dest="pmids", default=None,
+                    help="Comma list of PubMed ids (criteria.pmids)")
+    f6.add_argument("--appl-ids", dest="appl_ids", default=None,
+                    help="Comma list of NIH application ids from nih_reporter_projects (criteria.appl_ids)")
+    f6.add_argument("--limit", type=int, default=25, help="Page size (upstream param: limit; 1-500)")
+    f6.add_argument("--offset", type=int, default=0, help="Result offset")
+
+    f7 = v.add_parser("nsf_awards", parents=[g()],
+                      description="US NSF awards search (realtime, local execution; keyword full-text XOR id point lookup; rpp 1-200 per upstream doc; upstream sends no total — full page signals more via has_more)")
+    f7.add_argument("--keyword", default=None, help="Full-text search keyword (XOR --id)")
+    f7.add_argument("--id", default=None, help="NSF award id, point lookup (XOR --keyword)")
+    f7.add_argument("--rpp", type=int, default=25, help="Results per page (upstream param: rpp; 1-200)")
+    f7.add_argument("--offset", type=int, default=0, help="Result offset for pagination")
+
+    f8 = v.add_parser("openalex_institutions", parents=[g()],
+                      description="Research-institution candidate search (realtime, local execution; /institutions?search=; same-name multi-country clones coexist — the candidate list is NOT auto-picked: choose the ROR explicitly, then query output via openalex_works --ror)")
+    f8.add_argument("--search", required=True, help="Institution name (e.g. pfizer)")
+    f8.add_argument("--per-page", dest="per_page", type=int, default=25,
+                    help="Results per page (upstream param: per-page; 1-200)")
+
+    f9 = v.add_parser("openalex_works", parents=[g()],
+                      description="Global scholarly works search (realtime, local execution; api.openalex.org/works; search full-text × ror institution edge × doi × title_search combine AND into filter=; per_page 1-200 (explicit upstream 400 wall); cursor pagination via meta.next_cursor — the cursor encodes paging only, so re-send the same constraints when continuing; select is force-merged with core fields so slimming cannot strip id/doi/display_name/authorships/open_access)")
+    f9.add_argument("--search", default=None, help="Full-text search term")
+    f9.add_argument("--ror", default=None, help="ROR id (bare 01xdqrp08 or full https://ror.org/… → filter=institutions.ror:)")
+    f9.add_argument("--doi", default=None, help="DOI exact match (filter=doi:)")
+    f9.add_argument("--title-search", dest="title_search", default=None,
+                    help="Title substring match (filter=title.search:)")
+    f9.add_argument("--per-page", dest="per_page", type=int, default=25,
+                    help="Results per page (upstream param: per-page; 1-200)")
+    f9.add_argument("--cursor", default=None, help="Pagination cursor from next_cursor (constraints must be re-sent)")
+    f9.add_argument("--select", default=None,
+                    help="Comma list of extra fields (core fields are force-merged, never stripped)")
+
+    f10 = v.add_parser("opencitations_citations", parents=[g()],
+                      description="OpenCitations COCI citing-DOI list (realtime, local execution; /index/v1/citations/{doi}; doi: prefix and doi.org URLs are normalized away — upstream 400s otherwise; upstream limit param is silently ignored (probe: ?limit=3 still returns all 217) so no fake pagination, the full list comes back with a count)")
+    f10.add_argument("--doi", required=True, help="Cited DOI (doi: prefix auto-stripped)")
+
+FLAT = {
+    # research (grants axis stood up in 0.2.2, renamed research with 8 more faces in 0.2.4) / security 0.2.2 flat panel
+    # (upstream param names verbatim + our defaults; lambdas only shuttle names across namespaces)
+    "biorxiv_details": lambda a: {"from_date": a.from_date, "to_date": a.to_date, "cursor": a.cursor},
+    "crossref_works": lambda a: {"query": a.query, "funder": a.funder, "award_number": a.award_number,
+                                 "rows": a.rows, "offset": a.offset, "cursor": a.cursor,
+                                 "from_pub_date": a.from_pub_date, "to_pub_date": a.to_pub_date},
+    "gb_gtr_projects": lambda a: {"q": a.q, "s": a.s, "p": a.p},
+    "icite_pubs": lambda a: {"pmid": a.pmid},
+    "nih_reporter_projects": lambda a: {"org_name": a.org_name, "fiscal_year": a.fiscal_year,
+                                        "include_fields": [x for x in (a.include_fields or "").split(",") if x.strip()] or None,
+                                        "limit": a.limit, "offset": a.offset},
+    "nih_reporter_publications": lambda a: {"pmids": [int(x) for x in (a.pmids or "").split(",") if x.strip()] or None,
+                                            "appl_ids": [int(x) for x in (a.appl_ids or "").split(",") if x.strip()] or None,
+                                            "limit": a.limit, "offset": a.offset},
+    "nsf_awards": lambda a: {"keyword": a.keyword, "id": a.id, "rpp": a.rpp, "offset": a.offset},
+    "openalex_institutions": lambda a: {"search": a.search, "per_page": a.per_page},
+    "openalex_works": lambda a: {"search": a.search, "ror": a.ror, "doi": a.doi,
+                                 "title_search": a.title_search, "per_page": a.per_page,
+                                 "cursor": a.cursor, "select": a.select},
+    "opencitations_citations": lambda a: {"doi": a.doi},
+}
+
+
+# 0.3.10 jurisdiction chapter: face → jurisdiction (default = issuing authority's jurisdiction; institutions
+# covering global phenomena = GLOBAL; vocabulary ISO-3166 alpha-2 + EU + GLOBAL) —— merged by the root cli.py
+# into _JURISDICTION, the success envelope carries a jurisdiction chapter
+COUNTRY = {
+    "biorxiv_details": "GLOBAL",
+    "crossref_works": "GLOBAL",
+    "gb_gtr_projects": "GB",
+    "icite_pubs": "US",
+    "nih_reporter_projects": "US",
+    "nih_reporter_publications": "US",
+    "nsf_awards": "US",
+    "openalex_institutions": "GLOBAL",
+    "openalex_works": "GLOBAL",
+    "opencitations_citations": "GLOBAL",
+}

agentdata/research/crossref.py

@@ -0,0 +1,121 @@
+"""crossref — Crossref REST API (DOI registry paper search; keyless).
+
+No key needed (the polite pool officially recommends a mailto in the User-Agent; self-imposed
+5 rps = 1/10 of the documented cap of 50, same high-volume tier as edgar/openalex). works face
+(probed 2026-10-26): filter=funder:<funder DOI> collects funder output (NSF 10.13039/100000001
+→ 462,059 works, rows carry the award number in funder[].award[]); award.number:<number> combined
+with funder narrows exactly (OAC-1730655 → 10 works); query= with filter= is legal
+(funder×query=graphene → 2,962).
+Three traps: select=award reports select-not-available (awards live only inside rows, they
+cannot be selected); the deep-paging offset has a documented upstream wall at 10,000 (not probed
+point by point — documented figure) — **the proper way around it is cursor** (deep-pager, probed
+2026-10-28: cursor=*&filter=from-pub-date:2026-06-01 → 5 rows + next-cursor, all good).
+"""
+provider = {
+    "id": "crossref",
+    "baseUrl": "https://api.crossref.org",
+    "rate": {"rps": 5, "burst": 10},  # high-volume tier (edgar also 5rps; official cap 50 rps, quota semantics 10+5=15 ≤ 50, comfortable)
+    "ops": {
+        "works": {"path": "/works", "method": "GET", "idempotent": True},
+    },
+    "ttl": {"works": 24 * 3600},
+}
+
+_ROWS_CAP = 100        # default page-size cap (upstream doc allows up to 1000, politeness tier tightens it)
+_OFFSET_WALL = 10000   # upstream documented deep-paging wall (not probed point by point, documented figure; the cursor channel is not subject to it)
+
+
+def _work_row(it):
+    issued = ((it.get("issued") or {}).get("date-parts") or [[None]])[0]
+    return {
+        "doi": it.get("DOI"),
+        "title": (it.get("title") or [None])[0],
+        "container_title": (it.get("container-title") or [None])[0],
+        "type": it.get("type"),
+        "publisher": it.get("publisher"),
+        "year": issued[0] if issued else None,
+        "authors": [{"given": a.get("given"), "family": a.get("family"),
+                     "affiliation": [x.get("name") for x in a.get("affiliation") or []]}
+                    for a in it.get("author") or []],
+        "funders": [{"doi": f.get("DOI"), "name": f.get("name"), "award": f.get("award") or []}
+                    for f in it.get("funder") or []],
+        "is_referenced_by_count": it.get("is-referenced-by-count"),
+        "url": it.get("URL"),
+    }
+
+
+def works_raw(query=None, *, funder=None, award_number=None, rows=25, offset=0,
+              cursor=None, from_pub_date=None, to_pub_date=None,
+              cache=None, no_cache=False):
+    """→ {total, count, rows, next_cursor, params, _meta}; /works direct, verbatim name.
+    query full-text × funder (funder DOI→filter=funder:) × award_number (→filter=award.number:,
+    only meaningful when funder is given too) × from_pub_date/to_pub_date (→filter=from/until-pub-date:
+    bare date; from=2026-10-28 probed, until is the same family per the docs) can be combined;
+    **constraint = at least one of query/funder/award_number** (local wall: a bare whole-corpus
+    scan is served by the bulk pipeline — a date pair only narrows, it does not count as a
+    constraint, from_pub_date alone still scans the whole corpus and is blocked all the same);
+    rows 1-100 (politeness tightening, upstream doc cap 1000); offset+rows ≤10,000 (upstream doc wall).
+    **cursor (adopted in the 0.2.9 audit)**: deep-pager is the proper deep-paging channel — send
+    `cursor="*"` for the first page, feed back `next_cursor` for the next; mutually exclusive with
+    offset (the cursor channel is not subject to the 10,000 wall); next_cursor is echoed back
+    (null on the last page).
+    Award numbers live only inside funder[].award[] on the row, the select face cannot fetch
+    them (probe 2026-10-26: select-not-available)."""
+    from ..transport import call_provider
+    from ..errors import SdkError
+    import re
+
+    q = str(query or "").strip() or None
+    f_v = str(funder or "").strip() or None
+    a_v = str(award_number or "").strip() or None
+    cur = str(cursor or "").strip() or None
+    if not (q or f_v or a_v):
+        raise SdkError("usage_error", provider="crossref",
+                       detail="crossref_works requires at least one of query/funder/award_number"
+                              " (local safety wall: a bare whole-corpus scan is served by the bulk "
+                              "pipeline; from/to_pub_date is a narrowing pair, not a constraint)")
+    if a_v and not f_v:
+        raise SdkError("usage_error", provider="crossref",
+                       detail="award_number requires funder (the upstream award.number filter only "
+                              "makes sense combined with funder) — pass --funder as well")
+    n = int(rows)
+    if not 1 <= n <= _ROWS_CAP:
+        raise SdkError("usage_error", provider="crossref",
+                       detail=f"rows must be 1-{_ROWS_CAP} (politeness tightening; upstream doc cap 1000)")
+    params = {"rows": n}
+    if cur is not None:
+        params["cursor"] = cur
+    else:
+        off = int(offset)
+        if off < 0:
+            raise SdkError("usage_error", provider="crossref", detail="offset must be ≥0")
+        if off + n > _OFFSET_WALL:
+            raise SdkError("usage_error", provider="crossref",
+                           detail=f"offset+rows must be ≤{_OFFSET_WALL:,} (upstream documented "
+                                  "deep-paging wall) — deep paging goes through the cursor channel "
+                                  "(cursor=\"*\" first, then next_cursor)")
+        params["offset"] = off
+    if q:
+        params["query"] = q
+    filters = []
+    if f_v:
+        filters.append(f"funder:{f_v}")
+    if a_v:
+        filters.append(f"award.number:{a_v}")
+    for key, label, v in (("from_pub_date", "from-pub-date", from_pub_date),
+                          ("to_pub_date", "until-pub-date", to_pub_date)):
+        if v in (None, ""):
+            continue
+        if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", str(v)):
+            raise SdkError("usage_error", provider="crossref",
+                           detail=f"{key} must be a bare date YYYY-MM-DD (e.g. 2026-06-01)")
+        filters.append(f"{label}:{v}")
+    if filters:
+        params["filter"] = ",".join(filters)
+    r = call_provider(provider, "works", params, cache=cache, no_cache=no_cache)
+    msg = (r["data"] or {}).get("message") or {}
+    items = msg.get("items") or []
+    rows_out = [_work_row(it) for it in items]
+    return {"total": msg.get("total-results"), "count": len(rows_out), "rows": rows_out,
+            "next_cursor": msg.get("next-cursor") if cur is not None else None,
+            "params": params, "_meta": r}