agentdata 0.3.26__py3-none-any.whl

agentdata/esg/cli.py

@@ -0,0 +1,180 @@
+"""esg cli — domain command face registration (split per domain in 0.2.6, moved in from the root cli.py; purely flat since the 0.3.0 verb-layer removal).
+
+register(v, g): registers flat commands on the domain subparser (g = the root _leaf_globals factory,
+passed in as a parameter to avoid a cli ↔ domain-cli cycle); FLAT = the flat dispatch table, merged by the root cli for main to use.
+"""
+import argparse
+
+def register(v, g):
+    f1 = v.add_parser("climatetrace_owners", parents=[g()],
+                      help="Emissions-owner candidates (Climate TRACE v7 owners)",
+                      description="Climate TRACE emissions-owner candidates: name/limit/offset upstream-verbatim; zero hit = legal empty (upstream 404 JSON mapped); ordering is NOT stable across limits — fetch enough in one go")
+    f1.add_argument("--name", required=True, help="Owner name to search (upstream param: name)")
+    f1.add_argument("--limit", type=int, default=10, help="Rows (upstream param: limit; politeness cap 50 — upstream tested to 1000)")
+    f1.add_argument("--offset", type=int, default=None, help="Skip rows (upstream param: offset; ordering unstable — prefer one large fetch)")
+
+    f2 = v.add_parser("climatetrace_sources", parents=[g()],
+                      help="Facility-level GHG sources by owner/filter (Climate TRACE v7)",
+                      description="Climate TRACE owner/filter→facility/aggregate emission sources: owner_ids→ownerIds (comma-joined = union & dedup, tested; repeated param = upstream silently first-wins, do not use) XOR --owner-name sugar (resolves first candidate); year no local wall (out-of-window year → upstream null, 2021+ has data); gas verbatim — non-co2e gas switches row key to emissions_quantity (native unit); 0.2.9 audit additions: offset paging, sectors closed-set validated locally (invalid values silently return the WHOLE corpus upstream), subsectors passthrough (upstream 400 self-describes), location family gadm_id/city_id/country_group/continent passthrough (upstream 400 enforces one-at-a-time); owner_ids or at least one filter required (bare corpus scan walled politely)")
+    f2.add_argument("--owner-ids", dest="owner_ids", default=None,
+                    help="Owner id(s), upstream param ownerIds — E100000001492 or comma-joined for union")
+    f2.add_argument("--owner-name", dest="owner_name", default=None,
+                    help="Sugar — resolves first owners candidate (XOR with --owner-ids)")
+    f2.add_argument("--year", type=int, default=None, help="Emissions year (upstream param: year; default=current year; data window 2021+)")
+    f2.add_argument("--gas", default=None, help="Gas (upstream param: gas; default co2e_100yr = tCO2e)")
+    f2.add_argument("--limit", type=int, default=50, help="Rows (upstream param: limit; politeness cap 500)")
+    f2.add_argument("--offset", type=int, default=None,
+                    help="Skip rows (upstream param: offset; real paging tested 2026-10-28 — ordering unstable across limits, prefer one big fetch)")
+    f2.add_argument("--sectors", default=None,
+                    help="Sector slug, closed set validated locally: agriculture/buildings/fluorinated-gases/forestry-and-land-use/fossil-fuel-operations/manufacturing/mineral-extraction/power/transportation/waste + special all_no_forest (invalid values silently return the whole corpus upstream)")
+    f2.add_argument("--subsectors", default=None,
+                    help="Subsector slug(s) passthrough (68 values, GET /v7/definitions/subsectors; invalid value → upstream 400 'invalid subsector provided', self-describing)")
+    f2.add_argument("--gadm-id", dest="gadm_id", default=None, help="GADM region id (upstream param: gadmId)")
+    f2.add_argument("--city-id", dest="city_id", default=None, help="Metro area id (upstream param: cityId)")
+    f2.add_argument("--country-group", dest="country_group", default=None,
+                    help="Country-group slug, upstream param countryGroup — e.g. eu/aosis/g20 (values from GET /v7/definitions/countrygroups)")
+    f2.add_argument("--continent", default=None,
+                    help="Continent name (upstream param: continent; tested Europe/Asia 2026-10-28 — RUS counts as Europe)")
+
+    f3 = v.add_parser("eiti_companies", parents=[g()],
+                      help="Extractives filing companies (EITI Datasette; name candidates)",
+                      description="EITI extractives filing-company candidates: company_name__like is the upstream Datasette magic param verbatim (%/_ metachars yours to manage); --name sugar wraps %contains% (same slot, XOR); zero hit = legal empty")
+    f3.add_argument("--company-name__like", dest="company_name__like", default=None,
+                    help="Upstream param verbatim (SQL LIKE; %%/_ metachars passthrough)")
+    f3.add_argument("--name", default=None, help="Company name — sugar for company_name__like=%%name%% (same slot, XOR with --company-name__like)")
+    f3.add_argument("--size", type=int, default=10, help="Rows (upstream param: _size; politeness cap 1000 — upstream tested to 11000+)")
+
+    f4 = v.add_parser("eiti_payments", parents=[g()],
+                      help="Company→government payment lines (EITI; GFS-classified)",
+                      description="EITI company→government payment lines: id channel (eiti_id_company__exact verbatim XOR --eiti-id sugar adding the eiti_id_company: prefix) + name channel (company_name__like verbatim XOR --name sugar); channels may combine; at least one required (bare scan walled locally)")
+    f4.add_argument("--eiti-id-company__exact", dest="eiti_id_company__exact", default=None,
+                    help="Upstream param verbatim — full value incl. eiti_id_company: prefix")
+    f4.add_argument("--eiti-id", dest="eiti_id", default=None,
+                    help="Sugar — bare UUID, prefix auto-added (same slot, XOR with --eiti-id-company__exact)")
+    f4.add_argument("--company-name__like", dest="company_name__like", default=None,
+                    help="Upstream param verbatim (SQL LIKE)")
+    f4.add_argument("--name", default=None, help="Sugar — %%contains%% (same slot, XOR with --company-name__like)")
+    f4.add_argument("--size", type=int, default=50, help="Rows (upstream param: _size; politeness cap 1000)")
+
+    f5 = v.add_parser("epa_frs_facilities", parents=[g()],
+                      help="EPA facility master lookup (FRS frs_facility_site)",
+                      description="EPA FRS facility registry point query: seven filter columns upstream-verbatim (registry_id / primary_name / state_code / city_name + the 0.3.20 audit trio country_name / site_type_name / epa_region_code — ALL exact '=' match, combinable = AND; the upstream contains/~ operators silently return 0, fuzzy facility names go through ECHO; country_name unlocks the non-US facility slice, GERMANY=110 measured) + start/size paging (ROWS closed interval start:end, computed locally — an omitted/inverted ROWS segment makes the upstream dump the whole table); at least one filter required (bare 5.6M-row scan walled politely, batch pipeline serves that); each call = rows + count (total_matches)")
+    f5.add_argument("--registry-id", dest="registry_id", default=None,
+                    help="FRS Registry ID — authoritative facility key (ECHO/TRI/GHGRP join anchor)")
+    f5.add_argument("--primary-name", dest="primary_name", default=None,
+                    help="Facility primary name — EXACT match incl. spaces (contains/~ operators return 0 upstream)")
+    f5.add_argument("--state-code", dest="state_code", default=None,
+                    help="State code, 2-letter (lowercase input uppercased locally, e.g. DE)")
+    f5.add_argument("--city-name", dest="city_name", default=None, help="City name (exact)")
+    f5.add_argument("--country-name", dest="country_name", default=None,
+                    help="Country name (exact uppercase form, lowercase input uppercased locally — GERMANY=110 / "
+                         "UNITED STATES=3,129,491 measured; the non-US facility slice)")
+    f5.add_argument("--site-type-name", dest="site_type_name", default=None,
+                    help="Site type (exact uppercase, e.g. STATIONARY — 4,725,914 of 5,602,878 measured)")
+    f5.add_argument("--epa-region-code", dest="epa_region_code", default=None,
+                    help="EPA region (zero-padded two-digit string form '03'; region 10 = 214,197 measured)")
+    f5.add_argument("--start", type=int, default=0, help="Paging offset (upstream ROWS start, closed interval)")
+    f5.add_argument("--size", type=int, default=50, help="Rows (politeness cap 500)")
+
+    f6 = v.add_parser("euasl_datasets", parents=[g()],
+                      help="EU Air Safety List download catalog (banned/restricted air carriers)",
+                      description="EU Air Safety List official download catalog (keyless): one call returns all files — the machine-readable XLSX master (Annex A = carriers under an operating ban, Annex B = restrictions/conditions) + 24 official-language PDF renditions; the document UUIDs rotate on every republication (reconcile against this listing, never guess file names); zero params upstream; the binary bodies go through the batch pipeline, this face gives coordinates")
+    # Upstream takes zero params: no flags (the catalog ships in one shot; clients pick the xlsx master row)
+
+    f7 = v.add_parser("eutl_datasets", parents=[g()],
+                      help="EU carbon registry bulk-download catalog (EUTL)",
+                      description="EUTL official bulk-download catalog (keyless): one call returns all datasets — Azure daily extracts (accounts/operators/holdings csv.gz) + climate.ec.europa.eu yearly files (compliance/verified emissions 2021-2025); zero params upstream; file downloads themselves go through the batch pipeline, this face gives coordinates")
+    # Upstream takes zero params: no flags (the catalog ships in one shot; clients filter rows themselves)
+
+    # esg 0.3.5 port-state emissions/supervision axis (thetis_mrv + paris_mou; upstream param names mirrored) — folded in and seated lexicographically in 0.3.10
+    f8 = v.add_parser("paris_mou_bans", parents=[g()],
+                      help="Paris MoU banned ships register",
+                      description="Paris MoU banned-ships register (keyless): ban date/occurrence/order status/reason/minimum duration/banning authority, with the embedded ISM company flattened to row keys — the official ship x management-company edge (~2/3 of rows carry a company IMO number); zero params upstream")
+    # Upstream takes zero params: no flags (the bans list ships in one shot)
+
+    f9 = v.add_parser("paris_mou_detentions", parents=[g()],
+                      help="Paris MoU current detentions snapshot",
+                      description="Paris MoU (Atlantic/European port state control) currently-detained ships snapshot (keyless): daily-rolling live state — ship IMO number (asset key), detention port+country, reporting authority; historical detentions go through paris_mou_inspections window search; zero params upstream")
+    # Upstream takes zero params: no flags (the same-day snapshot ships in one shot)
+
+    f10 = v.add_parser("paris_mou_inspections", parents=[g()],
+                      help="Paris MoU historical PSC inspection window search",
+                      description="Paris MoU historical port state control inspections (THETIS form-encoded POST, keyless): period_min/period_max are dd/mm/yyyy STRICT (upstream native order day/month/year — wrong order silently returns zero) and the window must be <= 3 years (enforced locally, upstream silently empties beyond); page is the paging switch — the face always sends it (without page upstream ignores limit and dumps the whole window); page+start+limit = start-offset paging (page 1/2 verified zero overlap); ism_company_imo_number / ism_company_name filter straight to the management company (resume window stats ships/inspections/detentions/ISM deficiencies ride along); psc_inspection_regime upstream-verbatim (1 = PSC default regime, only verified value; omitted = upstream 200 empty); zero rows in window = legal empty")
+    f10.add_argument("--period-min", dest="period_min", required=True,
+                    help="Window start dd/mm/yyyy (upstream param periodMin; STRICT day-first)")
+    f10.add_argument("--period-max", dest="period_max", required=True,
+                    help="Window end dd/mm/yyyy (upstream param periodMax; window <= 3 years)")
+    f10.add_argument("--psc-inspection-regime", dest="psc_inspection_regime", type=int, default=1,
+                    help="Inspection regime (upstream param pscInspectionRegime; 1 = PSC default, only verified value)")
+    f10.add_argument("--page", type=int, default=1,
+                    help="Page number (upstream param page; the paging switch — always sent by this face)")
+    f10.add_argument("--start", type=int, default=0,
+                    help="Row offset within page (upstream param start; page+start+limit paging)")
+    f10.add_argument("--limit", type=int, default=50, help="Rows (politeness cap 500)")
+    f10.add_argument("--ism-company-name", dest="ism_company_name", default=None,
+                    help="ISM management company name filter (upstream param ismCompanyName)")
+    f10.add_argument("--ism-company-imo-number", dest="ism_company_imo_number", default=None,
+                    help="ISM company IMO number filter, 7-digit zero-padded (upstream param ismCompanyImoNumber, e.g. 0052193)")
+
+    f11 = v.add_parser("paris_mou_low_performance_companies", parents=[g()],
+                       help="Paris MoU low-performance ISM companies",
+                       description="Paris MoU low-performance ISM company list (rolling 6-month window, keyless): each row carries an IMO company number (7-digit zero-padded official string), with address/country/performance level (LOW dominant, some MEDIUM)/listing dates — company-risk attribute list; zero params upstream")
+    # Upstream takes zero params: no flags (the rolling-window list ships in one shot)
+
+    f12 = v.add_parser("thetis_mrv_datasets", parents=[g()],
+                       help="EU shipping MRV emission-report file catalog (THETIS-MRV)",
+                       description="THETIS-MRV versioned bulk-file catalog (keyless): reporting periods (2018+) x versioned regenerated XLSX publications (versions roll — reconcile by this list, never guess filenames; generation timestamps are early-morning batch runs); download_path template is served for the batch pipeline, this face gives coordinates; zero params upstream")
+    # Upstream takes zero params: no flags (reporting-period × version ledger composed; clients filter rows themselves)
+
+FLAT = {
+    # esg 0.2.0 flat panel (upstream params under original names + our defaults; the lambda only ferries names across namespaces)
+    "climatetrace_owners": lambda a: {"name": a.name, "limit": a.limit, "offset": a.offset},
+    "climatetrace_sources": lambda a: {"owner_ids": a.owner_ids, "owner_name": a.owner_name,
+                                       "year": a.year, "gas": a.gas, "limit": a.limit,
+                                       "offset": a.offset, "sectors": a.sectors,
+                                       "subsectors": a.subsectors, "gadm_id": a.gadm_id,
+                                       "city_id": a.city_id, "country_group": a.country_group,
+                                       "continent": a.continent},
+    "eiti_companies": lambda a: {"company_name__like": a.company_name__like,
+                                 "name": a.name, "size": a.size},
+    "eiti_payments": lambda a: {"eiti_id_company__exact": a.eiti_id_company__exact,
+                                "eiti_id": a.eiti_id, "company_name__like": a.company_name__like,
+                                "name": a.name, "size": a.size},
+    "epa_frs_facilities": lambda a: {"registry_id": a.registry_id,
+                                     "primary_name": a.primary_name,
+                                     "state_code": a.state_code, "city_name": a.city_name,
+                                     "country_name": a.country_name, "site_type_name": a.site_type_name,
+                                     "epa_region_code": a.epa_region_code,
+                                     "start": a.start, "size": a.size},
+    "euasl_datasets": lambda a: {},  # upstream zero params (download catalog in one shot; 0.3.22)
+    "eutl_datasets": lambda a: {},  # upstream zero params (catalog in one shot)
+    # esg 0.3.5 port-state emissions/supervision axis (thetis_mrv + paris_mou; upstream param names mirrored)
+    "paris_mou_bans": lambda a: {},        # upstream zero params (bans list in one shot)
+    "paris_mou_detentions": lambda a: {},  # upstream zero params (same-day snapshot in one shot)
+    "paris_mou_inspections": lambda a: {"period_min": a.period_min,
+                                        "period_max": a.period_max,
+                                        "psc_inspection_regime": a.psc_inspection_regime,
+                                        "page": a.page, "start": a.start, "limit": a.limit,
+                                        "ism_company_name": a.ism_company_name,
+                                        "ism_company_imo_number": a.ism_company_imo_number},
+    "paris_mou_low_performance_companies": lambda a: {},  # upstream zero params (rolling-window list in one shot)
+    "thetis_mrv_datasets": lambda a: {},  # upstream zero params (reporting-period × version ledger composed)
+}
+
+
+# 0.3.10 jurisdiction chapter: face → jurisdiction (default = the issuing authority's jurisdiction; bodies
+# covering global phenomena = GLOBAL; vocabulary = ISO-3166 alpha-2 + EU + GLOBAL) — merged into _JURISDICTION by the root cli.py; success envelopes carry the jurisdiction chapter
+COUNTRY = {
+    "climatetrace_owners": "GLOBAL",
+    "climatetrace_sources": "GLOBAL",
+    "eiti_companies": "GLOBAL",
+    "eiti_payments": "GLOBAL",
+    "epa_frs_facilities": "US",
+    "euasl_datasets": "EU",
+    "eutl_datasets": "EU",
+    "paris_mou_bans": "GLOBAL",
+    "paris_mou_detentions": "GLOBAL",
+    "paris_mou_inspections": "GLOBAL",
+    "paris_mou_low_performance_companies": "GLOBAL",
+    "thetis_mrv_datasets": "EU",
+}

agentdata/esg/climatetrace.py

@@ -0,0 +1,260 @@
+"""climatetrace — Climate TRACE v7 provider (api.climatetrace.org; keyless).
+
+Counterpart to the Node implementation (esg domain Wave 3: trace_owner + trace_sources — the
+facility/asset-level GHG emissions two-hop chain). 2026-10-23 measured shape
+(v7 = the geo-aggregation rework; the openapi.json endpoint serves YAML not JSON — do not json.load it):
+GET /v7/owners?name=<q>&limit=N → [{id:"E100000001492", name:"Gazprom PJSC"}] — **bare array, only id+name**,
+a LIKE-style candidate face (the E-prefixed id is the stable company key; same pattern as EITI eiti_name:
+candidates are not authoritative normalization);
+GET /v7/sources?ownerIds=<E…>&year=YYYY → bare array: {id,name,sector,subsector,sourceType(point-source|aggregate),
+country,emissionsQuantity(tCO2e),gas,year,capacity,capacityUnits,activity,activityUnits,assetType,
+centroid:{latitude,longitude}} — 19 columns all verified by probing.
+Parameter window: year data window 2021–current year (default current year), gas defaults to co2e_100yr.
+2026-10-23 wall-probing batch:
+**owners zero hits = 404 JSON** ({"detail":"No owners matched…"}) not a 200 empty array — after the op declares
+notFound it maps to a legal empty candidate list (previously mis-raised as upstream_unavailable); limit has no cap
+(1000 really returned 991 rows), **ordering is unstable across limits** (limit=5 first 5 ≠ limit=1000 first 5 —
+offset paging may skip or duplicate; on the candidate face fetch enough in one go);
+**sources does not error on out-of-window years**: year=2020 → 200 with a null body (data-window guidance, not a block);
+**gas goes straight through by name**: gas=ch4 returns the same shape, the gas field echoes ch4, and
+emissionsQuantity is that gas's native unit, not tCO2e;
+**multiple ownerIds comma-joined = union with dedup** (12∪2→13, a source shared across owners appears once);
+a repeated param = silent first-wins (only the first ownerIds counts, the rest are dropped — a trap shape, do not use).
+v7 dropped v6's assets endpoint (/v6/assets still 200 for backwards compatibility); EUTL is the EU statutory registry,
+TRACE covers global point sources — complementary, not substitutes.
+TTL: owners 30 days (stable-key face) / sources 30 days (annual data rolls slowly); politeness tier 1 rps.
+"""
+import re
+from datetime import datetime, timezone
+
+from ..errors import SdkError
+
+provider = {
+    "id": "climatetrace",
+    "baseUrl": "https://api.climatetrace.org",
+    "rate": {"rps": 1, "burst": 3},
+    "ops": {
+        # zero hits = 404 JSON "No owners matched" (measured 2026-10-23) → declared notFound, then mapped to a legal empty set
+        "owners": {"path": "/v7/owners", "notFound": [404]},    # params: name, limit, offset
+        "sources": {"path": "/v7/sources"},  # params: ownerIds (comma = union), year, gas, limit
+    },
+    "ttl": {"owners": 30 * 86400, "sources": 30 * 86400},
+}
+
+_E_ID = re.compile(r"^E\d+$")
+
+# sectors closed set (probed 2026-10-28 at /v7/definitions/sectors, 10 values + the special value all_no_forest).
+# The closed set comes from probing upstream — do not generalize from first-seen samples; invalid values make the
+# upstream **silently return the WHOLE corpus** (sectors=electricity returns a row set bit-for-bit identical to the
+# unfiltered one — a false-negative trap), so it is intercepted locally (same discipline as osv ecosystem).
+SECTORS = {
+    "agriculture", "buildings", "fluorinated-gases", "forestry-and-land-use",
+    "fossil-fuel-operations", "manufacturing", "mineral-extraction",
+    "power", "transportation", "waste",
+}
+SECTORS_SPECIAL = {"all_no_forest"}  # excludes forestry-and-land-use (openapi description verbatim; verified live 2026-10-28)
+
+
+def owners_from_upstream(data, size):
+    """Upstream response -> company candidates (pure function, offline-testable; owners echoes no total — a truncated list)."""
+    items = data if isinstance(data, list) else []
+    return {
+        "candidates": [{"trace_owner_id": o.get("id"),    # E-prefixed stable company key (anchor for the subsequent sources query)
+                        "owner_name": o.get("name")} for o in items],
+        "limit_applied": size,
+    }
+
+
+def sources_from_upstream(data, *, tco2e=True):
+    """Upstream response -> emission-source rows (pure function, offline-testable).
+
+    tco2e=True (default, gas=co2e_100yr basis): row key emissions_tco2e / total emissions_tco2e_sum —
+    the frozen face name stays; tco2e=False (explicit non-co2e gas): row key switches to
+    emissions_quantity / emissions_quantity_sum — that unit is the gas's native unit, not tCO2e
+    (measured 2026-10-23, gas=ch4, same shape); a key name that lies is worse than a switched key."""
+    qkey = "emissions_tco2e" if tco2e else "emissions_quantity"
+    rows = [{
+        "source_id": s.get("id"),                        # stable key of the facility/aggregate source
+        "name": s.get("name"),
+        "sector": s.get("sector"), "subsector": s.get("subsector"),
+        "source_type": s.get("sourceType"),              # point-source = single facility / non-point = aggregate
+        "country": s.get("country"),
+        "year": s.get("year"), "gas": s.get("gas"),      # gas defaults to co2e_100yr (100-year GWP basis)
+        qkey: s.get("emissionsQuantity"),                # emissions (basis follows gas; key name follows basis)
+        "capacity": s.get("capacity"), "capacity_units": s.get("capacityUnits"),
+        "capacity_factor": s.get("capacityFactor"),      # 0.2.9 audit addition (SourceSummary 19-column alignment)
+        "emissions_factor": s.get("emissionsFactor"),    # 0.2.9 audit addition
+        "emissions_factor_units": s.get("emissionsFactorUnits"),  # 0.2.9 audit addition
+        "activity": s.get("activity"), "activity_units": s.get("activityUnits"),
+        "asset_type": s.get("assetType"),
+        "location": ({"lat": (s.get("centroid") or {}).get("latitude"),
+                      "lon": (s.get("centroid") or {}).get("longitude"),  # SRID 4326
+                      "srid": (s.get("centroid") or {}).get("srid")}
+                     if s.get("centroid") else None),
+    } for s in (data if isinstance(data, list) else [])]
+    total = sum(r[qkey] for r in rows if isinstance(r[qkey], (int, float)))
+    return {"rows": rows, "count": len(rows),
+            f"{qkey}_sum": round(total, 2) if rows else None}
+
+
+def search_owners(name, *, limit=10, cache=None, no_cache=False):
+    """-> {candidates[], limit_applied, _meta}; company candidates (LIKE search).
+
+    Zero hits = upstream 404 JSON (measured 2026-10-23) — after the notFound declaration this becomes
+    a legal empty candidate list (not an error)."""
+    from ..transport import call_provider
+
+    q = str(name or "").strip()
+    if not q:
+        raise SdkError("usage_error", provider="climatetrace", detail="search_owners requires name (company name)")
+    size = min(max(1, int(limit or 10)), 50)
+    try:
+        r = call_provider(provider, "owners", {"name": q, "limit": size}, cache=cache, no_cache=no_cache)
+    except SdkError as e:
+        if e.code == "upstream_not_found":
+            # 404="No owners matched": legal empty (notFound is not cached, meta minted on the spot — same convention as the gleif parent-company face)
+            return {"candidates": [], "limit_applied": size, "_meta": {
+                "from_cache": False,
+                "fetched_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")}}
+        raise
+    return {**owners_from_upstream(r["data"], size), "_meta": r}
+
+
+def owners_raw(name, *, limit=10, offset=None, cache=None, no_cache=False):
+    """0.2.0 flat face internals: name/limit/offset straight to upstream under their own names (zero hits -> empty candidates).
+
+    -> {candidates[], limit_applied, params, _meta} — params are exactly the params sent upstream (echoed, replayable).
+    Ordering is unstable across limits with offset (measured 2026-10-23), paging may skip or
+    duplicate — on the candidate face fetch enough in one go."""
+    from ..transport import call_provider
+
+    q = str(name or "").strip()
+    if not q:
+        raise SdkError("usage_error", provider="climatetrace", detail="owners_raw requires name (company name, upstream param name)")
+    size = min(max(1, int(limit or 10)), 50)  # politeness tier 50 (upstream has no cap; measured 2026-10-23, limit=1000 really returned 991 rows)
+    params = {"name": q, "limit": size}
+    if offset is not None:
+        params["offset"] = int(offset)
+    try:
+        r = call_provider(provider, "owners", params, cache=cache, no_cache=no_cache)
+    except SdkError as e:
+        if e.code == "upstream_not_found":
+            return {"candidates": [], "limit_applied": size, "params": params, "_meta": {
+                "from_cache": False,
+                "fetched_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")}}
+        raise
+    return {**owners_from_upstream(r["data"], size), "params": params, "_meta": r}
+
+
+def sources_raw(owner_ids=None, *, year=None, gas=None, limit=50, offset=None,
+                sectors=None, subsectors=None, gadm_id=None, city_id=None,
+                country_group=None, continent=None, cache=None, no_cache=False):
+    """0.2.0 flat face internals; 0.2.9 audit parameter expansion: offset/sectors/subsectors/four location
+    params folded in (2026-10-28 openapi + live probing).
+
+    -> {rows[], count, <emissions_*_sum>, resolved_owner, year, gas, params, _meta}.
+    owner_ids: a single E key or comma-joined multiples (comma = union with dedup, measured 2026-10-23
+    12∪2→13; a repeated param = upstream silent first-wins trap, do not use); year has no local wall —
+    an out-of-window year makes upstream return null, not an error (measured 2026-10-23 year=2020→200 null,
+    data window starts 2021); gas defaults to co2e_100yr (row key emissions_tco2e), an explicit non-co2e
+    gas switches the row key to emissions_quantity (native unit, measured 2026-10-23 gas=ch4, same shape).
+    0.2.9 new params (all upstream names):
+    - offset: real paging (probed 2026-10-28, offset=0/5 row sets have zero overlap; also effective within
+      the ownerIds domain) — note ordering is unstable across limits (0.2.3 wall), offset paging may skip
+      or duplicate; on the audit face fetch enough in one go.
+    - sectors: 10-value closed set validated locally (SECTORS; + the special value all_no_forest =
+      excludes forestry) — invalid values make the upstream **silently return the WHOLE corpus**
+      (sectors=electricity returns a row set bit-for-bit identical to the unfiltered one, a
+      false-negative trap, probed 2026-10-28); valid values really filter (forestry-and-land-use 5/5
+      hits, all_no_forest top 10 has no forestry).
+    - subsectors: pass-through, no local closed set (68-value enum at /v7/definitions/subsectors;
+      invalid values get an upstream 400 self-describing "invalid subsector provided", not silent —
+      probed 2026-10-28, upstream self-validation suffices).
+    - four location params gadm_id/city_id/country_group/continent: pass-through (continent=Europe/Asia
+      probed, filtering effective — RUS counts as Europe, not Asia; countryGroup takes a group slug from
+      /v7/definitions/countrygroups, e.g. eu; eu probed, 5/5 rows in the EU list); upstream answers two
+      concurrent location params with a 400 self-describing the four-param enum ("choose no more than
+      one of: gadmId, countryGroup, continent, cityId") — mutual exclusion is upstream-proven, no local
+      wall is set.
+    - soft wall: at least one of owner_ids x sectors x subsectors x a location param — upstream allows a
+      bare whole-corpus query (year alone already returns a 200 with the whole corpus), bulk corpus scans
+      go through the batch pipeline service; this face's politeness tier intercepts.
+    Row shape completed in 0.2.9: capacity_factor/emissions_factor/emissions_factor_units +
+    location.srid (SourceSummary 19-column alignment, probed 2026-10-28, 18 keys all present)."""
+    from ..transport import call_provider
+
+    oid = str(owner_ids or "").strip()
+    parts = []
+    if oid:
+        parts = [p.strip() for p in oid.split(",") if p.strip()]
+        for p in parts:
+            if not _E_ID.match(p):
+                raise SdkError("usage_error", provider="climatetrace",
+                               detail=f"owner_ids looks like E100000001492 (comma-separated for multiple owners; got {p!r})")
+    sec = str(sectors).strip() if sectors is not None else None
+    if sec is not None and sec not in SECTORS | SECTORS_SPECIAL:
+        raise SdkError("usage_error", provider="climatetrace",
+                       detail=f"sectors {sec!r} not in the closed set ("
+                              + ", ".join(sorted(SECTORS | SECTORS_SPECIAL)) +
+                              "; invalid values make the upstream silently return the whole corpus — the false-negative trap is intercepted here; enum = /v7/definitions/sectors, probed 2026-10-28)")
+    try:
+        want_year = int(year) if year is not None else datetime.now(timezone.utc).year
+    except (TypeError, ValueError):
+        raise SdkError("usage_error", provider="climatetrace", detail=f"year must be an integer year (got {year!r})")
+    g = str(gas).strip() if gas is not None else None
+    size = min(max(1, int(limit or 50)), 500)  # politeness tier 500 (upstream documents no paging param)
+    loc = {"gadmId": gadm_id, "cityId": city_id, "countryGroup": country_group, "continent": continent}
+    loc_params = {k: str(v).strip() for k, v in loc.items() if v not in (None, "")}
+    if not parts and not sec and subsectors in (None, "") and not loc_params:
+        raise SdkError("usage_error", provider="climatetrace",
+                       detail="sources_raw requires at least one of owner_ids (E key), sectors/subsectors, "
+                              "or a location param (gadm_id/city_id/country_group/continent) — a bare "
+                              "whole-corpus scan goes through the batch pipeline service "
+                              "(upstream allows it, but this face's politeness tier intercepts bulk corpus scans)")
+    params = {"year": want_year, "limit": size}
+    if parts:
+        params["ownerIds"] = ",".join(parts)
+    if offset is not None:
+        params["offset"] = int(offset)
+    if sec is not None:
+        params["sectors"] = sec
+    if subsectors not in (None, ""):
+        params["subsectors"] = str(subsectors).strip()
+    params.update(loc_params)
+    if g is not None:
+        params["gas"] = g
+    r = call_provider(provider, "sources", params, cache=cache, no_cache=no_cache)
+    tco2e = g in (None, "co2e_100yr")
+    out = {**sources_from_upstream(r["data"], tco2e=tco2e),
+           "year": want_year, "gas": g or "co2e_100yr", "params": params, "_meta": r}
+    if parts:
+        out["resolved_owner"] = {"trace_owner_ids": parts}
+    return out
+
+
+def owner_sources(owner_id=None, owner_name=None, *, year=None, limit=50, cache=None, no_cache=False):
+    """-> {rows[], count, emissions_tco2e_sum, resolved_owner, year, _meta}; company emission-source rows.
+    owner_id (E key) is the authoritative anchor; owner_name is fallback only, resolving to an id first
+    (first candidate — same fallback semantics as eiti name: not authoritative normalization)."""
+    from ..transport import call_provider
+
+    oid = str(owner_id or "").strip()
+    want_year = int(year) if year else datetime.now(timezone.utc).year
+    if want_year < 2021:
+        raise SdkError("usage_error", provider="climatetrace", detail="year must be a year >= 2021")
+    if not oid:
+        name = str(owner_name or "").strip()
+        if not name:
+            raise SdkError("usage_error", provider="climatetrace",
+                           detail="owner_sources requires trace_owner_id (E key) or name (fallback: resolve to an id first)")
+        s = search_owners(name, limit=5, cache=cache, no_cache=no_cache)
+        if not s["candidates"]:
+            return {**sources_from_upstream([]), "resolved_owner": None, "_meta": s["_meta"]}
+        oid = s["candidates"][0]["trace_owner_id"]
+    if not _E_ID.match(oid):
+        raise SdkError("usage_error", provider="climatetrace", detail=f"trace_owner_id looks like E100000001492 (got {oid!r})")
+    size = min(max(1, int(limit or 50)), 500)  # upstream exposes no explicit paging param — politeness tier truncates at 500
+    r = call_provider(provider, "sources", {"ownerIds": oid, "year": want_year, "limit": size},
+                      cache=cache, no_cache=no_cache)
+    return {**sources_from_upstream(r["data"]), "resolved_owner": {"trace_owner_id": oid},
+            "year": want_year, "_meta": r}

agentdata/esg/eiti.py

@@ -0,0 +1,187 @@
+"""eiti — EITI extractives transparency database provider (Datasette instance; keyless).
+
+Counterpart to the Node implementation (esg domain Wave 3, P0). eiti-database.eiti.org.
+2026-10-20 measured shape: view path /eiti_database/{view}.json?_size=N&_shape=objects&<col>__like=<pattern>
+(**the SQL face ?sql= errors out on views — unusable** — a view is not a table; filtering goes through
+the Datasette magic params __like/__exact);
+real column names: eiti_id_company values look like "eiti_id_company:<UUID>" (with the prefix!),
+hq_country_name, first_year_reported/last_year_reported; with _shape=objects the rows are already
+dicts (do not zip columns again).
+view_payments_detailed has 104,484 rows, view_companies has 10,200 companies; companies carry no local
+registration number (the UUID is the stable key).
+2026-10-21 audit batch B probe (gates 1-2):
+**_size has no 100 cap** (the earlier note "upstream really caps at 100" was a 0.1.8 mis-recording) —
+measured 11,000 rows per call sent as-is, not clamped;
+our politeness cap is 1,000 (the realtime single-query face does not chase bulk; bulk goes through the
+corpus batch pipeline). `_extra=count` is officially supported:
+it returns count = total matching rows ignoring paging, but **count itself is capped at 10,001**
+(count_truncated=true means ">=10,001").
+__like= is SQL LIKE (%/_ metacharacters apply to user input verbatim; ASCII case-insensitive as measured),
+__exact = equality.
+Column fill rates (Equinor slice, 358 rows measured): gov_entity_name 100% / gfs_code 92.5% /
+payment_value_usd 88.3% (in-kind payments etc. have no USD) / project_name 50.6% (across the whole
+surface, not 23%).
+companies face: hq_country_name fill rate is only 62/10,201 (hq_country is mostly null); estma_id has
+90 companies = ESTMA (Extractive Industries Transparency Standard) filers, **including foreign-owned
+companies operating in Canada** (AngloGold Ghana etc.), not "Canadian companies only".
+TTL 7 days (last_updated rolls slowly); politeness tier 0.5 rps.
+"""
+from ..errors import SdkError
+
+provider = {
+    "id": "eiti",
+    "baseUrl": "https://eiti-database.eiti.org",
+    "rate": {"rps": 0.5, "burst": 2},
+    "ops": {
+        "companies": {"path": "/eiti_database/view_companies.json"},        # params: company_name__like + _size/_shape/_extra
+        "payments": {"path": "/eiti_database/view_payments_detailed.json"},  # params: eiti_id_company__exact | company_name__like + _size/_shape/_extra
+    },
+    "ttl": {"companies": 7 * 86400, "payments": 7 * 86400},
+}
+
+
+def _rows_of(data):
+    return (data or {}).get("rows") if isinstance(data, dict) else (data if isinstance(data, list) else [])
+
+
+def _norm_id(v):
+    """eiti_id_company official values carry the "eiti_id_company:" prefix — add it when the caller passes a bare UUID."""
+    s = str(v or "").strip()
+    if not s:
+        return ""
+    return s if s.startswith("eiti_id_company:") else f"eiti_id_company:{s}"
+
+
+def _polite_size(v, dflt):
+    """Our politeness cap is 1,000 (the realtime single-query face does not chase bulk) — upstream has no hard 100 limit as measured (probe 2026-10-21, >=11,000 per call)."""
+    return min(max(1, int(v or dflt)), 1000)
+
+
+def total_display(n, capped):
+    """Display form of the upstream count: count_truncated=true means count itself is capped at 10,001 -> display ">10000"."""
+    if n is None:
+        return "?"
+    return f">{n - 1}" if capped else str(n)
+
+
+def companies_from_upstream(data, size):
+    """Upstream response -> companies face result (pure function, offline-testable)."""
+    candidates = [{
+        "eiti_id_company": _norm_id(c.get("eiti_id_company")),  # own stable UUID (no local registration number)
+        "company_name": c.get("company_name"),
+        "hq_country": c.get("hq_country_name"),                 # fill rate only 62/10,201 (measured 2026-10-21) — mostly null
+        "sectors": c.get("sectors"),
+        "countries_of_operation": c.get("countries_of_operation"),
+        "estma_id": c.get("estma_id"),                          # ESTMA filer id; filers include foreign-owned companies operating in Canada
+        "first_year_reported": c.get("first_year_reported"),
+        "last_year_reported": c.get("last_year_reported"),
+    } for c in _rows_of(data)]
+    return {
+        "candidates": candidates,
+        "limit_applied": size,                                  # truncation visible (limit contract)
+        "total_matches": data.get("count") if isinstance(data, dict) and isinstance(data.get("count"), int) else None,
+        "total_matches_capped": bool(isinstance(data, dict) and data.get("count_truncated")),
+    }
+
+
+def payments_from_upstream(data, size):
+    """Upstream response -> payments face result (pure function, offline-testable)."""
+    rows = [{
+        "eiti_id_company": _norm_id(p.get("eiti_id_company")),
+        "company_name": p.get("company_name"),
+        "year": p.get("year"),
+        "country_iso3": p.get("country_iso3"),              # country where the payment happened (resource country)
+        "project_name": p.get("project_name"),              # fill rate about half (Equinor slice 50.6%, measured 2026-10-21)
+        "gov_entity": p.get("gov_entity_name"),             # receiving government body (named down to bureau level; 100% non-null in the slice)
+        "revenue_stream": p.get("revenue_stream_name"),     # tax type, upstream verbatim name
+        "gfs_code": p.get("gfs_code"), "gfs_name": p.get("gfs_name"),  # IMF GFS standard classification (92.5% non-null in the slice)
+        "payment_value": p.get("payment_value"),            # local currency
+        "payment_value_usd": p.get("payment_value_usd"),    # official conversion (basis varies across filing batches; 88.3% non-null in the slice)
+        "payment_in_kind": p.get("payment_in_kind"),        # in-kind payment flag
+    } for p in _rows_of(data)]
+    return {
+        "rows": rows, "count": len(rows),
+        "limit_applied": size,                              # truncation visible (limit contract)
+        "total_rows": data.get("count") if isinstance(data, dict) and isinstance(data.get("count"), int) else None,
+        "total_rows_capped": bool(isinstance(data, dict) and data.get("count_truncated")),
+    }
+
+
+def search_companies(name, *, limit=10, cache=None, no_cache=False):
+    """-> {candidates[], limit_applied, total_matches, _meta}; extractives company candidates
+    (name LIKE search, ASCII case-insensitive; %/_ metacharacters apply verbatim; candidates are not authoritative normalization)."""
+    from ..transport import call_provider
+
+    q = str(name or "").strip()
+    if not q:
+        raise SdkError("usage_error", provider="eiti", detail="search_companies requires name")
+    size = _polite_size(limit, 10)
+    r = call_provider(provider, "companies", {
+        "company_name__like": f"%{q}%",
+        "_size": size,
+        "_shape": "objects",
+        "_extra": "count",  # total matches ignoring paging (capped at 10,001, flagged by total_matches_capped)
+    }, cache=cache, no_cache=no_cache)
+    return {**companies_from_upstream(r["data"], size), "_meta": r}
+
+
+def company_payments(eiti_id=None, name=None, *, limit=50, cache=None, no_cache=False):
+    """-> {rows[], count, limit_applied, total_rows, _meta}; company->government payment rows (eiti_id is the authoritative anchor; name is fallback only).
+    Rich row keys: gov_entity_name named down to bureau level + gfs_code (IMF GFS standard revenue classification — the key for cross-country comparability).
+    total_rows = upstream _extra=count (capped at 10,001, flagged by total_rows_capped) — the in-page sum is only a subtotal, not the company's total (lesson P1-3)."""
+    from ..transport import call_provider
+
+    eid = _norm_id(eiti_id)
+    q = str(name or "").strip()
+    if not eid and not q:
+        raise SdkError("usage_error", provider="eiti", detail="company_payments requires eiti_id or name")
+    params = {"eiti_id_company__exact": eid} if eid else {"company_name__like": f"%{q}%"}
+    size = _polite_size(limit, 50)
+    r = call_provider(provider, "payments", {
+        **params,
+        "_size": size,
+        "_shape": "objects",
+        "_extra": "count",  # total rows ignoring paging (capped at 10,001, flagged by total_rows_capped)
+    }, cache=cache, no_cache=no_cache)
+    return {**payments_from_upstream(r["data"], size), "_meta": r}
+
+
+def companies_raw(company_name__like, *, size=10, cache=None, no_cache=False):
+    """0.2.0 flat face internals: company_name__like straight to upstream under its own name (%/_ metacharacters yours to manage).
+
+    -> {candidates[], limit_applied, total_matches, total_matches_capped, params, _meta}
+    — params are exactly the params sent upstream (including the _size/_shape/_extra magic params)."""
+    from ..transport import call_provider
+
+    like = str(company_name__like or "").strip()
+    if not like:
+        raise SdkError("usage_error", provider="eiti",
+                       detail="companies_raw requires company_name__like (Datasette magic param, upstream name; %/_ yours to manage)")
+    sz = _polite_size(size, 10)
+    params = {"company_name__like": like, "_size": sz, "_shape": "objects", "_extra": "count"}
+    r = call_provider(provider, "companies", params, cache=cache, no_cache=no_cache)
+    return {**companies_from_upstream(r["data"], sz), "params": params, "_meta": r}
+
+
+def payments_raw(eiti_id_company__exact=None, company_name__like=None, *, size=50,
+                 cache=None, no_cache=False):
+    """0.2.0 flat face internals: eiti_id_company__exact / company_name__like straight to upstream under their own names.
+
+    Both filters may be combined (AND across different upstream columns); at least one is required by a client-layer wall
+    (a bare scan of all 104,484 rows belongs to the batch pipeline).
+    -> {rows[], count, limit_applied, total_rows, total_rows_capped, params, _meta}."""
+    from ..transport import call_provider
+
+    eid = str(eiti_id_company__exact or "").strip() or None
+    like = str(company_name__like or "").strip() or None
+    if not eid and not like:
+        raise SdkError("usage_error", provider="eiti",
+                       detail="payments_raw requires at least one of eiti_id_company__exact or company_name__like")
+    sz = _polite_size(size, 50)
+    params = {"_size": sz, "_shape": "objects", "_extra": "count"}
+    if eid:
+        params["eiti_id_company__exact"] = eid
+    if like:
+        params["company_name__like"] = like
+    r = call_provider(provider, "payments", params, cache=cache, no_cache=no_cache)
+    return {**payments_from_upstream(r["data"], sz), "params": params, "_meta": r}