agentdata 0.3.26__py3-none-any.whl

agentdata/esg/epa_frs.py

@@ -0,0 +1,138 @@
+"""epa_frs — EPA FRS facility master registry Envirofacts efservice provider (keyless).
+
+FRS (Facility Registry Service) = the authoritative key source of the shared facility master registry in the
+esg domain (ECHO/TRI/GHGRP converge on the Registry ID); this face opens a facility point query:
+the frs_facility_site table has 5,602,878 rows and 52 columns.
+Measured shape (probed 2026-10-29):
+- Real host **data.epa.gov/efservice** (enviro.epa.gov redirects 301 to the same path; the old
+  ofmpub.epa.gov is 404).
+- Filter syntax = **column/operator/value as three separate path segments**: `/STATE_CODE/=/DE/` — a
+  single segment `/=DE/` silently returns []; the table name must be lowercase (FRS_FACILITY_SITE in
+  uppercase = silent empty result, not an error); multiple filters chain into AND
+  (STATE_CODE+City_NAME hit as measured); values are URL-encoded (space->%20, exact names verified).
+- Paging `/ROWS/{start}:{end}/` is a **closed interval** (0:2 = 3 rows); **when end<start the upstream
+  silently degrades to offset=start with an unbounded dump** (ROWS/3:2 measured 18,791 rows = DE
+  18,794−3) — this face pins end=start+size−1, which is always valid, and never omits the ROWS segment
+  (omitting it dumps the whole 5.6M-row table).
+- Operator lineage: the `=` family works; **`~`/`!~`/LIKE silently return 0** (do not use); **a value
+  containing % gives nginx 403** (WAF).
+- Name search is therefore an **exact match** (PRIMARY_NAME/=/MILTON RIVERFRONT LLC/ verified) —
+  fuzzy facility-name search goes through the ECHO face (a separate line); this face adds no sugar.
+- Total `/COUNT/JSON` -> [{TOTALQUERYRESULTS}] (same syntax in filtered state, DE=18,794 as measured).
+  52 columns of note: registry_id (authoritative key) /primary_name/std_name/address family
+  (location_address·city·county·state·postal·std_* standardized family)/site_type_name/
+  interest_status_code/operating_status/epa_region_code/parent_registry_id/fips_code. This table has
+  **no coordinate columns** (lat/long live in other FRS tables).
+Politeness tier: size<=500, rate 1rps; every call = rows + count (two requests) (total_matches contract).
+"""
+from ..errors import SdkError
+
+provider = {
+    "id": "epa_frs",
+    "baseUrl": "https://data.epa.gov/efservice",
+    "rate": {"rps": 1.0, "burst": 2},
+    # ops are dynamic: the paths of facilities/count are assembled at call time from the effective filter columns (_op_for) —
+    # the build_url {token} template eats the filter values (quoted, space->%20 is exactly the verified form).
+    # ttl at the top level must be a dict (transport's fallback lookup is eager) — the real values hang on the dynamic opdef.
+    "ops": {},
+    "ttl": {},
+}
+
+_FILTERS = ("registry_id", "primary_name", "state_code", "city_name", "country_name", "site_type_name",
+            "epa_region_code")  # order = URL segment order (stable, replayable); the 0.3.20 audit added the
+# three trailing columns (probed 2026-11-02: COUNTRY_NAME/=/GERMANY -> COUNT 110 — FRS covers non-US facilities,
+# UNITED STATES -> 3,129,491; SITE_TYPE_NAME/=/STATIONARY -> 4,725,914; EPA_REGION_CODE/=/10 -> 214,197, the
+# value form is the zero-padded two-digit string '03'). interest_status_code/operating_status are null on the
+# master table (they live on the interest subtables) — not exposed.
+TTL = 86400
+
+
+def _op_for(cols, *, count=False):
+    """Effective filter columns -> efservice path template (three-segment separated operator form; count=True takes /COUNT/JSON)."""
+    segs = "".join(f"/{c.upper()}/=/{{{c}}}" for c in cols)
+    tail = "/COUNT/JSON" if count else "/ROWS/{start}:{end}/JSON"
+    return {"path": f"/frs_facility_site{segs}{tail}", "ttl": TTL}
+
+
+def _polite_size(v, dflt):
+    """Politeness tier 500 (rows average ~1.5KB across 52 fat columns; upstream has no hard limit, but an invalid ROWS segment = the unbounded-dump trap)."""
+    return min(max(1, int(v or dflt)), 500)
+
+
+def facilities_from_upstream(data, size, start):
+    """Upstream response -> facilities face result (pure function, offline-testable; projects 18 of the 52 columns)."""
+    rows = [{
+        "registry_id": f.get("registry_id"),            # authoritative FRS facility key (ECHO/TRI/GHGRP convergence anchor)
+        "primary_name": f.get("primary_name"),
+        "std_name": f.get("std_name"),
+        "location_address": f.get("location_address"),
+        "city_name": f.get("city_name"),
+        "county_name": f.get("county_name"),
+        "state_code": f.get("state_code"),
+        "state_name": f.get("state_name"),              # dirty upstream (measured: empty strings and the 'DE' abbreviation form)
+        "postal_code": f.get("postal_code"),
+        "country_name": f.get("country_name"),
+        "std_full_address": f.get("std_full_address"),  # USPS standardized full address
+        "site_type_name": f.get("site_type_name"),
+        "interest_status_code": f.get("interest_status_code"),
+        "operating_status": f.get("operating_status"),
+        "epa_region_code": f.get("epa_region_code"),
+        "parent_registry_id": f.get("parent_registry_id"),  # parent facility key (not the corporate parent)
+        "fips_code": f.get("fips_code"),
+        "create_date": f.get("create_date"),
+    } for f in (data or []) if isinstance(f, dict)]
+    return {"rows": rows, "count": len(rows), "limit_applied": size, "offset": start}
+
+
+def _norm_filters(registry_id, primary_name, state_code, city_name, country_name, site_type_name, epa_region_code):
+    out = {}
+    upper_cols = {"state_code", "country_name", "site_type_name", "epa_region_code"}  # upstream stores uppercase forms
+    for k, v in (("registry_id", registry_id), ("primary_name", primary_name),
+                 ("state_code", state_code), ("city_name", city_name),
+                 ("country_name", country_name), ("site_type_name", site_type_name),
+                 ("epa_region_code", epa_region_code)):
+        v = str(v or "").strip()
+        if k in upper_cols and v:
+            v = v.upper()  # upstream uses the uppercase form (DE / GERMANY / STATIONARY / '03'); tolerate lowercase input
+        if v:
+            out[k] = v
+    return out
+
+
+def facilities_raw(registry_id=None, primary_name=None, state_code=None, city_name=None, *,
+                   country_name=None, site_type_name=None, epa_region_code=None,
+                   start=0, size=50, cache=None, no_cache=False):
+    """0.3.2 flat face internals: seven filter columns (all `=` exact, combinable = AND) + start/size paging.
+
+    The four 0.3.2 columns (registry_id/primary_name/state_code/city_name) plus the three the 0.3.20 audit
+    added (country_name/site_type_name/epa_region_code — country_name unlocks the non-US facility slice,
+    GERMANY=110 measured). Every call makes two requests: rows (ROWS/{start}:{end} closed interval, end
+    always computed locally) + count (TOTALQUERYRESULTS).
+    -> {rows[], count, limit_applied, offset, total_matches, params, _meta, _count_meta}.
+    Local safety wall: at least one of the seven filters (a bare scan of all 5,602,878 rows is served by
+    the batch pipeline as national_single.zip)."""
+    from ..transport import call_provider
+
+    filters = _norm_filters(registry_id, primary_name, state_code, city_name,
+                            country_name, site_type_name, epa_region_code)
+    if not filters:
+        raise SdkError("usage_error", provider="epa_frs",
+                       detail="facilities_raw requires at least one of registry_id / primary_name / state_code / "
+                              "city_name / country_name / site_type_name / epa_region_code "
+                              "(local safety wall: a bare scan of all 5,602,878 rows is served by the "
+                              "batch pipeline as national_single.zip)")
+    cols = [c for c in _FILTERS if c in filters]
+    sz = _polite_size(size, 50)
+    st = max(0, int(start or 0))
+    end = st + sz - 1  # right end of the closed interval; always >= start — an invalid ROWS segment = unbounded upstream dump (trap noted in the header)
+    params = {**filters, "start": st, "end": end}
+    dyn = {**provider, "ops": {"facilities": _op_for(cols)}}
+    r = call_provider(dyn, "facilities", params, cache=cache, no_cache=no_cache)
+    dync = {**provider, "ops": {"count": _op_for(cols, count=True)}}
+    rc = call_provider(dync, "count", dict(filters), cache=cache, no_cache=no_cache)
+    total = None
+    if isinstance(rc["data"], list) and rc["data"]:
+        total = rc["data"][0].get("TOTALQUERYRESULTS")
+    out = facilities_from_upstream(r["data"], sz, st)
+    return {**out, "total_matches": total, "filters": filters,
+            "params": params, "_meta": r, "_count_meta": rc}

agentdata/esg/euasl.py

@@ -0,0 +1,75 @@
+"""euasl — EU Air Safety List (third-country carriers banned or restricted, European Commission) file-catalog provider (keyless).
+
+esg transport-compliance axis (2026-11-02, 0.3.22 batch): the list lives on a Liferay page
+(transport.ec.europa.eu/transport-themes/eu-air-safety-list_en — the pre-migration /aviation-policy path is
+dead 404) whose only machine-readable form is a single XLSX (~28KB: Annex A = carriers subject to an
+operating ban, 188 rows measured 2026-10-20; Annex B = carriers operating under restrictions and conditions,
+2 rows — AIR KORYO / IRAN AIR) plus 24 official-language PDF renditions off the same page. The SDK does not
+proxy binary bodies (the JSON-only transport would corrupt a zip — same posture as eutl/thetis_mrv): this
+face returns the live download catalog. Measured shape (2026-11-02, fresh page 200/127KB):
+- 25 download links, href="/document/download/<uuid>_<lang>?filename=air-safety-list[_<lang>][.ext]" —
+  **the document UUIDs rotate on every republication** (they are publish-time content ids, not stable keys),
+  so reconcile harvests against this listing and never guess file names. One XLSX master (uuid 67b75752-…,
+  lang en) + 24 language PDFs (uuid 3195fabb-…; the hr rendition carries an _2 suffix on the filename only —
+  the language token is still the path segment).
+- Update cadence: "whenever the Commission considers it necessary" (historically several revisions a year).
+- The XLSX parses offline but not naively: ~71 self-closing <c/> cells in the sheet mean position-guessing
+  fails — cell references (r="A<n>") are the parse key, sharedStrings holds the strings (batch-pipeline
+  concern; the catalog face is what keeps the link honest).
+TTL 1 day (the page re-renders on each republication; links are stable between them). Politeness tier 0.5 rps.
+"""
+import re
+
+_DOWNLOAD_RE = re.compile(r'href="(/document/download/([^"]+?))"')
+_FILENAME_RE = re.compile(r'filename=([^&"]+)')
+
+provider = {
+    "id": "euasl",
+    "baseUrl": "https://transport.ec.europa.eu",
+    "rate": {"rps": 0.5, "burst": 2},
+    "ops": {
+        "page": {"path": "/transport-themes/eu-air-safety-list_en", "accept": "text"},  # Liferay HTML — links parsed at this layer
+    },
+    "ttl": {"page": 86400},
+}
+
+
+def datasets_from_upstream(html):
+    """Page HTML -> download-catalog rows (pure function, offline-testable).
+
+    rows are sorted xlsx-master first then by language code (deterministic); download_path is a site path
+    (prefix the baseUrl — the batch pipeline assembles the URL from it); `master` carries the machine-readable
+    XLSX row (or None — the page without the xlsx link is an abnormal state the caller surfaces)."""
+    rows, seen = [], set()
+    for full, tail in _DOWNLOAD_RE.findall(html or ""):
+        if full in seen:
+            continue
+        seen.add(full)
+        path_part, _, query = tail.partition("?")  # the query (?filename=…) rides the href too — split it off before the language split
+        uuid, _, lang = path_part.rpartition("_")  # uuid carries hyphens only, the language token is the last path segment
+        fm = _FILENAME_RE.search(query)
+        file_name = fm.group(1) if fm else None
+        fmt = ("xlsx" if (file_name or "").lower().endswith(".xlsx")
+               else "pdf" if (file_name or "").lower().endswith(".pdf") else None)
+        rows.append({
+            "document_id": uuid or None,
+            "language": lang or None,
+            "file_name": file_name,
+            "format": fmt,
+            "download_path": full,
+        })
+    rows.sort(key=lambda r: (0 if r["format"] == "xlsx" else 1, r["language"] or "", r["file_name"] or ""))
+    master = next(({"file_name": r["file_name"], "download_path": r["download_path"]}
+                   for r in rows if r["format"] == "xlsx"), None)
+    return {"rows": rows, "count": len(rows), "master": master}
+
+
+def datasets_raw(*, cache=None, no_cache=False):
+    """0.3.22 flat face internals: the live EU Air Safety List download catalog in one request (upstream has no filter params).
+
+    -> {rows[], count, master, params, _meta} — params={} (zero upstream params)."""
+    from ..transport import call_provider
+
+    params = {}
+    r = call_provider(provider, "page", params, cache=cache, no_cache=no_cache)
+    return {**datasets_from_upstream(r["data"]), "params": params, "_meta": r}

agentdata/esg/eutl.py

@@ -0,0 +1,66 @@
+"""eutl — EU Emissions Trading System registry (Union Registry) official bulk-download catalog provider (keyless).
+
+Sibling face of the esg batch pipeline: EUTL's 15 bulk files (the carbon-emissions surface) previously had
+download coordinates but no API face; 2026-10-29 overturned that: the portal ships its own catalog API
+`GET /api/data-download` that returns a keyless JSON array of 15 files
+`{name, pane_index, group_index, sort_index, url, action}` — this face adopts it as the **download-catalog
+query face** (the file bodies still go through the batch pipeline; this SDK does not proxy large-file downloads).
+Measured shape (2026-10-29):
+- Host **union-registry-data.ec.europa.eu** (the old euetsdata.europa.eu is ENOTFOUND — do not probe it again).
+- The 15 files fall into three tiers (measured 4+10+1): 4 Azure daily extracts
+  (dlsclimabi.blob.core.windows.net…csv.gz: Accounts/Operators/Operators Yearly Activity/Registry Holdings Daily)
+  + 1 Transactions file + 10 climate.ec.europa.eu yearly files (Compliance 2021-2025 x5 + Verified Emissions 2021-2025).
+- Yearly file URLs are direct document/download/<uuid> links (no suffix); daily file URLs carry the cadence in the path.
+- The action field is a portal frontend hook name (d_accounts_daily etc.), not a stable contract — pass through, do not interpret.
+TTL 1 day (the catalog file set rolls slowly); politeness tier 0.5 rps.
+"""
+from ..errors import SdkError
+
+provider = {
+    "id": "eutl",
+    "baseUrl": "https://union-registry-data.ec.europa.eu",
+    "rate": {"rps": 0.5, "burst": 2},
+    "ops": {
+        "datasets": {"path": "/api/data-download"},  # no params: one request returns the whole 15-file catalog
+    },
+    "ttl": {"datasets": 86400},
+}
+
+
+def _cadence(name, url):
+    """Derive the cadence tier from the file name/URL: daily = Azure daily extract / yearly = portal yearly file /
+    static = the Transactions file (it hangs on the same climate.ec.europa.eu domain but is not a rolling yearly
+    file — split by name; measured 15 files = 4+10+1)."""
+    n, u = name or "", url or ""
+    if "blob.core.windows.net" in u:
+        return "daily"
+    if "transaction" in n.lower():
+        return "static"
+    if "climate.ec.europa.eu" in u:
+        return "yearly"
+    return "other"
+
+
+def datasets_from_upstream(data):
+    """Upstream response -> datasets face result (pure function, offline-testable)."""
+    rows = [{
+        "name": d.get("name"),
+        "url": d.get("url"),                       # direct download link (daily = .csv.gz on Azure; yearly = document uuid)
+        "cadence": _cadence(d.get("name"), d.get("url")),  # daily/yearly/other
+        "group_index": d.get("group_index"),
+        "pane_index": d.get("pane_index"),
+        "sort_index": d.get("sort_index"),
+        "action": d.get("action"),                 # portal frontend hook name, not a stable contract
+    } for d in (data or []) if isinstance(d, dict)]
+    return {"rows": rows, "count": len(rows)}
+
+
+def datasets_raw(*, cache=None, no_cache=False):
+    """0.3.2 flat face internals: the whole catalog in one request (upstream has no filter params — the client filters rows locally).
+
+    -> {rows[], count, params, _meta} — params={} (zero upstream params, replayability proven by the empty param set)."""
+    from ..transport import call_provider
+
+    params = {}
+    r = call_provider(provider, "datasets", params, cache=cache, no_cache=no_cache)
+    return {**datasets_from_upstream(r["data"]), "params": params, "_meta": r}