eolas-data 1.3.2__tar.gz → 1.3.4__tar.gz

{eolas_data-1.3.2 → eolas_data-1.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eolas-data
-Version: 1.3.2
+Version: 1.3.4
 Summary: Python client for the eolas.fyi statistical data API (NZ, Australia, OECD)
 Project-URL: Homepage, https://eolas.fyi
 Project-URL: Documentation, https://docs.eolas.fyi/
@@ -21,7 +21,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering
 Requires-Python: >=3.10
 Requires-Dist: pandas>=1.5
-Requires-Dist: pyarrow>=14
+Requires-Dist: pyarrow>=24
 Requires-Dist: requests>=2.28
 Requires-Dist: rich>=13
 Requires-Dist: tqdm>=4.60

{eolas_data-1.3.2 → eolas_data-1.3.4}/eolas_data/__init__.py

@@ -15,7 +15,7 @@ from .exceptions import (
     WatermarkExpired,
 )
 
-__version__ = "1.3.2"
+__version__ = "1.3.3"
 
 __all__ = [
     "Client",

eolas_data-1.3.4/eolas_data/_regen_names.py

@@ -0,0 +1,58 @@
+"""Regenerate ``_dataset_names.py`` from the live API.
+
+Run before each release:
+
+    python -m eolas_data._regen_names
+
+Uses the same API-key resolution as :class:`~eolas_data.client.Client`
+(env var → keyring → ``~/.eolas/config.json``). The catalog endpoint
+requires authentication.
+
+Writes to the same file inside the package. Commit the result.
+"""
+from __future__ import annotations
+
+import datetime as _dt
+import pathlib as _pathlib
+
+
+def regenerate() -> None:
+    from .client import Client
+
+    df = Client().list()
+    names = sorted(df["name"].tolist())
+    today = _dt.date.today().isoformat()
+
+    out = _pathlib.Path(__file__).with_name("_dataset_names.py")
+    lines: list[str] = []
+    lines.append('"""')
+    lines.append("Type stubs for dataset names.")
+    lines.append("")
+    lines.append("Auto-generated from https://api.eolas.fyi/v1/datasets at release time.")
+    lines.append(f"Snapshot: {today} ({len(names)} datasets).")
+    lines.append("Regenerate before each release with `python -m eolas_data._regen_names`.")
+    lines.append("")
+    lines.append("At runtime this is just a string — `Literal[...]` only constrains static type")
+    lines.append("checkers like mypy/pyright, so passing a name not in this list still works,")
+    lines.append("it just doesn't autocomplete.")
+    lines.append('"""')
+    lines.append("from typing import Literal")
+    lines.append("")
+    lines.append(f'CATALOG_SNAPSHOT_DATE = "{today}"')
+    lines.append(f"CATALOG_SNAPSHOT_COUNT = {len(names)}")
+    lines.append("")
+    lines.append("DatasetName = Literal[")
+    for n in names:
+        lines.append(f"    {n!r},")
+    lines.append("]")
+    lines.append("")
+    lines.append("ALL_NAMES: tuple[str, ...] = (")
+    for n in names:
+        lines.append(f"    {n!r},")
+    lines.append(")")
+    out.write_text("\n".join(lines) + "\n")
+    print(f"wrote {len(names)} datasets to {out}")
+
+
+if __name__ == "__main__":
+    regenerate()

{eolas_data-1.3.2 → eolas_data-1.3.4}/eolas_data/client.py

@@ -15,7 +15,7 @@ import requests
 from .console import nag_json_transport_once
 from .dataset import Dataset
 from .library import resolve_library_dir
-from .meta import attach_meta, split_meta
+from .meta import attach_meta, merge_provenance, split_meta
 from .exceptions import (
     APIError,
     AuthenticationError,
@@ -269,6 +269,8 @@ class Client:
         *,
         source: str = "",
         meta: bool = True,
+        provenance: Optional[dict] = None,
+        data_sources: Optional[list] = None,
     ) -> "pd.DataFrame":
         table_meta: dict = {}
         column_meta = None
@@ -277,10 +279,14 @@ class Client:
                 table_meta, column_meta = split_meta(self._info_cached(name))
             except Exception:
                 pass
+        if provenance:
+            table_meta = merge_provenance(table_meta, provenance)
+        if data_sources:
+            table_meta = {**table_meta, "data_sources": data_sources}
         return attach_meta(
             result,
             name=str(name),
-            source=source,
+            source=source or (table_meta.get("source") or ""),
             table_meta=table_meta,
             column_meta=column_meta,
         )
@@ -325,6 +331,19 @@ class Client:
         "csv_gz":     ".csv.gz",
         "geoparquet": ".geo.parquet",
     }
+    # Mirrors the API guard in datasets.py — unbounded live pulls on datasets
+    # above this row count (or with geometry) return HTTP 413.
+    _LARGE_DATASET_ROW_THRESHOLD = 100_000
+
+    @staticmethod
+    def _bulk_export_allowed(meta: dict) -> bool:
+        return (meta.get("bulk_export_class") or "").lower() not in ("", "none")
+
+    @classmethod
+    def _live_pull_blocked(cls, meta: dict) -> bool:
+        """True when limit=0 with no date bounds would hit the API 413 guard."""
+        row_count = int(meta.get("row_count_at_last_refresh") or 0)
+        return bool(meta.get("has_geometry")) or row_count > cls._LARGE_DATASET_ROW_THRESHOLD
 
     @staticmethod
     def _require_bulk_export(meta: dict, name: Union[str, "DatasetName"]) -> None:
@@ -1885,22 +1904,37 @@ class Client:
                 return _pa.Table.from_pandas(pd.read_csv(file_path), preserve_index=False)
             return _pq.read_table(file_path)
 
-        if fmt == "geoparquet":
-            if resolved_as_geo:
+        def _read_bulk_file(path: pathlib.Path, bulk_fmt: str):
+            if bulk_fmt == "csv_gz":
+                return pd.read_csv(path)
+            if bulk_fmt == "geoparquet" and resolved_as_geo:
                 try:
                     import geopandas as gpd
-                    result = gpd.read_parquet(file_path)
-                    return self._attach_dataset_meta(result, name, meta=meta)
+                    return gpd.read_parquet(path)
                 except ImportError:
                     pass
-            result = pd.read_parquet(file_path)
-            return self._attach_dataset_meta(result, name, meta=meta)
-        elif fmt == "csv_gz":
-            result = pd.read_csv(file_path)
-            return self._attach_dataset_meta(result, name, meta=meta)
-        else:  # parquet
-            result = pd.read_parquet(file_path)
-            return self._attach_dataset_meta(result, name, meta=meta)
+            return pd.read_parquet(path)
+
+        try:
+            result = _read_bulk_file(file_path, fmt)
+        except OSError as exc:
+            # Bulk snapshots are written with pyarrow 24+ (Parquet 2.6). Older
+            # pyarrow builds fail with "Repetition level histogram size mismatch".
+            # csv_gz is always generated alongside parquet — fall back to it.
+            if fmt in ("parquet", "geoparquet") and "histogram" in str(exc).lower():
+                _log.warning(
+                    "Parquet read failed for %s (%s) — falling back to csv_gz bulk",
+                    name, exc,
+                )
+                csv_path = cache_path / f"{name}{self._BULK_EXTENSIONS['csv_gz']}"
+                self.sync_bulk(
+                    name, path=csv_path, format="csv_gz",
+                    freshness=freshness, progress=progress,
+                )
+                result = _read_bulk_file(csv_path, "csv_gz")
+            else:
+                raise
+        return self._attach_dataset_meta(result, name, meta=meta)
 
     # ------------------------------------------------------------------
     # Core data fetch
@@ -1917,6 +1951,7 @@ class Client:
         as_geo: Optional[bool] = None,
         as_arrow: bool = False,
         meta: bool = True,
+        envelope: bool = False,
     ) -> Dataset:
         """Fetch dataset rows as a pandas (or polars / geopandas) DataFrame.
 
@@ -1954,6 +1989,9 @@ class Client:
             meta: When ``True`` (default), attach table/column metadata from
                     :meth:`info` (session-cached on this client). Pass
                     ``False`` to skip the extra round-trip.
+            envelope: When ``True``, request ``?envelope=1`` (JSON only) and
+                    attach the ``data_sources`` licence block alongside rows.
+                    Response ``X-Eolas-*`` headers are merged into metadata.
 
         Returns:
             A :class:`Dataset` (pandas DataFrame subclass), a polars DataFrame
@@ -1976,6 +2014,45 @@ class Client:
                 "as_geo materialises geometry as shapely objects in a GeoDataFrame. "
                 "Choose one."
             )
+        if envelope and (format != "json" or as_arrow):
+            raise ValueError(
+                "envelope=True requires format='json' and as_arrow=False."
+            )
+
+        from .rows import apply_row_limit, resolve_fetch_limit, sort_by_date
+
+        # ---- whole-dataset pull on large/geo tables → bulk cache -------------
+        # Matches the API 413 guard: limit=0 with no start/end on >100k-row or
+        # geometry datasets is refused. Transparently serve from get_local()
+        # (CDN-backed Parquet/GeoParquet) so client.get("nz_addresses") works.
+        if (
+            limit is None
+            and start is None
+            and end is None
+            and format == "json"
+            and not envelope
+            and not as_arrow
+        ):
+            try:
+                info_meta = self._info_cached(name)
+                if self._bulk_export_allowed(info_meta) and self._live_pull_blocked(info_meta):
+                    result = self.get_local(
+                        name, as_geo=as_geo, as_arrow=False, meta=meta,
+                    )
+                    if engine == "polars":
+                        try:
+                            import polars as pl
+                            return pl.from_pandas(result)
+                        except ImportError:
+                            raise ImportError(
+                                "polars is required for engine='polars'. "
+                                "Install with: pip install eolas-data[polars]"
+                            )
+                    return result
+            except (BulkLicenceRestricted, BulkUpgradeRequired, BulkNotYetAvailable):
+                raise
+            except Exception:
+                pass  # fall through to live path (e.g. metadata lookup failed)
 
         # ---- early in-memory cache check -------------------------------------
         _early_cache_key = f"{name}:{start}:{end}:{format}:{0 if limit is None else int(limit)}:{as_geo}"
@@ -1988,21 +2065,35 @@ class Client:
             params["start"] = start
         if end:
             params["end"] = end
-        from .rows import apply_row_limit, resolve_fetch_limit, sort_by_date
 
         fetch_limit, user_limit = resolve_fetch_limit(limit)
+        # Positive limits on large/geo datasets must be sent to the API — the
+        # client normally uses limit=0 and trims client-side for dated series,
+        # but limit=0 triggers the API 413 guard on geometry / >100k tables.
+        if user_limit and int(user_limit) > 0 and start is None and end is None:
+            try:
+                info_meta = self._info_cached(name)
+                if self._live_pull_blocked(info_meta):
+                    fetch_limit = int(user_limit)
+            except Exception:
+                pass
         params["limit"] = fetch_limit
 
         cache_key = f"{name}:{start}:{end}:{format}:{0 if limit is None else int(limit)}:{as_geo}"
         if self._cache is not None and cache_key in self._cache:
             return self._cache[cache_key]
 
+        data_sources = None
+        provenance = None
         if format == "csv":
             from io import StringIO
             resp = self._raw_get(f"/v1/datasets/{name}/data", params={"format": "csv", **params})
+            provenance = resp.headers
             df   = pd.read_csv(StringIO(resp.text))
         else:
-            df = self._fetch_dataframe(name, params)
+            df, provenance, data_sources = self._fetch_dataframe(
+                name, params, envelope=envelope,
+            )
             if "date" in df.columns:
                 df["date"] = pd.to_datetime(df["date"])
 
@@ -2020,7 +2111,10 @@ class Client:
             return tbl
 
         result = Dataset(df)
-        result = self._attach_dataset_meta(result, name, meta=meta)
+        result = self._attach_dataset_meta(
+            result, name, meta=meta,
+            provenance=provenance, data_sources=data_sources,
+        )
 
         if engine == "polars":
             try:
@@ -2048,13 +2142,27 @@ class Client:
     # HTTP helpers
     # ------------------------------------------------------------------
 
-    def _fetch_dataframe(self, name, params: dict) -> pd.DataFrame:
+    def _fetch_dataframe(
+        self, name, params: dict, *, envelope: bool = False,
+    ) -> tuple[pd.DataFrame, object, Optional[list]]:
         """Fetch dataset rows as a DataFrame, negotiating Arrow IPC over the
         wire (≈5x faster end-to-end, ≈82x faster parse than JSON on large
         pulls — benchmarked 2026-05-18). Transparently falls back to JSON for
         older servers, unexpected content-types, or any pyarrow issue, so the
         returned DataFrame is identical either way.
+
+        Returns ``(dataframe, response_headers, data_sources_or_none)``.
         """
+        if envelope:
+            resp = self._raw_get(
+                f"/v1/datasets/{name}/data",
+                params={**params, "envelope": 1},
+            )
+            payload = resp.json()
+            records = payload.get("data", payload) if isinstance(payload, dict) else payload
+            sources = payload.get("data_sources") if isinstance(payload, dict) else None
+            return pd.DataFrame(records), resp.headers, sources
+
         if self._arrow_supported is not False:
             try:
                 import io
@@ -2067,7 +2175,7 @@ class Client:
                 if "arrow" in resp.headers.get("content-type", ""):
                     self._arrow_supported = True
                     tbl = pa.ipc.open_stream(io.BytesIO(resp.content)).read_all()
-                    return tbl.to_pandas()
+                    return tbl.to_pandas(), resp.headers, None
                 # Old server ignored format=arrow and returned JSON. Remember
                 # so we don't pay the failed round-trip on every future call.
                 self._arrow_supported = False
@@ -2076,9 +2184,11 @@ class Client:
                 self._arrow_supported = False
                 nag_json_transport_once()
 
-        data = self._get(f"/v1/datasets/{name}/data", params=params)
+        resp = self._raw_get(f"/v1/datasets/{name}/data", params=params)
+        data = resp.json()
         records = data.get("data", data) if isinstance(data, dict) else data
-        return pd.DataFrame(records)
+        sources = data.get("data_sources") if isinstance(data, dict) else None
+        return pd.DataFrame(records), resp.headers, sources
 
     def _get(self, path: str, params: Optional[dict] = None) -> dict:
         return self._raw_get(path, params=params).json()

{eolas_data-1.3.2 → eolas_data-1.3.4}/eolas_data/meta.py

@@ -6,6 +6,35 @@ from typing import Any, Optional
 import pandas as pd
 
 # Table-level fields we keep on Dataset.eolas_meta (exclude nested columns).
+_PROVENANCE_HEADERS = {
+    "X-Eolas-Attribution": "attribution_text",
+    "X-Eolas-Licence": "licence",
+    "X-Eolas-Source": "source",
+    "X-Eolas-Source-URL": "source_url",
+    "X-Eolas-Namespace": "namespace",
+}
+
+
+def provenance_from_headers(headers) -> dict:
+    """Extract response provenance from X-Eolas-* headers on /data responses."""
+    out: dict[str, str] = {}
+    get = headers.get if hasattr(headers, "get") else lambda k, d=None: headers.get(k, d)
+    for hdr, key in _PROVENANCE_HEADERS.items():
+        val = get(hdr)
+        if val:
+            out[key] = str(val)
+    return out
+
+
+def merge_provenance(table_meta: Optional[dict], headers) -> dict:
+    """Merge catalogue metadata with live response headers (headers win when set)."""
+    merged = dict(table_meta or {})
+    for key, val in provenance_from_headers(headers).items():
+        if val:
+            merged[key] = val
+    return merged
+
+
 _TABLE_META_KEYS = (
     "name",
     "namespace",

{eolas_data-1.3.2 → eolas_data-1.3.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "eolas-data"
-version = "1.3.2"
+version = "1.3.4"
 description = "Python client for the eolas.fyi statistical data API (NZ, Australia, OECD)"
 readme = "README.md"
 license = { text = "MIT" }
@@ -13,7 +13,7 @@ requires-python = ">=3.10"
 dependencies = [
     "requests>=2.28",
     "pandas>=1.5",
-    "pyarrow>=14",
+    "pyarrow>=24",
     "tqdm>=4.60",
     "rich>=13",
     # typer powers the `eolas` console script, which is always registered in

{eolas_data-1.3.2 → eolas_data-1.3.4}/tests/test_as_arrow.py

@@ -44,7 +44,7 @@ def test_get_live_as_arrow_returns_arrow_table(client):
     """as_arrow=True on the live path converts the JSON-fetched DataFrame to pa.Table."""
     fake_df = pd.DataFrame({"date": ["2023-01-01"], "value": [1100.5]})
 
-    with patch.object(client, "_fetch_dataframe", return_value=fake_df):
+    with patch.object(client, "_fetch_dataframe", return_value=(fake_df, {}, None)):
         result = client.get("nz_cpi", as_arrow=True)
 
     assert isinstance(result, pa.Table)

{eolas_data-1.3.2 → eolas_data-1.3.4}/tests/test_client.py

@@ -153,9 +153,13 @@ def test_get_limit_returns_most_recent(client):
 
 @resp_lib.activate
 def test_get_limit_requests_full_window_from_server(client):
+    resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_cpi",
+                 json={"name": "nz_cpi", "source": "Stats NZ", "has_geometry": False,
+                       "row_count_at_last_refresh": 500})
     resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_cpi/data", json={"data": RECORDS})
     client.get("nz_cpi", limit=2)
-    assert "limit=0" in resp_lib.calls[0].request.url
+    data_reqs = [c for c in resp_lib.calls if "/data?" in c.request.url or c.request.url.endswith("/data")]
+    assert any("limit=0" in c.request.url for c in data_reqs)
 
 
 @resp_lib.activate
@@ -167,6 +171,38 @@ def test_get_passes_date_params(client):
     assert "end=2023-06-30" in req.url
 
 
+@resp_lib.activate
+def test_get_merges_provenance_headers(client):
+    resp_lib.add(
+        resp_lib.GET,
+        f"{BASE}/v1/datasets/nz_cpi/data",
+        json={"data": RECORDS},
+        adding_headers={
+            "X-Eolas-Attribution": "Source: OECD, CC-BY 4.0.",
+            "X-Eolas-Licence": "CC-BY 4.0",
+            "X-Eolas-Source": "OECD",
+        },
+    )
+    df = client.get("nz_cpi", meta=False)
+    meta = df.attrs.get("eolas_meta", {})
+    assert meta.get("attribution_text") == "Source: OECD, CC-BY 4.0."
+    assert meta.get("licence") == "CC-BY 4.0"
+    assert meta.get("source") == "OECD"
+
+
+@resp_lib.activate
+def test_get_envelope_attaches_data_sources(client):
+    payload = {
+        "data_sources": [{"name": "nz_cpi", "licence": "CC-BY 4.0"}],
+        "data": RECORDS,
+    }
+    resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_cpi/data", json=payload)
+    df = client.get("nz_cpi", envelope=True, meta=False)
+    meta = df.attrs.get("eolas_meta", {})
+    assert meta.get("data_sources") == payload["data_sources"]
+    assert "envelope=1" in resp_lib.calls[0].request.url
+
+
 @resp_lib.activate
 def test_get_csv_returns_dataframe(client):
     csv_body = "date,period,value\n2023-01-01,2023Q1,100.0\n"
@@ -213,6 +249,42 @@ def test_get_as_geo_false_keeps_wkt(client):
     assert df["geometry_wkt"].iloc[0].startswith("POINT")
 
 
+LARGE_GEO_META = {
+    "name": "nz_addresses",
+    "namespace": "linz",
+    "bulk_export_class": "materialised",
+    "has_geometry": True,
+    "geometry_type": "point",
+    "row_count_at_last_refresh": 2_418_264,
+}
+
+
+@resp_lib.activate
+def test_get_whole_dataset_defers_to_get_local_for_large_geo(client):
+    """Whole-dataset pulls on geometry tables route to get_local(), not /data."""
+    sentinel = pd.DataFrame({"address_id": [1], "geometry_wkt": ["POINT (0 0)"]})
+    with patch.object(client, "get_local", return_value=sentinel) as mock_local:
+        resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_addresses",
+                     json=LARGE_GEO_META, status=200)
+        result = client.get("nz_addresses")
+    mock_local.assert_called_once()
+    assert result is sentinel
+
+
+@resp_lib.activate
+def test_get_limit_on_large_geo_sends_bounded_limit(client):
+    """Positive limits on geometry tables must not send limit=0 (API 413)."""
+    resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_addresses",
+                 json=LARGE_GEO_META, status=200)
+    resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_addresses/data",
+                 json={"data": GEO_RECORDS})
+    client.get("nz_addresses", limit=2)
+    data_reqs = [c for c in resp_lib.calls if "/data?" in c.request.url or c.request.url.endswith("/data")]
+    assert data_reqs, "expected a /data request"
+    assert any("limit=2" in c.request.url for c in data_reqs)
+    assert not any("limit=0" in c.request.url for c in data_reqs)
+
+
 @resp_lib.activate
 def test_get_no_geometry_column_returns_dataset(client):
     """Datasets without a geometry_wkt column still return a Dataset even with as_geo=None."""

eolas_data-1.3.2/eolas_data/_regen_names.py

@@ -1,57 +0,0 @@
-"""Regenerate ``_dataset_names.py`` from the live API.
-
-Run before each release:
-
-    python -m eolas_data._regen_names
-
-Writes to the same file inside the package. Commit the result.
-"""
-from __future__ import annotations
-
-import datetime as _dt
-import json as _json
-import pathlib as _pathlib
-import urllib.request as _req
-
-API = "https://api.eolas.fyi/v1/datasets"
-
-
-def regenerate() -> None:
-    with _req.urlopen(API, timeout=30) as r:
-        data = _json.load(r)
-    names = sorted({d["name"] for d in data})
-    today = _dt.date.today().isoformat()
-
-    out = _pathlib.Path(__file__).with_name("_dataset_names.py")
-    lines: list[str] = []
-    lines.append('"""')
-    lines.append('Type stubs for dataset names.')
-    lines.append('')
-    lines.append('Auto-generated from https://api.eolas.fyi/v1/datasets at release time.')
-    lines.append(f'Snapshot: {today} ({len(names)} datasets).')
-    lines.append('Regenerate before each release with `python -m eolas_data._regen_names`.')
-    lines.append('')
-    lines.append('At runtime this is just a string — `Literal[...]` only constrains static type')
-    lines.append("checkers like mypy/pyright, so passing a name not in this list still works,")
-    lines.append("it just doesn't autocomplete.")
-    lines.append('"""')
-    lines.append('from typing import Literal')
-    lines.append('')
-    lines.append(f'CATALOG_SNAPSHOT_DATE = "{today}"')
-    lines.append(f'CATALOG_SNAPSHOT_COUNT = {len(names)}')
-    lines.append('')
-    lines.append('DatasetName = Literal[')
-    for n in names:
-        lines.append(f'    {n!r},')
-    lines.append(']')
-    lines.append('')
-    lines.append('ALL_NAMES: tuple[str, ...] = (')
-    for n in names:
-        lines.append(f'    {n!r},')
-    lines.append(')')
-    out.write_text("\n".join(lines) + "\n")
-    print(f"wrote {len(names)} datasets to {out}")
-
-
-if __name__ == "__main__":
-    regenerate()