eolas-data 1.3.2__tar.gz → 1.3.5__tar.gz

{eolas_data-1.3.2 → eolas_data-1.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eolas-data
-Version: 1.3.2
+Version: 1.3.5
 Summary: Python client for the eolas.fyi statistical data API (NZ, Australia, OECD)
 Project-URL: Homepage, https://eolas.fyi
 Project-URL: Documentation, https://docs.eolas.fyi/
@@ -21,7 +21,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering
 Requires-Python: >=3.10
 Requires-Dist: pandas>=1.5
-Requires-Dist: pyarrow>=14
+Requires-Dist: pyarrow>=24
 Requires-Dist: requests>=2.28
 Requires-Dist: rich>=13
 Requires-Dist: tqdm>=4.60
@@ -231,7 +231,7 @@ r    = client.sync_bulk("nz_cpi", path="nz_cpi.parquet")
 path = client.download_bulk("treasury_fiscal_spending", path="t.parquet")
 ```
 
-**Progress bars:** `download_bulk`, `sync_bulk`, and `get_local` all show a `tqdm` progress bar automatically in interactive terminals and VSCode notebooks, so 1+ GB files are never silent. Pass `progress=False` to suppress in scripts, or set `EOLAS_NO_PROGRESS=1` in the environment for a CI-wide escape hatch. The `--no-progress` flag does the same from the CLI.
+**Progress bars:** `get_local()` shows two phases in interactive sessions — a **download** byte bar while fetching from CDN, then a **read** spinner while Parquet/GeoParquet is loaded (often the slow part on multi-million-row geo datasets). Control with `progress=True` (both), `False` (neither), `"download"`, or `"read"`. Set `EOLAS_NO_PROGRESS=1` to suppress both in batch scripts. Cached files skip the download bar and print an informative message instead.
 
 CLI mirror: `eolas download <name>` for one-shot, `eolas sync <name> [--watch hourly]` for an incremental check. Full docs: [docs.eolas.fyi/bulk-downloads/](https://docs.eolas.fyi/bulk-downloads/).
 

{eolas_data-1.3.2 → eolas_data-1.3.5}/README.md

@@ -183,7 +183,7 @@ r    = client.sync_bulk("nz_cpi", path="nz_cpi.parquet")
 path = client.download_bulk("treasury_fiscal_spending", path="t.parquet")
 ```
 
-**Progress bars:** `download_bulk`, `sync_bulk`, and `get_local` all show a `tqdm` progress bar automatically in interactive terminals and VSCode notebooks, so 1+ GB files are never silent. Pass `progress=False` to suppress in scripts, or set `EOLAS_NO_PROGRESS=1` in the environment for a CI-wide escape hatch. The `--no-progress` flag does the same from the CLI.
+**Progress bars:** `get_local()` shows two phases in interactive sessions — a **download** byte bar while fetching from CDN, then a **read** spinner while Parquet/GeoParquet is loaded (often the slow part on multi-million-row geo datasets). Control with `progress=True` (both), `False` (neither), `"download"`, or `"read"`. Set `EOLAS_NO_PROGRESS=1` to suppress both in batch scripts. Cached files skip the download bar and print an informative message instead.
 
 CLI mirror: `eolas download <name>` for one-shot, `eolas sync <name> [--watch hourly]` for an incremental check. Full docs: [docs.eolas.fyi/bulk-downloads/](https://docs.eolas.fyi/bulk-downloads/).
 

{eolas_data-1.3.2 → eolas_data-1.3.5}/eolas_data/__init__.py

@@ -15,7 +15,7 @@ from .exceptions import (
     WatermarkExpired,
 )
 
-__version__ = "1.3.2"
+__version__ = "1.3.3"
 
 __all__ = [
     "Client",

eolas_data-1.3.5/eolas_data/_regen_names.py

@@ -0,0 +1,58 @@
+"""Regenerate ``_dataset_names.py`` from the live API.
+
+Run before each release:
+
+    python -m eolas_data._regen_names
+
+Uses the same API-key resolution as :class:`~eolas_data.client.Client`
+(env var → keyring → ``~/.eolas/config.json``). The catalog endpoint
+requires authentication.
+
+Writes to the same file inside the package. Commit the result.
+"""
+from __future__ import annotations
+
+import datetime as _dt
+import pathlib as _pathlib
+
+
+def regenerate() -> None:
+    from .client import Client
+
+    df = Client().list()
+    names = sorted(df["name"].tolist())
+    today = _dt.date.today().isoformat()
+
+    out = _pathlib.Path(__file__).with_name("_dataset_names.py")
+    lines: list[str] = []
+    lines.append('"""')
+    lines.append("Type stubs for dataset names.")
+    lines.append("")
+    lines.append("Auto-generated from https://api.eolas.fyi/v1/datasets at release time.")
+    lines.append(f"Snapshot: {today} ({len(names)} datasets).")
+    lines.append("Regenerate before each release with `python -m eolas_data._regen_names`.")
+    lines.append("")
+    lines.append("At runtime this is just a string — `Literal[...]` only constrains static type")
+    lines.append("checkers like mypy/pyright, so passing a name not in this list still works,")
+    lines.append("it just doesn't autocomplete.")
+    lines.append('"""')
+    lines.append("from typing import Literal")
+    lines.append("")
+    lines.append(f'CATALOG_SNAPSHOT_DATE = "{today}"')
+    lines.append(f"CATALOG_SNAPSHOT_COUNT = {len(names)}")
+    lines.append("")
+    lines.append("DatasetName = Literal[")
+    for n in names:
+        lines.append(f"    {n!r},")
+    lines.append("]")
+    lines.append("")
+    lines.append("ALL_NAMES: tuple[str, ...] = (")
+    for n in names:
+        lines.append(f"    {n!r},")
+    lines.append(")")
+    out.write_text("\n".join(lines) + "\n")
+    print(f"wrote {len(names)} datasets to {out}")
+
+
+if __name__ == "__main__":
+    regenerate()

{eolas_data-1.3.2 → eolas_data-1.3.5}/eolas_data/client.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import contextlib
 import datetime
 import json
 import logging
@@ -7,7 +8,10 @@ import os
 import pathlib
 import sys
 from dataclasses import dataclass
-from typing import Optional, Union
+from typing import Literal, Optional, Union
+
+ProgressControl = Union[bool, str, None]
+ProgressPhase = Literal["download", "read"]
 
 import pandas as pd
 import requests
@@ -15,7 +19,7 @@ import requests
 from .console import nag_json_transport_once
 from .dataset import Dataset
 from .library import resolve_library_dir
-from .meta import attach_meta, split_meta
+from .meta import attach_meta, merge_provenance, split_meta
 from .exceptions import (
     APIError,
     AuthenticationError,
@@ -269,6 +273,8 @@ class Client:
         *,
         source: str = "",
         meta: bool = True,
+        provenance: Optional[dict] = None,
+        data_sources: Optional[list] = None,
     ) -> "pd.DataFrame":
         table_meta: dict = {}
         column_meta = None
@@ -277,10 +283,14 @@ class Client:
                 table_meta, column_meta = split_meta(self._info_cached(name))
             except Exception:
                 pass
+        if provenance:
+            table_meta = merge_provenance(table_meta, provenance)
+        if data_sources:
+            table_meta = {**table_meta, "data_sources": data_sources}
         return attach_meta(
             result,
             name=str(name),
-            source=source,
+            source=source or (table_meta.get("source") or ""),
             table_meta=table_meta,
             column_meta=column_meta,
         )
@@ -325,6 +335,19 @@ class Client:
         "csv_gz":     ".csv.gz",
         "geoparquet": ".geo.parquet",
     }
+    # Mirrors the API guard in datasets.py — unbounded live pulls on datasets
+    # above this row count (or with geometry) return HTTP 413.
+    _LARGE_DATASET_ROW_THRESHOLD = 100_000
+
+    @staticmethod
+    def _bulk_export_allowed(meta: dict) -> bool:
+        return (meta.get("bulk_export_class") or "").lower() not in ("", "none")
+
+    @classmethod
+    def _live_pull_blocked(cls, meta: dict) -> bool:
+        """True when limit=0 with no date bounds would hit the API 413 guard."""
+        row_count = int(meta.get("row_count_at_last_refresh") or 0)
+        return bool(meta.get("has_geometry")) or row_count > cls._LARGE_DATASET_ROW_THRESHOLD
 
     @staticmethod
     def _require_bulk_export(meta: dict, name: Union[str, "DatasetName"]) -> None:
@@ -343,7 +366,7 @@ class Client:
         freshness: str = "auto",
         format: str = "parquet",
         path: Optional[Union[str, "pathlib.Path"]] = None,
-        progress: Optional[bool] = None,
+        progress: ProgressControl = None,
     ) -> "Union[pathlib.Path, bytes]":
         """Download a complete dataset as a single binary file.
 
@@ -368,14 +391,10 @@ class Client:
                 bytes. Pass a ``str`` or ``pathlib.Path`` to write to disk and
                 return the resolved path. Parent directories are created if
                 needed.
-            progress: Control the download progress bar.
-                ``None`` (default) auto-detects: bar shown when
-                ``sys.stdout.isatty()`` is True (interactive terminal or
-                notebook), hidden when piped or in CI.
-                ``True`` forces the bar on regardless (useful in log-tailing
-                scenarios).  ``False`` forces it off.  When ``path`` is
-                ``None`` (bytes mode) progress is always disabled — there is
-                no file path to label.
+            progress: Control the download progress bar (``"download"`` phase).
+                See :meth:`get_local` for the full selector vocabulary
+                (``"read"`` applies only to :meth:`get_local`). When ``path``
+                is ``None`` (bytes mode) progress is always disabled.
 
         Returns:
             ``pathlib.Path`` when ``path`` is set (the resolved, written path).
@@ -448,13 +467,13 @@ class Client:
         out = pathlib.Path(path).expanduser().resolve()
         out.parent.mkdir(parents=True, exist_ok=True)
 
-        show = self._resolve_show_progress(progress)
+        show = self._resolve_show_progress(progress, "download")
         resp = self._raw_bulk_get(bulk_path, params=params, stream=True)
         total = int(resp.headers.get("Content-Length", 0)) or None
         self._stream_to_file_with_progress(
             resp, out,
             total_bytes=total,
-            label=out.name,
+            label=f"Downloading {out.name}",
             show_progress=show,
         )
         return out
@@ -466,7 +485,7 @@ class Client:
         path: Union[str, "pathlib.Path"],
         format: str = "parquet",
         freshness: str = "auto",
-        progress: Optional[bool] = None,
+        progress: ProgressControl = None,
     ) -> SyncResult:
         """Incrementally sync a bulk dataset file — only re-download when the snapshot changes.
 
@@ -495,11 +514,10 @@ class Client:
                 ``"geoparquet"``.
             freshness: ``"auto"`` (default), ``"monthly"``, or ``"current"``.
                 Passed verbatim to the bulk endpoint.
-            progress: Control the download progress bar.
-                ``None`` (default) auto-detects via ``sys.stdout.isatty()``.
-                ``True`` forces the bar on; ``False`` forces it off.
-                When ``status="unchanged"`` no data is transferred so no bar
-                is shown regardless of this setting.
+            progress: Control the download progress bar (``"download"`` phase).
+                See :meth:`get_local` for the full selector vocabulary.
+                When ``status="unchanged"`` no download bar is shown; an
+                informative cached-file message is printed instead.
 
         Returns:
             A :class:`SyncResult` dataclass with ``status``,
@@ -587,6 +605,7 @@ class Client:
             and prev.get("snapshot_id") == current_sid
             and out.exists()
         ):
+            print(f"Using cached {out.name} (up to date).", file=sys.stderr)
             return SyncResult(
                 status="unchanged",
                 previous_snapshot_id=prev.get("snapshot_id"),
@@ -598,14 +617,14 @@ class Client:
         # Download (atomic replace).
         out.parent.mkdir(parents=True, exist_ok=True)
         tmp = out.with_suffix(out.suffix + f".eolas-tmp-{os.urandom(4).hex()}")
-        show = self._resolve_show_progress(progress)
+        show = self._resolve_show_progress(progress, "download")
         try:
             resp = self._raw_bulk_get(bulk_path, params=params, stream=True)
             total = int(resp.headers.get("Content-Length", 0)) or None
             bytes_dl = self._stream_to_file_with_progress(
                 resp, tmp,
                 total_bytes=total,
-                label=out.name,
+                label=f"Downloading {out.name}",
                 show_progress=show,
             )
             if bytes_dl == 0:
@@ -1140,28 +1159,73 @@ class Client:
     # ------------------------------------------------------------------
 
     @staticmethod
-    def _resolve_show_progress(progress: Optional[bool]) -> bool:
-        """Resolve the tri-state ``progress`` kwarg to a concrete bool.
-
-        Priority (highest first):
-        1. Explicit ``progress=True/False`` kwarg from the caller.
-        2. ``EOLAS_NO_PROGRESS=1`` environment variable — always suppresses.
-        3. Jupyter / IPython kernel detection — Jupyter wraps stdout in an
-           ``OutStream``, so ``sys.stdout.isatty()`` returns False even when
-           the user is clearly in an interactive notebook session. If
-           ``ipykernel`` is loaded (the standard signal used by tqdm itself),
-           we're in a notebook → show the bar; ``tqdm.auto`` then renders it
-           as an ipywidget (or text bar as fallback).
-        4. ``sys.stdout.isatty()`` auto-detection for plain terminals.
-        """
-        if progress is not None:
-            return bool(progress)
+    def _progress_auto_detect() -> bool:
+        """Whether progress feedback should show when ``progress`` is None."""
         if os.getenv("EOLAS_NO_PROGRESS", "").strip() in ("1", "true", "yes"):
             return False
         if "ipykernel" in sys.modules:
             return True
         return sys.stdout.isatty()
 
+    @staticmethod
+    def _resolve_progress_phases(progress: ProgressControl) -> dict[str, bool]:
+        """Map ``progress`` to download/read phase flags.
+
+        Accepts ``None``, ``True``/``False``, or ``"both"``, ``"download"``,
+        ``"read"``, ``"none"`` (and ``"all"`` as alias for ``"both"``).
+        """
+        if progress is False:
+            return {"download": False, "read": False}
+        if progress is True:
+            return {"download": True, "read": True}
+        if isinstance(progress, str):
+            key = progress.strip().lower()
+            table = {
+                "both": (True, True),
+                "all": (True, True),
+                "download": (True, False),
+                "read": (False, True),
+                "none": (False, False),
+            }
+            if key not in table:
+                raise ValueError(
+                    "progress must be True, False, None, or one of "
+                    "'both', 'download', 'read', 'none'."
+                )
+            d, r = table[key]
+            return {"download": d, "read": r}
+        auto = Client._progress_auto_detect()
+        return {"download": auto, "read": auto}
+
+    @staticmethod
+    def _resolve_show_progress(
+        progress: ProgressControl,
+        phase: ProgressPhase = "download",
+    ) -> bool:
+        """Resolve ``progress`` for one bulk phase (download or read)."""
+        return Client._resolve_progress_phases(progress)[phase]
+
+    @staticmethod
+    @contextlib.contextmanager
+    def _with_read_progress(label: str, show: bool):
+        """Indeterminate spinner while materialising a cached bulk file."""
+        if not show:
+            yield
+            return
+        import tqdm.auto
+
+        bar = tqdm.auto.tqdm(
+            total=1,
+            desc=f"Loading {label} from disk",
+            leave=False,
+            bar_format="{desc}…",
+        )
+        try:
+            yield
+        finally:
+            bar.update(1)
+            bar.close()
+
     @staticmethod
     def _stream_to_file_with_progress(
         resp: "requests.Response",
@@ -1777,7 +1841,7 @@ class Client:
         as_geo: Optional[bool] = None,
         as_arrow: bool = False,
         meta: bool = True,
-        progress: Optional[bool] = None,
+        progress: ProgressControl = None,
     ) -> "pd.DataFrame":
         """Download (or serve from cache) a whole dataset as a local DataFrame.
 
@@ -1814,7 +1878,14 @@ class Client:
             meta: When ``True`` (default), attach table/column metadata from
                 :meth:`info` (session-cached). Pass ``False`` to skip the extra
                 round-trip.
-            progress: Forwarded to :meth:`sync_bulk` (``None`` auto-detects TTY).
+            progress: Control progress for two phases: **download** (streaming
+                byte bar via :meth:`sync_bulk`) and **read** (spinner while
+                Parquet/GeoParquet is loaded into a DataFrame). ``None``
+                (default) enables both in interactive sessions. ``True``/``False``
+                force both on/off. Use ``"download"``, ``"read"``, ``"both"``,
+                or ``"none"`` for one phase only. Suppressed by
+                ``EOLAS_NO_PROGRESS=1``. Cached snapshots skip the download bar
+                and print an informative message instead.
 
         Returns:
             ``pd.DataFrame`` (tabular) or ``geopandas.GeoDataFrame`` (geo +
@@ -1877,30 +1948,52 @@ class Client:
         # ---- sync (download if needed, HEAD check if cached) -----------------
         self.sync_bulk(name, path=file_path, format=fmt, freshness=freshness, progress=progress)
 
+        show_read = self._resolve_show_progress(progress, "read")
+        read_label = file_path.name
+
         # ---- read the local file into a DataFrame ----------------------------
         if as_arrow:
             import pyarrow.parquet as _pq
-            if fmt == "csv_gz":
-                import pyarrow as _pa
-                return _pa.Table.from_pandas(pd.read_csv(file_path), preserve_index=False)
-            return _pq.read_table(file_path)
-
-        if fmt == "geoparquet":
-            if resolved_as_geo:
-                try:
-                    import geopandas as gpd
-                    result = gpd.read_parquet(file_path)
-                    return self._attach_dataset_meta(result, name, meta=meta)
-                except ImportError:
-                    pass
-            result = pd.read_parquet(file_path)
-            return self._attach_dataset_meta(result, name, meta=meta)
-        elif fmt == "csv_gz":
-            result = pd.read_csv(file_path)
-            return self._attach_dataset_meta(result, name, meta=meta)
-        else:  # parquet
-            result = pd.read_parquet(file_path)
-            return self._attach_dataset_meta(result, name, meta=meta)
+            with self._with_read_progress(read_label, show_read):
+                if fmt == "csv_gz":
+                    import pyarrow as _pa
+                    return _pa.Table.from_pandas(
+                        pd.read_csv(file_path), preserve_index=False,
+                    )
+                return _pq.read_table(file_path)
+
+        def _read_bulk_file(path: pathlib.Path, bulk_fmt: str):
+            with self._with_read_progress(read_label, show_read):
+                if bulk_fmt == "csv_gz":
+                    return pd.read_csv(path)
+                if bulk_fmt == "geoparquet" and resolved_as_geo:
+                    try:
+                        import geopandas as gpd
+                        return gpd.read_parquet(path)
+                    except ImportError:
+                        pass
+                return pd.read_parquet(path)
+
+        try:
+            result = _read_bulk_file(file_path, fmt)
+        except OSError as exc:
+            # Bulk snapshots are written with pyarrow 24+ (Parquet 2.6). Older
+            # pyarrow builds fail with "Repetition level histogram size mismatch".
+            # csv_gz is always generated alongside parquet — fall back to it.
+            if fmt in ("parquet", "geoparquet") and "histogram" in str(exc).lower():
+                _log.warning(
+                    "Parquet read failed for %s (%s) — falling back to csv_gz bulk",
+                    name, exc,
+                )
+                csv_path = cache_path / f"{name}{self._BULK_EXTENSIONS['csv_gz']}"
+                self.sync_bulk(
+                    name, path=csv_path, format="csv_gz",
+                    freshness=freshness, progress=progress,
+                )
+                result = _read_bulk_file(csv_path, "csv_gz")
+            else:
+                raise
+        return self._attach_dataset_meta(result, name, meta=meta)
 
     # ------------------------------------------------------------------
     # Core data fetch
@@ -1917,6 +2010,7 @@ class Client:
         as_geo: Optional[bool] = None,
         as_arrow: bool = False,
         meta: bool = True,
+        envelope: bool = False,
     ) -> Dataset:
         """Fetch dataset rows as a pandas (or polars / geopandas) DataFrame.
 
@@ -1954,6 +2048,9 @@ class Client:
             meta: When ``True`` (default), attach table/column metadata from
                     :meth:`info` (session-cached on this client). Pass
                     ``False`` to skip the extra round-trip.
+            envelope: When ``True``, request ``?envelope=1`` (JSON only) and
+                    attach the ``data_sources`` licence block alongside rows.
+                    Response ``X-Eolas-*`` headers are merged into metadata.
 
         Returns:
             A :class:`Dataset` (pandas DataFrame subclass), a polars DataFrame
@@ -1976,6 +2073,45 @@ class Client:
                 "as_geo materialises geometry as shapely objects in a GeoDataFrame. "
                 "Choose one."
             )
+        if envelope and (format != "json" or as_arrow):
+            raise ValueError(
+                "envelope=True requires format='json' and as_arrow=False."
+            )
+
+        from .rows import apply_row_limit, resolve_fetch_limit, sort_by_date
+
+        # ---- whole-dataset pull on large/geo tables → bulk cache -------------
+        # Matches the API 413 guard: limit=0 with no start/end on >100k-row or
+        # geometry datasets is refused. Transparently serve from get_local()
+        # (CDN-backed Parquet/GeoParquet) so client.get("nz_addresses") works.
+        if (
+            limit is None
+            and start is None
+            and end is None
+            and format == "json"
+            and not envelope
+            and not as_arrow
+        ):
+            try:
+                info_meta = self._info_cached(name)
+                if self._bulk_export_allowed(info_meta) and self._live_pull_blocked(info_meta):
+                    result = self.get_local(
+                        name, as_geo=as_geo, as_arrow=False, meta=meta,
+                    )
+                    if engine == "polars":
+                        try:
+                            import polars as pl
+                            return pl.from_pandas(result)
+                        except ImportError:
+                            raise ImportError(
+                                "polars is required for engine='polars'. "
+                                "Install with: pip install eolas-data[polars]"
+                            )
+                    return result
+            except (BulkLicenceRestricted, BulkUpgradeRequired, BulkNotYetAvailable):
+                raise
+            except Exception:
+                pass  # fall through to live path (e.g. metadata lookup failed)
 
         # ---- early in-memory cache check -------------------------------------
         _early_cache_key = f"{name}:{start}:{end}:{format}:{0 if limit is None else int(limit)}:{as_geo}"
@@ -1988,21 +2124,35 @@ class Client:
             params["start"] = start
         if end:
             params["end"] = end
-        from .rows import apply_row_limit, resolve_fetch_limit, sort_by_date
 
         fetch_limit, user_limit = resolve_fetch_limit(limit)
+        # Positive limits on large/geo datasets must be sent to the API — the
+        # client normally uses limit=0 and trims client-side for dated series,
+        # but limit=0 triggers the API 413 guard on geometry / >100k tables.
+        if user_limit and int(user_limit) > 0 and start is None and end is None:
+            try:
+                info_meta = self._info_cached(name)
+                if self._live_pull_blocked(info_meta):
+                    fetch_limit = int(user_limit)
+            except Exception:
+                pass
         params["limit"] = fetch_limit
 
         cache_key = f"{name}:{start}:{end}:{format}:{0 if limit is None else int(limit)}:{as_geo}"
         if self._cache is not None and cache_key in self._cache:
             return self._cache[cache_key]
 
+        data_sources = None
+        provenance = None
         if format == "csv":
             from io import StringIO
             resp = self._raw_get(f"/v1/datasets/{name}/data", params={"format": "csv", **params})
+            provenance = resp.headers
             df   = pd.read_csv(StringIO(resp.text))
         else:
-            df = self._fetch_dataframe(name, params)
+            df, provenance, data_sources = self._fetch_dataframe(
+                name, params, envelope=envelope,
+            )
             if "date" in df.columns:
                 df["date"] = pd.to_datetime(df["date"])
 
@@ -2020,7 +2170,10 @@ class Client:
             return tbl
 
         result = Dataset(df)
-        result = self._attach_dataset_meta(result, name, meta=meta)
+        result = self._attach_dataset_meta(
+            result, name, meta=meta,
+            provenance=provenance, data_sources=data_sources,
+        )
 
         if engine == "polars":
             try:
@@ -2048,13 +2201,27 @@ class Client:
     # HTTP helpers
     # ------------------------------------------------------------------
 
-    def _fetch_dataframe(self, name, params: dict) -> pd.DataFrame:
+    def _fetch_dataframe(
+        self, name, params: dict, *, envelope: bool = False,
+    ) -> tuple[pd.DataFrame, object, Optional[list]]:
         """Fetch dataset rows as a DataFrame, negotiating Arrow IPC over the
         wire (≈5x faster end-to-end, ≈82x faster parse than JSON on large
         pulls — benchmarked 2026-05-18). Transparently falls back to JSON for
         older servers, unexpected content-types, or any pyarrow issue, so the
         returned DataFrame is identical either way.
+
+        Returns ``(dataframe, response_headers, data_sources_or_none)``.
         """
+        if envelope:
+            resp = self._raw_get(
+                f"/v1/datasets/{name}/data",
+                params={**params, "envelope": 1},
+            )
+            payload = resp.json()
+            records = payload.get("data", payload) if isinstance(payload, dict) else payload
+            sources = payload.get("data_sources") if isinstance(payload, dict) else None
+            return pd.DataFrame(records), resp.headers, sources
+
         if self._arrow_supported is not False:
             try:
                 import io
@@ -2067,7 +2234,7 @@ class Client:
                 if "arrow" in resp.headers.get("content-type", ""):
                     self._arrow_supported = True
                     tbl = pa.ipc.open_stream(io.BytesIO(resp.content)).read_all()
-                    return tbl.to_pandas()
+                    return tbl.to_pandas(), resp.headers, None
                 # Old server ignored format=arrow and returned JSON. Remember
                 # so we don't pay the failed round-trip on every future call.
                 self._arrow_supported = False
@@ -2076,9 +2243,11 @@ class Client:
                 self._arrow_supported = False
                 nag_json_transport_once()
 
-        data = self._get(f"/v1/datasets/{name}/data", params=params)
+        resp = self._raw_get(f"/v1/datasets/{name}/data", params=params)
+        data = resp.json()
         records = data.get("data", data) if isinstance(data, dict) else data
-        return pd.DataFrame(records)
+        sources = data.get("data_sources") if isinstance(data, dict) else None
+        return pd.DataFrame(records), resp.headers, sources
 
     def _get(self, path: str, params: Optional[dict] = None) -> dict:
         return self._raw_get(path, params=params).json()

{eolas_data-1.3.2 → eolas_data-1.3.5}/eolas_data/meta.py

@@ -6,6 +6,35 @@ from typing import Any, Optional
 import pandas as pd
 
 # Table-level fields we keep on Dataset.eolas_meta (exclude nested columns).
+_PROVENANCE_HEADERS = {
+    "X-Eolas-Attribution": "attribution_text",
+    "X-Eolas-Licence": "licence",
+    "X-Eolas-Source": "source",
+    "X-Eolas-Source-URL": "source_url",
+    "X-Eolas-Namespace": "namespace",
+}
+
+
+def provenance_from_headers(headers) -> dict:
+    """Extract response provenance from X-Eolas-* headers on /data responses."""
+    out: dict[str, str] = {}
+    get = headers.get if hasattr(headers, "get") else lambda k, d=None: headers.get(k, d)
+    for hdr, key in _PROVENANCE_HEADERS.items():
+        val = get(hdr)
+        if val:
+            out[key] = str(val)
+    return out
+
+
+def merge_provenance(table_meta: Optional[dict], headers) -> dict:
+    """Merge catalogue metadata with live response headers (headers win when set)."""
+    merged = dict(table_meta or {})
+    for key, val in provenance_from_headers(headers).items():
+        if val:
+            merged[key] = val
+    return merged
+
+
 _TABLE_META_KEYS = (
     "name",
     "namespace",

{eolas_data-1.3.2 → eolas_data-1.3.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "eolas-data"
-version = "1.3.2"
+version = "1.3.5"
 description = "Python client for the eolas.fyi statistical data API (NZ, Australia, OECD)"
 readme = "README.md"
 license = { text = "MIT" }
@@ -13,7 +13,7 @@ requires-python = ">=3.10"
 dependencies = [
     "requests>=2.28",
     "pandas>=1.5",
-    "pyarrow>=14",
+    "pyarrow>=24",
     "tqdm>=4.60",
     "rich>=13",
     # typer powers the `eolas` console script, which is always registered in

{eolas_data-1.3.2 → eolas_data-1.3.5}/tests/test_as_arrow.py

@@ -44,7 +44,7 @@ def test_get_live_as_arrow_returns_arrow_table(client):
     """as_arrow=True on the live path converts the JSON-fetched DataFrame to pa.Table."""
     fake_df = pd.DataFrame({"date": ["2023-01-01"], "value": [1100.5]})
 
-    with patch.object(client, "_fetch_dataframe", return_value=fake_df):
+    with patch.object(client, "_fetch_dataframe", return_value=(fake_df, {}, None)):
         result = client.get("nz_cpi", as_arrow=True)
 
     assert isinstance(result, pa.Table)

{eolas_data-1.3.2 → eolas_data-1.3.5}/tests/test_client.py

@@ -153,9 +153,13 @@ def test_get_limit_returns_most_recent(client):
 
 @resp_lib.activate
 def test_get_limit_requests_full_window_from_server(client):
+    resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_cpi",
+                 json={"name": "nz_cpi", "source": "Stats NZ", "has_geometry": False,
+                       "row_count_at_last_refresh": 500})
     resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_cpi/data", json={"data": RECORDS})
     client.get("nz_cpi", limit=2)
-    assert "limit=0" in resp_lib.calls[0].request.url
+    data_reqs = [c for c in resp_lib.calls if "/data?" in c.request.url or c.request.url.endswith("/data")]
+    assert any("limit=0" in c.request.url for c in data_reqs)
 
 
 @resp_lib.activate
@@ -167,6 +171,38 @@ def test_get_passes_date_params(client):
     assert "end=2023-06-30" in req.url
 
 
+@resp_lib.activate
+def test_get_merges_provenance_headers(client):
+    resp_lib.add(
+        resp_lib.GET,
+        f"{BASE}/v1/datasets/nz_cpi/data",
+        json={"data": RECORDS},
+        adding_headers={
+            "X-Eolas-Attribution": "Source: OECD, CC-BY 4.0.",
+            "X-Eolas-Licence": "CC-BY 4.0",
+            "X-Eolas-Source": "OECD",
+        },
+    )
+    df = client.get("nz_cpi", meta=False)
+    meta = df.attrs.get("eolas_meta", {})
+    assert meta.get("attribution_text") == "Source: OECD, CC-BY 4.0."
+    assert meta.get("licence") == "CC-BY 4.0"
+    assert meta.get("source") == "OECD"
+
+
+@resp_lib.activate
+def test_get_envelope_attaches_data_sources(client):
+    payload = {
+        "data_sources": [{"name": "nz_cpi", "licence": "CC-BY 4.0"}],
+        "data": RECORDS,
+    }
+    resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_cpi/data", json=payload)
+    df = client.get("nz_cpi", envelope=True, meta=False)
+    meta = df.attrs.get("eolas_meta", {})
+    assert meta.get("data_sources") == payload["data_sources"]
+    assert "envelope=1" in resp_lib.calls[0].request.url
+
+
 @resp_lib.activate
 def test_get_csv_returns_dataframe(client):
     csv_body = "date,period,value\n2023-01-01,2023Q1,100.0\n"
@@ -213,6 +249,42 @@ def test_get_as_geo_false_keeps_wkt(client):
     assert df["geometry_wkt"].iloc[0].startswith("POINT")
 
 
+LARGE_GEO_META = {
+    "name": "nz_addresses",
+    "namespace": "linz",
+    "bulk_export_class": "materialised",
+    "has_geometry": True,
+    "geometry_type": "point",
+    "row_count_at_last_refresh": 2_418_264,
+}
+
+
+@resp_lib.activate
+def test_get_whole_dataset_defers_to_get_local_for_large_geo(client):
+    """Whole-dataset pulls on geometry tables route to get_local(), not /data."""
+    sentinel = pd.DataFrame({"address_id": [1], "geometry_wkt": ["POINT (0 0)"]})
+    with patch.object(client, "get_local", return_value=sentinel) as mock_local:
+        resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_addresses",
+                     json=LARGE_GEO_META, status=200)
+        result = client.get("nz_addresses")
+    mock_local.assert_called_once()
+    assert result is sentinel
+
+
+@resp_lib.activate
+def test_get_limit_on_large_geo_sends_bounded_limit(client):
+    """Positive limits on geometry tables must not send limit=0 (API 413)."""
+    resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_addresses",
+                 json=LARGE_GEO_META, status=200)
+    resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_addresses/data",
+                 json={"data": GEO_RECORDS})
+    client.get("nz_addresses", limit=2)
+    data_reqs = [c for c in resp_lib.calls if "/data?" in c.request.url or c.request.url.endswith("/data")]
+    assert data_reqs, "expected a /data request"
+    assert any("limit=2" in c.request.url for c in data_reqs)
+    assert not any("limit=0" in c.request.url for c in data_reqs)
+
+
 @resp_lib.activate
 def test_get_no_geometry_column_returns_dataset(client):
     """Datasets without a geometry_wkt column still return a Dataset even with as_geo=None."""

{eolas_data-1.3.2 → eolas_data-1.3.5}/tests/test_progress.py

@@ -421,6 +421,25 @@ def test_eolas_no_progress_env_suppresses_bar(client, tmp_path, monkeypatch):
     assert all(disabled_values), "EOLAS_NO_PROGRESS=1 must disable tqdm even when isatty=True"
 
 
+def test_progress_phase_selectors():
+    from eolas_data.client import Client
+
+    phases = Client._resolve_progress_phases("download")
+    assert phases == {"download": True, "read": False}
+
+    phases = Client._resolve_progress_phases("read")
+    assert phases == {"download": False, "read": True}
+
+    phases = Client._resolve_progress_phases("both")
+    assert phases == {"download": True, "read": True}
+
+    phases = Client._resolve_progress_phases("none")
+    assert phases == {"download": False, "read": False}
+
+    assert Client._resolve_show_progress("read", "read") is True
+    assert Client._resolve_show_progress("download", "read") is False
+
+
 def test_progress_resolver_detects_jupyter():
     """Jupyter wraps stdout so isatty() is False, but the user IS interactive
     and tqdm.auto can render a widget. Resolver must return True when

eolas_data-1.3.2/eolas_data/_regen_names.py

@@ -1,57 +0,0 @@
-"""Regenerate ``_dataset_names.py`` from the live API.
-
-Run before each release:
-
-    python -m eolas_data._regen_names
-
-Writes to the same file inside the package. Commit the result.
-"""
-from __future__ import annotations
-
-import datetime as _dt
-import json as _json
-import pathlib as _pathlib
-import urllib.request as _req
-
-API = "https://api.eolas.fyi/v1/datasets"
-
-
-def regenerate() -> None:
-    with _req.urlopen(API, timeout=30) as r:
-        data = _json.load(r)
-    names = sorted({d["name"] for d in data})
-    today = _dt.date.today().isoformat()
-
-    out = _pathlib.Path(__file__).with_name("_dataset_names.py")
-    lines: list[str] = []
-    lines.append('"""')
-    lines.append('Type stubs for dataset names.')
-    lines.append('')
-    lines.append('Auto-generated from https://api.eolas.fyi/v1/datasets at release time.')
-    lines.append(f'Snapshot: {today} ({len(names)} datasets).')
-    lines.append('Regenerate before each release with `python -m eolas_data._regen_names`.')
-    lines.append('')
-    lines.append('At runtime this is just a string — `Literal[...]` only constrains static type')
-    lines.append("checkers like mypy/pyright, so passing a name not in this list still works,")
-    lines.append("it just doesn't autocomplete.")
-    lines.append('"""')
-    lines.append('from typing import Literal')
-    lines.append('')
-    lines.append(f'CATALOG_SNAPSHOT_DATE = "{today}"')
-    lines.append(f'CATALOG_SNAPSHOT_COUNT = {len(names)}')
-    lines.append('')
-    lines.append('DatasetName = Literal[')
-    for n in names:
-        lines.append(f'    {n!r},')
-    lines.append(']')
-    lines.append('')
-    lines.append('ALL_NAMES: tuple[str, ...] = (')
-    for n in names:
-        lines.append(f'    {n!r},')
-    lines.append(')')
-    out.write_text("\n".join(lines) + "\n")
-    print(f"wrote {len(names)} datasets to {out}")
-
-
-if __name__ == "__main__":
-    regenerate()