eolas-data 1.3.17__tar.gz → 1.3.20__tar.gz

eolas_data-1.3.20/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Virtus Solutions
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

{eolas_data-1.3.17 → eolas_data-1.3.20}/PKG-INFO

@@ -1,13 +1,14 @@
 Metadata-Version: 2.4
 Name: eolas-data
-Version: 1.3.17
-Summary: Python client for the eolas.fyi statistical data API (NZ, Australia, OECD)
+Version: 1.3.20
+Summary: Python client for the eolas.fyi statistical data API (NZ + OECD; Australia on roadmap)
 Project-URL: Homepage, https://eolas.fyi
 Project-URL: Documentation, https://docs.eolas.fyi/
 Project-URL: Repository, https://github.com/phildonovan/eolas-data
 Project-URL: Bug Tracker, https://github.com/phildonovan/eolas-data/issues
 Author-email: Virtus Solutions <phil@virtus-solutions.io>
 License: MIT
+License-File: LICENSE
 Keywords: api,australia,economics,eolas,new-zealand,statistics
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
@@ -54,7 +55,7 @@ Description-Content-Type: text/markdown
 [![Python](https://img.shields.io/pypi/pyversions/eolas-data)](https://pypi.org/project/eolas-data/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
-Python client for the [eolas.fyi](https://eolas.fyi) statistical data API — 1,400+ official New Zealand statistical & geospatial datasets, plus OECD data for international comparisons, served as tidy `pandas` DataFrames (or `polars` / `geopandas` if you prefer).
+Python client for the [eolas.fyi](https://eolas.fyi) statistical data API — 1,500+ official New Zealand statistical & geospatial datasets, plus OECD data for international comparisons, served as tidy `pandas` DataFrames (or `polars` / `geopandas` if you prefer).
 
 _Coverage is New Zealand + OECD today. Australian sources are on the roadmap — not yet available; OECD data already includes Australia (and other OECD members) for cross-country comparisons._
 
@@ -121,7 +122,7 @@ The keyring slot and config file are shared with the R `eolas` client — a key
 
 ## Command-line interface
 
-`pip install eolas-data[cli]` adds an `eolas` command for browsing, fetching, and
+`pip install eolas-data` includes the `eolas` CLI for browsing, fetching, and
 scheduling — useful for shell scripts, cron jobs, and AI-agent workflows. Rich
 tables by default; pass ``--json`` for newline-delimited JSON in scripts.
 
@@ -241,6 +242,30 @@ path = client.download_bulk("treasury_fiscal_spending", path="t.parquet")
 
 CLI mirror: `eolas download <name>` for one-shot, `eolas sync <name> [--watch hourly]` for an incremental check. Full docs: [docs.eolas.fyi/bulk-downloads/](https://docs.eolas.fyi/bulk-downloads/).
 
+## Sync — always-fresh local copy
+
+`client.sync(name, path)` keeps a local file current, automatically choosing *how* based on the dataset's CDC serving tier — you make the same call either way:
+
+- **snapshot-tier** datasets → full-snapshot download, re-fetched only when the server snapshot changes (`sync_bulk()`).
+- **changelog-tier** datasets (e.g. the LINZ SCD2 layers) → incremental: the first call downloads a baseline, then later calls fetch *only what changed* from the `/changes` feed and pk-merge it into your file (`sync_changes()`).
+
+```python
+# Same call regardless of tier:
+r = client.sync("nz_building_outlines", path="buildings.parquet")
+r.status        # "downloaded" (baseline) | "updated" | "unchanged"
+r.sync_mode     # "changelog" for changelog-tier datasets
+r.ops_applied   # number of change rows applied this run
+r.current_seq   # feed watermark after this sync
+
+# First call baselines; subsequent calls apply only new changes:
+r = client.sync("nz_building_outlines", path="buildings.parquet")
+r.ops_applied   # e.g. 1240
+```
+
+A sidecar at `str(path) + ".eolas-meta.json"` records the snapshot id / feed watermark so the next call fetches only new data. For SCD2 datasets the merge keeps only the current rows (`is_current = true`), so `buildings.parquet` is always a clean current-state snapshot — the SCD2 history is handled for you. A `410` (watermark expired) self-heals by re-baselining.
+
+The changelog sidecar (`schema_version` 2) is byte-compatible with the R `eolas` client: a file synced from Python can be resumed from R and vice versa.
+
 ## Geospatial
 
 Datasets with a `geometry_wkt` column auto-convert to `geopandas.GeoDataFrame` if `geopandas` is installed:
@@ -332,4 +357,5 @@ Before each release: `python -m eolas_data._regen_names` to refresh the dataset
 
 ## License
 
-MIT
+MIT — applies to this client software only. Dataset use is subject to each
+source's licence and your [eolas API plan](https://eolas.fyi/#pricing).

{eolas_data-1.3.17 → eolas_data-1.3.20}/README.md

@@ -6,7 +6,7 @@
 [![Python](https://img.shields.io/pypi/pyversions/eolas-data)](https://pypi.org/project/eolas-data/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
-Python client for the [eolas.fyi](https://eolas.fyi) statistical data API — 1,400+ official New Zealand statistical & geospatial datasets, plus OECD data for international comparisons, served as tidy `pandas` DataFrames (or `polars` / `geopandas` if you prefer).
+Python client for the [eolas.fyi](https://eolas.fyi) statistical data API — 1,500+ official New Zealand statistical & geospatial datasets, plus OECD data for international comparisons, served as tidy `pandas` DataFrames (or `polars` / `geopandas` if you prefer).
 
 _Coverage is New Zealand + OECD today. Australian sources are on the roadmap — not yet available; OECD data already includes Australia (and other OECD members) for cross-country comparisons._
 
@@ -73,7 +73,7 @@ The keyring slot and config file are shared with the R `eolas` client — a key
 
 ## Command-line interface
 
-`pip install eolas-data[cli]` adds an `eolas` command for browsing, fetching, and
+`pip install eolas-data` includes the `eolas` CLI for browsing, fetching, and
 scheduling — useful for shell scripts, cron jobs, and AI-agent workflows. Rich
 tables by default; pass ``--json`` for newline-delimited JSON in scripts.
 
@@ -193,6 +193,30 @@ path = client.download_bulk("treasury_fiscal_spending", path="t.parquet")
 
 CLI mirror: `eolas download <name>` for one-shot, `eolas sync <name> [--watch hourly]` for an incremental check. Full docs: [docs.eolas.fyi/bulk-downloads/](https://docs.eolas.fyi/bulk-downloads/).
 
+## Sync — always-fresh local copy
+
+`client.sync(name, path)` keeps a local file current, automatically choosing *how* based on the dataset's CDC serving tier — you make the same call either way:
+
+- **snapshot-tier** datasets → full-snapshot download, re-fetched only when the server snapshot changes (`sync_bulk()`).
+- **changelog-tier** datasets (e.g. the LINZ SCD2 layers) → incremental: the first call downloads a baseline, then later calls fetch *only what changed* from the `/changes` feed and pk-merge it into your file (`sync_changes()`).
+
+```python
+# Same call regardless of tier:
+r = client.sync("nz_building_outlines", path="buildings.parquet")
+r.status        # "downloaded" (baseline) | "updated" | "unchanged"
+r.sync_mode     # "changelog" for changelog-tier datasets
+r.ops_applied   # number of change rows applied this run
+r.current_seq   # feed watermark after this sync
+
+# First call baselines; subsequent calls apply only new changes:
+r = client.sync("nz_building_outlines", path="buildings.parquet")
+r.ops_applied   # e.g. 1240
+```
+
+A sidecar at `str(path) + ".eolas-meta.json"` records the snapshot id / feed watermark so the next call fetches only new data. For SCD2 datasets the merge keeps only the current rows (`is_current = true`), so `buildings.parquet` is always a clean current-state snapshot — the SCD2 history is handled for you. A `410` (watermark expired) self-heals by re-baselining.
+
+The changelog sidecar (`schema_version` 2) is byte-compatible with the R `eolas` client: a file synced from Python can be resumed from R and vice versa.
+
 ## Geospatial
 
 Datasets with a `geometry_wkt` column auto-convert to `geopandas.GeoDataFrame` if `geopandas` is installed:
@@ -284,4 +308,5 @@ Before each release: `python -m eolas_data._regen_names` to refresh the dataset
 
 ## License
 
-MIT
+MIT — applies to this client software only. Dataset use is subject to each
+source's licence and your [eolas API plan](https://eolas.fyi/#pricing).

{eolas_data-1.3.17 → eolas_data-1.3.20}/eolas_data/__init__.py

@@ -15,7 +15,7 @@ from .exceptions import (
     WatermarkExpired,
 )
 
-__version__ = "1.3.17"
+__version__ = "1.3.20"
 
 __all__ = [
     "Client",

{eolas_data-1.3.17 → eolas_data-1.3.20}/eolas_data/cli.py

@@ -342,7 +342,7 @@ def get_cmd(
 
 
 # ────────────────────────────────────────────────────────────────────────────
-# download command — bulk single-file downloads via /v1/bulk/{namespace}/{table}
+# download command — bulk snapshot when available; live API otherwise (OECD/CPI)
 # ────────────────────────────────────────────────────────────────────────────
 
 # Map CLI --format aliases to the values the server (and client.download_bulk) accept.
@@ -353,6 +353,14 @@ _DOWNLOAD_FORMAT_MAP = {
     "geoparquet": "geoparquet",
 }
 
+# Live /v1/datasets/{name}/data formats when bulk export is unavailable.
+_LIVE_DOWNLOAD_FORMAT_MAP = {
+    "parquet":    "parquet",
+    "csv":        "csv",
+    "csv_gz":     "csv",
+    "geoparquet": "parquet",
+}
+
 # Default output-file extensions for each format (used when --out is not set).
 _DOWNLOAD_EXT = {
     "parquet":    ".parquet",
@@ -360,6 +368,13 @@ _DOWNLOAD_EXT = {
     "geoparquet": ".geo.parquet",
 }
 
+_LIVE_DOWNLOAD_EXT = {
+    "parquet":    ".parquet",
+    "csv":        ".csv",
+    "csv_gz":     ".csv",
+    "geoparquet": ".parquet",
+}
+
 
 @app.command(name="download")
 def download_cmd(
@@ -386,18 +401,22 @@ def download_cmd(
         False, "--no-progress",
         help="Disable the download progress bar (useful when output is captured by a log collector).",
     ),
+    show_progress: bool        = typer.Option(
+        False, "--progress",
+        help="Always show the download progress bar (even when stdout/stderr are not a TTY).",
+    ),
     api_key:     Optional[str]  = typer.Option(None, "--api-key"),
 ) -> None:
-    """Download a complete dataset as a single file (Parquet, CSV.gz, or GeoParquet).
+    """Download a dataset as a single file (Parquet, CSV, or GeoParquet).
 
-    Uses the /v1/bulk/{namespace}/{table} endpoint. Monthly snapshots for Free
-    accounts are served from Cloudflare's edge cache. Pro accounts get the
-    current Iceberg snapshot on demand.
+    Uses /v1/bulk when the dataset permits bulk export. OECD and other
+    licence-restricted datasets (e.g. nz_cpi) are downloaded via the live
+    /v1/datasets/{name}/data endpoint instead.
 
     Examples
     --------
         eolas download nz_cpi
-        eolas download nz_cpi --format csv --out cpi.csv.gz
+        eolas download nz_cpi --format csv --out cpi.csv
         eolas download nz_cpi --freshness monthly
         eolas download territorial_authority_2023 --format geoparquet
     """
@@ -413,37 +432,85 @@ def download_cmd(
             EXIT_USAGE,
         )
 
-    server_fmt = _DOWNLOAD_FORMAT_MAP[fmt_lower]
-    if out is None:
-        ext = _DOWNLOAD_EXT[server_fmt]
-        out = Path.cwd() / f"{name}{ext}"
-    else:
-        out = out.expanduser().resolve()
-
+    client = _client(api_key)
+    use_live = False
     try:
-        result_path = _client(api_key).download_bulk(
-            name,
-            freshness=freshness,
-            format=server_fmt,
-            path=out,
-            progress=False if no_progress else None,
-        )
-    except BulkUpgradeRequired as e:
-        err_console.print(f"[red]error:[/red] {e}")
-        err_console.print("[dim]→ https://eolas.fyi/pricing[/dim]")
-        raise typer.Exit(code=EXIT_AUTH)
-    except BulkLicenceRestricted as e:
-        err_console.print(f"[red]error:[/red] {e}")
-        err_console.print(
-            "[dim]Use `eolas get` to query this dataset via the live API instead.[/dim]"
-        )
-        raise typer.Exit(code=EXIT_AUTH)
-    except BulkNotYetAvailable as e:
-        err_console.print(f"[yellow]unavailable:[/yellow] {e}")
-        raise typer.Exit(code=EXIT_API)
+        meta = client.info(name)
+        # Only OECD/licence-blocked datasets (bulk_export_class=none) use the
+        # live /data path by default. Missing field → bulk (same as download_bulk).
+        use_live = (meta.get("bulk_export_class") or "").lower() == "none"
     except EolasError as e:
         _bail(str(e), _exit_for(e))
 
+    if show_progress and no_progress:
+        _bail("--progress and --no-progress are mutually exclusive", EXIT_USAGE)
+    if show_progress:
+        _progress: Optional[bool] = True
+    elif no_progress:
+        _progress = False
+    else:
+        _progress = None
+
+    if use_live:
+        if freshness != "auto":
+            err_console.print(
+                "[yellow]note:[/yellow] --freshness is ignored for live API downloads "
+                "(bulk snapshots are not available for this dataset)."
+            )
+        live_fmt = _LIVE_DOWNLOAD_FORMAT_MAP[fmt_lower]
+        reported_fmt = live_fmt
+        if out is None:
+            out = Path.cwd() / f"{name}{_LIVE_DOWNLOAD_EXT[fmt_lower]}"
+        else:
+            out = out.expanduser().resolve()
+        try:
+            result_path = client.download(
+                name,
+                path=out,
+                format=live_fmt,
+                progress=_progress,
+            )
+        except EolasError as e:
+            _bail(str(e), _exit_for(e))
+    else:
+        server_fmt = _DOWNLOAD_FORMAT_MAP[fmt_lower]
+        reported_fmt = server_fmt
+        if out is None:
+            ext = _DOWNLOAD_EXT[server_fmt]
+            out = Path.cwd() / f"{name}{ext}"
+        else:
+            out = out.expanduser().resolve()
+
+        try:
+            result_path = client.download_bulk(
+                name,
+                freshness=freshness,
+                format=server_fmt,
+                path=out,
+                progress=_progress,
+            )
+        except BulkUpgradeRequired as e:
+            err_console.print(f"[red]error:[/red] {e}")
+            err_console.print("[dim]→ https://eolas.fyi/pricing[/dim]")
+            raise typer.Exit(code=EXIT_AUTH)
+        except BulkLicenceRestricted:
+            live_fmt = _LIVE_DOWNLOAD_FORMAT_MAP[fmt_lower]
+            reported_fmt = live_fmt
+            try:
+                result_path = client.download(
+                    name,
+                    path=out,
+                    format=live_fmt,
+                    progress=_progress,
+                )
+            except EolasError as e:
+                _bail(str(e), _exit_for(e))
+        except BulkNotYetAvailable as e:
+            err_console.print(f"[yellow]unavailable:[/yellow] {e}")
+            raise typer.Exit(code=EXIT_API)
+        except EolasError as e:
+            _bail(str(e), _exit_for(e))
+
     size_bytes = result_path.stat().st_size
     if size_bytes >= 1_048_576:
         size_str = f"{size_bytes / 1_048_576:.1f} MB"
@@ -465,7 +532,7 @@ def download_cmd(
             json.dumps({
                 "path": str(result_path),
                 "bytes": size_bytes,
-                "format": server_fmt,
+                "format": reported_fmt,
                 "freshness": freshness,
             })
         )

{eolas_data-1.3.17 → eolas_data-1.3.20}/eolas_data/client.py

@@ -365,6 +365,13 @@ class Client:
         "csv_gz":     ".csv.gz",
         "geoparquet": ".geo.parquet",
     }
+    _LIVE_DOWNLOAD_FORMATS = {"csv", "parquet", "arrow", "json"}
+    _LIVE_DOWNLOAD_EXTENSIONS = {
+        "csv":     ".csv",
+        "parquet": ".parquet",
+        "arrow":   ".arrow",
+        "json":    ".json",
+    }
     # Mirrors the API guard in datasets.py — unbounded live pulls on datasets
     # above this row count (or with geometry) return HTTP 413.
     _LARGE_DATASET_ROW_THRESHOLD = 100_000
@@ -508,6 +515,86 @@ class Client:
         )
         return out
 
+    def download(
+        self,
+        name: Union[str, "DatasetName"],
+        *,
+        path: Optional[Union[str, "pathlib.Path"]] = None,
+        format: str = "csv",
+        start: Optional[str] = None,
+        end: Optional[str] = None,
+        limit: Optional[int] = None,
+        progress: ProgressControl = None,
+    ) -> "Union[pathlib.Path, bytes]":
+        """Download a dataset via the live ``/v1/datasets/{name}/data`` endpoint.
+
+        Works for **all** datasets — including OECD and other licence-restricted
+        tables where bulk export is unavailable (e.g. ``nz_cpi``). For whole-dataset
+        pulls on very large or geospatial tables, prefer :meth:`download_bulk` when
+        bulk export is permitted.
+
+        Args:
+            name: Dataset identifier, e.g. ``"nz_cpi"``.
+            path: Where to write the file. ``None`` (default) returns raw bytes.
+            format: ``"csv"`` (default), ``"parquet"``, ``"arrow"``, or ``"json"``.
+            start: ISO date lower bound.
+            end: ISO date upper bound.
+            limit: Max rows. ``None`` (default) requests the full dataset (subject
+                to plan caps). Pass an integer to cap rows.
+            progress: Download progress bar control (``"download"`` phase only).
+
+        Returns:
+            ``pathlib.Path`` when ``path`` is set; ``bytes`` when ``path`` is ``None``.
+
+        Examples::
+
+            client.download("nz_cpi", path="nz_cpi.csv")
+            client.download("nz_cpi", format="parquet", path="nz_cpi.parquet")
+            raw = client.download("nz_cpi", format="csv")
+        """
+        fmt = format.lower()
+        if fmt not in self._LIVE_DOWNLOAD_FORMATS:
+            raise ValueError(
+                f"Unknown format {format!r}. Expected one of: "
+                f"{', '.join(sorted(self._LIVE_DOWNLOAD_FORMATS))}."
+            )
+
+        params: dict = {"format": fmt}
+        if start:
+            params["start"] = start
+        if end:
+            params["end"] = end
+        if limit is not None:
+            from .rows import resolve_fetch_limit
+            fetch_limit, _ = resolve_fetch_limit(limit)
+            params["limit"] = fetch_limit
+        elif start is None and end is None:
+            params["limit"] = 0
+        else:
+            params["limit"] = 0
+
+        if path is None:
+            resp = self._raw_get(f"/v1/datasets/{name}/data", params=params)
+            return resp.content
+
+        out = pathlib.Path(path).expanduser().resolve()
+        if not out.suffix and fmt in self._LIVE_DOWNLOAD_EXTENSIONS:
+            out = out.with_suffix(self._LIVE_DOWNLOAD_EXTENSIONS[fmt])
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        show = self._resolve_show_progress(progress, "download")
+        resp = self._raw_get(
+            f"/v1/datasets/{name}/data", params=params, stream=True,
+        )
+        total = int(resp.headers.get("Content-Length", 0)) or None
+        self._stream_to_file_with_progress(
+            resp, out,
+            total_bytes=total,
+            label=f"Downloading {out.name}",
+            show_progress=show,
+        )
+        return out
+
     def sync_bulk(
         self,
         name: Union[str, "DatasetName"],
@@ -1209,7 +1296,9 @@ class Client:
             return False
         if "ipykernel" in sys.modules:
             return True
-        return sys.stdout.isatty()
+        # tqdm writes to stderr; many terminals/IDEs pipe stdout but keep stderr
+        # on the controlling TTY (Cursor, VS Code, script wrappers).
+        return sys.stderr.isatty() or sys.stdout.isatty()
 
     @staticmethod
     def _resolve_progress_phases(progress: ProgressControl) -> dict[str, bool]:
@@ -1314,8 +1403,10 @@ class Client:
             unit_scale=True,
             unit_divisor=1024,
             desc=label,
+            file=sys.stderr,
+            dynamic_ncols=True,
             disable=not show_progress,
-            leave=False,
+            leave=True,
         ) as bar:
             with dest.open("wb") as fh:
                 # 1 MiB chunks: responsive bar updates (bar refreshes ~once per
@@ -2136,8 +2227,27 @@ class Client:
                 "envelope=True requires format='json' and as_arrow=False."
             )
 
+        from .meta import resolve_date_bounds
         from .rows import apply_row_limit, resolve_fetch_limit, sort_by_date
 
+        # ---- start/end only apply when the dataset has a date filter column ---
+        _date_bounds_info: Optional[dict] = None
+        if start is not None or end is not None:
+            try:
+                _date_bounds_info = self._info_cached(name)
+            except Exception:
+                pass
+            start, end, _stripped = resolve_date_bounds(_date_bounds_info, start, end)
+            if _stripped:
+                import warnings
+                warnings.warn(
+                    f"start=/end= ignored for {name!r}: this dataset has no date "
+                    "filter column (not a time-series table). Use limit= for row "
+                    "caps or get_local() for the full table.",
+                    UserWarning,
+                    stacklevel=2,
+                )
+
         # ---- whole-dataset pull on large/geo tables → bulk cache -------------
         # Matches the API 413 guard: limit=0 with no start/end on >100k-row or
         # geometry datasets is refused. Transparently serve from get_local()
@@ -2315,9 +2425,15 @@ class Client:
     def _get(self, path: str, params: Optional[dict] = None) -> dict:
         return self._raw_get(path, params=params).json()
 
-    def _raw_get(self, path: str, params: Optional[dict] = None) -> requests.Response:
+    def _raw_get(
+        self,
+        path: str,
+        params: Optional[dict] = None,
+        *,
+        stream: bool = False,
+    ) -> requests.Response:
         url  = f"{self._base}{path}"
-        resp = self._session.get(url, params=params)
+        resp = self._session.get(url, params=params, stream=stream)
         self._raise_for_status(resp)
         return resp
 

{eolas_data-1.3.17 → eolas_data-1.3.20}/eolas_data/meta.py

@@ -55,6 +55,16 @@ _TABLE_META_KEYS = (
     "refresh_cadence",
     "last_refreshed_at",
     "previous_snapshots",
+    "date_filter_column",
+)
+
+# Mirrors app/streaming.py::detect_date_col — columns that /data?start=&end= filter on.
+DATE_FILTER_CANDIDATES = (
+    "date",
+    "time_frame",
+    "open_date",
+    "awarded_date",
+    "start_date",
 )
 
 
@@ -172,4 +182,44 @@ def column_label(
     if val is None or (isinstance(val, float) and pd.isna(val)):
         return None
     text = str(val).strip()
-    return text or None
+    return text or None
+
+
+def date_filter_column_from_info(info: dict) -> Optional[str]:
+    """Return the column name used for start/end filtering, or None if unsupported."""
+    if not info:
+        return None
+    if "date_filter_column" in info:
+        col = info.get("date_filter_column")
+        return col if col else None
+    raw_cols = info.get("columns")
+    if not raw_cols:
+        return None
+    if isinstance(raw_cols, pd.DataFrame):
+        names = set(raw_cols["name"].astype(str)) if "name" in raw_cols.columns else set()
+    else:
+        names = {
+            str(c["name"])
+            for c in raw_cols
+            if isinstance(c, dict) and c.get("name")
+        }
+    return next((c for c in DATE_FILTER_CANDIDATES if c in names), None)
+
+
+def resolve_date_bounds(
+    info: Optional[dict],
+    start: Optional[str],
+    end: Optional[str],
+) -> tuple[Optional[str], Optional[str], bool]:
+    """Return (start, end, stripped) — clear bounds when the dataset has no date axis.
+
+    When stripped is True, callers should warn that start/end were ignored.
+    If metadata is unavailable, bounds are passed through unchanged (safe default).
+    """
+    if start is None and end is None:
+        return None, None, False
+    if info is None:
+        return start, end, False
+    if date_filter_column_from_info(info):
+        return start, end, False
+    return None, None, True

{eolas_data-1.3.17 → eolas_data-1.3.20}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "hatchling.build"
 
 [project]
 name = "eolas-data"
-version = "1.3.17"
-description = "Python client for the eolas.fyi statistical data API (NZ, Australia, OECD)"
+version = "1.3.20"
+description = "Python client for the eolas.fyi statistical data API (NZ + OECD; Australia on roadmap)"
 readme = "README.md"
 license = { text = "MIT" }
 authors = [{ name = "Virtus Solutions", email = "phil@virtus-solutions.io" }]

{eolas_data-1.3.17 → eolas_data-1.3.20}/tests/test_cli.py

@@ -597,7 +597,10 @@ def test_download_csv_format(tmp_path, monkeypatch):
                                  "--format", "csv", "--api-key", "k"])
     assert result.exit_code == 0
     assert (tmp_path / "nz_cpi.csv.gz").exists()
-    bulk_req = resp_lib.calls[1].request
+    bulk_req = next(
+        c.request for c in resp_lib.calls
+        if "/v1/bulk/" in c.request.url
+    )
     assert "format=csv_gz" in bulk_req.url
 
 
@@ -617,18 +620,21 @@ def test_download_402_exits_auth_code(tmp_path, monkeypatch):
 
 
 @resp_lib.activate
-def test_download_403_licence_exits_auth_code(tmp_path, monkeypatch):
-    """HTTP 403 licence restriction should exit with EXIT_AUTH."""
+def test_download_403_licence_falls_back_to_live_api(tmp_path, monkeypatch):
+    """bulk_export_class=none uses live /data; bulk 403 also falls back."""
+    live_csv = b"date,period,value\n2023-01-01,2023Q1,100.0\n"
     resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/oecd_gdp",
                  json={**BULK_DATASET_META, "name": "oecd_gdp",
-                       "namespace": "oecd", "table": "oecd_gdp"},
+                       "namespace": "oecd", "table": "oecd_gdp",
+                       "bulk_export_class": "none"},
                  status=200)
-    resp_lib.add(resp_lib.GET, f"{BASE}/v1/bulk/oecd/oecd_gdp",
-                 json={"detail": "This dataset is not available as a bulk download (licence: OECD)."},
-                 status=403)
+    resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/oecd_gdp/data",
+                 body=live_csv, content_type="text/csv", status=200)
     monkeypatch.chdir(tmp_path)
-    result = runner.invoke(app, ["download", "oecd_gdp", "--api-key", "k"])
-    assert result.exit_code == cli_module.EXIT_AUTH
+    result = runner.invoke(app, ["download", "oecd_gdp",
+                                 "--format", "csv", "--api-key", "k"])
+    assert result.exit_code == 0
+    assert (tmp_path / "oecd_gdp.csv").read_bytes() == live_csv
 
 
 @resp_lib.activate

{eolas_data-1.3.17 → eolas_data-1.3.20}/tests/test_client.py

@@ -164,9 +164,16 @@ def test_get_limit_requests_full_window_from_server(client):
 
 @resp_lib.activate
 def test_get_passes_date_params(client):
+    resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_cpi",
+                 json={"name": "nz_cpi", "date_filter_column": "date"})
     resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_cpi/data", json={"data": RECORDS})
     client.get("nz_cpi", start="2023-01-01", end="2023-06-30")
-    req = resp_lib.calls[0].request
+    data_reqs = [
+        c for c in resp_lib.calls
+        if c.request.url.endswith("/data") or "/data?" in c.request.url
+    ]
+    assert data_reqs, "expected a /data request"
+    req = data_reqs[0].request
     assert "start=2023-01-01" in req.url
     assert "end=2023-06-30" in req.url
 
@@ -447,6 +454,54 @@ def test_integration_403_raises_authentication_error_with_server_detail(client):
     assert "Enterprise" in str(e.value)
 
 
+# ---------------------------------------------------------------------------
+# Client.download() — live /v1/datasets/{name}/data
+# ---------------------------------------------------------------------------
+
+FAKE_CSV = b"date,period,value\n2023-01-01,2023Q1,100.0\n"
+
+
+@resp_lib.activate
+def test_download_live_writes_csv(client, tmp_path):
+    resp_lib.add(
+        resp_lib.GET,
+        f"{BASE}/v1/datasets/nz_cpi/data",
+        body=FAKE_CSV,
+        content_type="text/csv",
+        status=200,
+    )
+    dest = tmp_path / "nz_cpi.csv"
+    result = client.download("nz_cpi", path=dest)
+    import pathlib
+    assert isinstance(result, pathlib.Path)
+    assert result.read_bytes() == FAKE_CSV
+
+
+@resp_lib.activate
+def test_download_live_sends_limit_zero_for_full_dataset(client, tmp_path):
+    resp_lib.add(
+        resp_lib.GET,
+        f"{BASE}/v1/datasets/nz_cpi/data",
+        body=FAKE_CSV,
+        content_type="text/csv",
+        status=200,
+    )
+    client.download("nz_cpi", path=tmp_path / "nz_cpi.csv")
+    assert "limit=0" in resp_lib.calls[0].request.url
+
+
+@resp_lib.activate
+def test_download_live_returns_bytes_when_no_path(client):
+    resp_lib.add(
+        resp_lib.GET,
+        f"{BASE}/v1/datasets/nz_cpi/data",
+        body=FAKE_CSV,
+        content_type="text/csv",
+        status=200,
+    )
+    assert client.download("nz_cpi") == FAKE_CSV
+
+
 # ---------------------------------------------------------------------------
 # Client.download_bulk() — /v1/bulk/{namespace}/{table}
 # ---------------------------------------------------------------------------

eolas_data-1.3.20/tests/test_date_filter.py

@@ -0,0 +1,81 @@
+"""Tests for start/end date-bound resolution on non-temporal datasets."""
+from __future__ import annotations
+
+import warnings
+
+import pandas as pd
+import pytest
+import responses as resp_lib
+from unittest.mock import patch
+
+from eolas_data import Client
+from eolas_data.meta import (
+    date_filter_column_from_info,
+    resolve_date_bounds,
+)
+
+BASE = "https://api.eolas.fyi"
+
+TEMPORAL_INFO = {
+    "name": "nz_cpi",
+    "date_filter_column": "date",
+    "columns": [{"name": "date"}, {"name": "value"}],
+}
+
+NON_TEMPORAL_INFO = {
+    "name": "nz_addresses",
+    "date_filter_column": None,
+    "has_geometry": True,
+    "bulk_export_class": "materialised",
+    "row_count_at_last_refresh": 2_418_264,
+    "columns": [{"name": "address_id"}, {"name": "geometry_wkt"}],
+}
+
+
+@pytest.fixture()
+def client():
+    return Client("eolas_testkey123", base_url=BASE)
+
+
+def test_date_filter_column_from_explicit_field():
+    assert date_filter_column_from_info(TEMPORAL_INFO) == "date"
+    assert date_filter_column_from_info(NON_TEMPORAL_INFO) is None
+
+
+def test_date_filter_column_inferred_from_columns():
+    info = {"columns": [{"name": "awarded_date"}, {"name": "amount"}]}
+    assert date_filter_column_from_info(info) == "awarded_date"
+
+
+def test_resolve_date_bounds_keeps_temporal():
+    start, end, stripped = resolve_date_bounds(TEMPORAL_INFO, "2020-01-01", "2024-12-31")
+    assert start == "2020-01-01"
+    assert end == "2024-12-31"
+    assert stripped is False
+
+
+def test_resolve_date_bounds_strips_non_temporal():
+    start, end, stripped = resolve_date_bounds(NON_TEMPORAL_INFO, "2020-01-01", None)
+    assert start is None
+    assert end is None
+    assert stripped is True
+
+
+def test_resolve_date_bounds_unknown_metadata_passthrough():
+    start, end, stripped = resolve_date_bounds(None, "2020-01-01", None)
+    assert start == "2020-01-01"
+    assert stripped is False
+
+
+@resp_lib.activate
+def test_get_start_on_non_temporal_warns_and_routes_to_get_local(client):
+    """start= on a geo table must not block smart-routing to get_local()."""
+    sentinel = pd.DataFrame({"address_id": [1], "geometry_wkt": ["POINT (0 0)"]})
+    with (
+        patch.object(client, "_info_cached", return_value=NON_TEMPORAL_INFO),
+        patch.object(client, "get_local", return_value=sentinel) as mock_local,
+        pytest.warns(UserWarning, match="start=/end= ignored"),
+    ):
+        result = client.get("nz_addresses", start="2020-01-01")
+    mock_local.assert_called_once()
+    assert result is sentinel

{eolas_data-1.3.17 → eolas_data-1.3.20}/tests/test_progress.py

@@ -478,6 +478,20 @@ def test_progress_phase_selectors():
     assert Client._resolve_show_progress("download", "read") is False
 
 
+def test_progress_auto_detect_uses_stderr_tty():
+    """tqdm writes to stderr; progress should show when stderr is a TTY even if stdout is piped."""
+    from unittest.mock import patch
+    from eolas_data.client import Client
+
+    with patch("sys.stdout.isatty", return_value=False), \
+         patch("sys.stderr.isatty", return_value=True):
+        assert Client._progress_auto_detect() is True
+
+    with patch("sys.stdout.isatty", return_value=False), \
+         patch("sys.stderr.isatty", return_value=False):
+        assert Client._progress_auto_detect() is False
+
+
 def test_progress_resolver_detects_jupyter():
     """Jupyter wraps stdout so isatty() is False, but the user IS interactive
     and tqdm.auto can render a widget. Resolver must return True when