eolas-data 1.3.14__tar.gz → 1.3.17__tar.gz

{eolas_data-1.3.14 → eolas_data-1.3.17}/.github/workflows/catalog-drift.yml

@@ -17,7 +17,7 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - uses: actions/setup-python@v5
         with:

{eolas_data-1.3.14 → eolas_data-1.3.17}/.github/workflows/pre-release.yml

@@ -9,7 +9,7 @@ jobs:
     env:
       PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - uses: actions/setup-python@v5
         with:
           python-version: "3.12"

{eolas_data-1.3.14 → eolas_data-1.3.17}/.github/workflows/publish.yml

@@ -9,7 +9,7 @@ jobs:
   version-parity:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Compare tag to eolas-r DESCRIPTION
         env:
           TAG: ${{ github.ref_name }}
@@ -34,7 +34,7 @@ jobs:
     env:
       PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - uses: actions/setup-python@v5
         with:
@@ -59,7 +59,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: test
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
@@ -84,7 +84,7 @@ jobs:
       id-token: write
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - uses: actions/setup-python@v5
         with:

{eolas_data-1.3.14 → eolas_data-1.3.17}/.github/workflows/smoke.yml

@@ -9,7 +9,7 @@ jobs:
   smoke:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - uses: actions/setup-python@v5
         with:

{eolas_data-1.3.14 → eolas_data-1.3.17}/.github/workflows/test.yml

@@ -9,7 +9,7 @@ jobs:
   contract:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
@@ -26,7 +26,7 @@ jobs:
     env:
       PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - uses: actions/setup-python@v5
         with:
@@ -44,7 +44,7 @@ jobs:
     env:
       PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - uses: actions/setup-python@v5
         with:
@@ -65,17 +65,20 @@ jobs:
   coverage:
     runs-on: ubuntu-latest
     needs: contract
+    permissions:
+      id-token: write
     env:
       PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
       - run: pip install -e ".[dev,geo,secure]" pytest-cov
       - run: pytest -q -m "not integration" --cov=eolas_data --cov-report=xml
       - name: Upload coverage
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           files: coverage.xml
+          token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: false

{eolas_data-1.3.14 → eolas_data-1.3.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eolas-data
-Version: 1.3.14
+Version: 1.3.17
 Summary: Python client for the eolas.fyi statistical data API (NZ, Australia, OECD)
 Project-URL: Homepage, https://eolas.fyi
 Project-URL: Documentation, https://docs.eolas.fyi/
@@ -48,6 +48,12 @@ Description-Content-Type: text/markdown
 
 # eolas-data
 
+[![Tests](https://github.com/phildonovan/eolas-data/actions/workflows/test.yml/badge.svg)](https://github.com/phildonovan/eolas-data/actions/workflows/test.yml)
+[![codecov](https://codecov.io/gh/phildonovan/eolas-data/graph/badge.svg)](https://codecov.io/gh/phildonovan/eolas-data)
+[![PyPI](https://img.shields.io/pypi/v/eolas-data)](https://pypi.org/project/eolas-data/)
+[![Python](https://img.shields.io/pypi/pyversions/eolas-data)](https://pypi.org/project/eolas-data/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
 Python client for the [eolas.fyi](https://eolas.fyi) statistical data API — 1,400+ official New Zealand statistical & geospatial datasets, plus OECD data for international comparisons, served as tidy `pandas` DataFrames (or `polars` / `geopandas` if you prefer).
 
 _Coverage is New Zealand + OECD today. Australian sources are on the roadmap — not yet available; OECD data already includes Australia (and other OECD members) for cross-country comparisons._
@@ -316,7 +322,7 @@ pytest -q -m "not integration"
 EOLAS_API_KEY=vs_... pytest -q -m integration tests/test_smoke_live.py
 ```
 
-CI runs the unit suite on Python 3.10, 3.12, and 3.13 on every push/PR. A weekly workflow optionally runs live smoke tests when `EOLAS_API_KEY` is configured as a repository secret.
+CI runs the unit suite on Python 3.10, 3.12, and 3.13 on every push/PR, with [coverage](https://codecov.io/gh/phildonovan/eolas-data) uploaded to Codecov. A weekly workflow optionally runs live smoke tests when `EOLAS_API_KEY` is configured as a repository secret.
 
 ## Releasing
 

{eolas_data-1.3.14 → eolas_data-1.3.17}/README.md

@@ -1,5 +1,11 @@
 # eolas-data
 
+[![Tests](https://github.com/phildonovan/eolas-data/actions/workflows/test.yml/badge.svg)](https://github.com/phildonovan/eolas-data/actions/workflows/test.yml)
+[![codecov](https://codecov.io/gh/phildonovan/eolas-data/graph/badge.svg)](https://codecov.io/gh/phildonovan/eolas-data)
+[![PyPI](https://img.shields.io/pypi/v/eolas-data)](https://pypi.org/project/eolas-data/)
+[![Python](https://img.shields.io/pypi/pyversions/eolas-data)](https://pypi.org/project/eolas-data/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
 Python client for the [eolas.fyi](https://eolas.fyi) statistical data API — 1,400+ official New Zealand statistical & geospatial datasets, plus OECD data for international comparisons, served as tidy `pandas` DataFrames (or `polars` / `geopandas` if you prefer).
 
 _Coverage is New Zealand + OECD today. Australian sources are on the roadmap — not yet available; OECD data already includes Australia (and other OECD members) for cross-country comparisons._
@@ -268,7 +274,7 @@ pytest -q -m "not integration"
 EOLAS_API_KEY=vs_... pytest -q -m integration tests/test_smoke_live.py
 ```
 
-CI runs the unit suite on Python 3.10, 3.12, and 3.13 on every push/PR. A weekly workflow optionally runs live smoke tests when `EOLAS_API_KEY` is configured as a repository secret.
+CI runs the unit suite on Python 3.10, 3.12, and 3.13 on every push/PR, with [coverage](https://codecov.io/gh/phildonovan/eolas-data) uploaded to Codecov. A weekly workflow optionally runs live smoke tests when `EOLAS_API_KEY` is configured as a repository secret.
 
 ## Releasing
 

{eolas_data-1.3.14 → eolas_data-1.3.17}/eolas_data/__init__.py

@@ -15,7 +15,7 @@ from .exceptions import (
     WatermarkExpired,
 )
 
-__version__ = "1.3.13"
+__version__ = "1.3.17"
 
 __all__ = [
     "Client",

{eolas_data-1.3.14 → eolas_data-1.3.17}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "eolas-data"
-version = "1.3.14"
+version = "1.3.17"
 description = "Python client for the eolas.fyi statistical data API (NZ, Australia, OECD)"
 readme = "README.md"
 license = { text = "MIT" }
@@ -58,3 +58,8 @@ packages = ["eolas_data"]
 markers = [
     "integration: marks tests as requiring live API access (deselected by default)",
 ]
+
+[tool.coverage.run]
+omit = [
+    "eolas_data/_regen_names.py",
+]

eolas_data-1.3.17/tests/fixtures/source_helpers.json

@@ -0,0 +1,37 @@
+{
+  "acc": "ACC",
+  "akl_council": "Auckland Council",
+  "akl_transport": "Auckland Transport",
+  "bay_of_plenty": "Bay of Plenty Councils",
+  "charities": "Charities Services",
+  "colab_waikato": "Co-Lab Waikato",
+  "doc": "DOC",
+  "ecan_canterbury": "ECan / Canterbury",
+  "edcounts": "Education Counts",
+  "eeca": "EECA",
+  "geonet": "GeoNet",
+  "hawkes_bay": "Hawke's Bay Councils",
+  "immigration": "Immigration NZ",
+  "linz": "LINZ",
+  "lris": "Manaaki Whenua / LRIS",
+  "manawatu_whanganui": "Manawatu-Whanganui Councils",
+  "mbie": "MBIE",
+  "msd": "MSD",
+  "napier_whanganui": "Napier + Whanganui",
+  "northland": "Northland Councils",
+  "nzta": "Waka Kotahi",
+  "oecd": "OECD",
+  "otago": "Otago Councils",
+  "pharmac": "PHARMAC",
+  "police": "NZ Police / MoJ",
+  "rbnz": "RBNZ",
+  "southland": "Southland Councils",
+  "statsnz": "Stats NZ",
+  "statsnz_geo": "Stats NZ",
+  "taranaki": "Taranaki Councils",
+  "top_of_south": "Gisborne / Top of South Councils",
+  "treasury": "NZ Treasury",
+  "wellington": "Wellington Region Councils",
+  "west_coast": "West Coast (Te Tai o Poutini)",
+  "worksafe": "WorkSafe NZ"
+}

eolas_data-1.3.17/tests/test_changes_http.py

@@ -0,0 +1,51 @@
+"""HTTP error-path tests for Client._raw_changes_get (changelog endpoint)."""
+from __future__ import annotations
+
+import pytest
+import responses as resp_lib
+
+from eolas_data import Client
+from eolas_data.exceptions import (
+    AuthenticationError,
+    ChangesLicenceRestricted,
+    ChangesUpgradeRequired,
+    WatermarkExpired,
+)
+
+BASE = "https://api.eolas.fyi"
+
+
+@pytest.fixture()
+def client():
+    return Client("eolas_testkey123", base_url=BASE)
+
+
+@resp_lib.activate
+@pytest.mark.parametrize(
+    "status,body,exc_cls,match",
+    [
+        (402, {"detail": "Pro required"}, ChangesUpgradeRequired, "Pro"),
+        (
+            403,
+            {"detail": "licence: OECD prohibits export"},
+            ChangesLicenceRestricted,
+            "licence",
+        ),
+        (403, {"detail": "API key is inactive"}, AuthenticationError, "inactive"),
+        (
+            410,
+            {"error": "watermark_expired", "min_available_seq": 42},
+            WatermarkExpired,
+            "watermark",
+        ),
+    ],
+)
+def test_raw_changes_get_typed_errors(client, status, body, exc_cls, match):
+    resp_lib.add(
+        resp_lib.GET,
+        f"{BASE}/v1/datasets/nz_parcels/changes",
+        json=body,
+        status=status,
+    )
+    with pytest.raises(exc_cls, match=match):
+        client._raw_changes_get("/v1/datasets/nz_parcels/changes", params={"since_seq": 0})

eolas_data-1.3.17/tests/test_get_local.py

@@ -0,0 +1,276 @@
+"""Tests for Client.get_local() — mirror of eolas-r/tests/testthat/test-get-local.R."""
+from __future__ import annotations
+
+import gzip
+import pathlib
+from unittest.mock import patch
+
+import pandas as pd
+import pytest
+
+from eolas_data import Client, SyncResult
+from eolas_data.exceptions import (
+    BulkLicenceRestricted,
+    BulkNotYetAvailable,
+    BulkUpgradeRequired,
+)
+
+BASE = "https://api.eolas.fyi"
+SNAPSHOT_ID = "snap_abc123"
+
+BULK_DATASET_META = {
+    "name": "nz_cpi",
+    "title": "NZ CPI",
+    "source": "Stats NZ",
+    "namespace": "statsnz",
+    "table": "nz_cpi",
+}
+
+GEO_DATASET_META = {
+    "name": "nz_parcels",
+    "title": "NZ Parcels",
+    "source": "LINZ",
+    "namespace": "linz",
+    "table": "nz_parcels",
+    "geometry_type": "MultiPolygon",
+    "bulk_export_class": "geoparquet",
+    "row_count_at_last_refresh": 3_000_000,
+}
+
+
+@pytest.fixture()
+def client():
+    return Client("eolas_testkey123", base_url=BASE)
+
+
+def _sync_downloaded(name, *, path, format, **kwargs) -> SyncResult:
+    return SyncResult(
+        status="downloaded",
+        previous_snapshot_id=None,
+        current_snapshot_id=SNAPSHOT_ID,
+        path=path,
+        bytes_downloaded=1024,
+    )
+
+
+def _sync_unchanged(name, *, path, format, **kwargs) -> SyncResult:
+    return SyncResult(
+        status="unchanged",
+        previous_snapshot_id=SNAPSHOT_ID,
+        current_snapshot_id=SNAPSHOT_ID,
+        path=path,
+        bytes_downloaded=0,
+    )
+
+
+def _write_csv_gz(path: pathlib.Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with gzip.open(path, "wt", encoding="utf-8") as fh:
+        fh.write("date,value\n2023-01-01,1100.5\n")
+
+
+# ---------------------------------------------------------------------------
+# First call / cached read
+# ---------------------------------------------------------------------------
+
+def test_get_local_returns_dataset_csv_gz(client, tmp_path):
+    """First call reads a csv_gz bulk file into a Dataset."""
+
+    def fake_sync(name, *, path, format, **kwargs):
+        _write_csv_gz(path)
+        return _sync_downloaded(name, path=path, format=format)
+
+    with (
+        patch.object(client, "_info_cached", return_value=BULK_DATASET_META),
+        patch.object(client, "sync_bulk", side_effect=fake_sync),
+    ):
+        result = client.get_local(
+            "nz_cpi", format="csv_gz", cache_dir=tmp_path,
+        )
+
+    assert isinstance(result, pd.DataFrame)
+    assert list(result.columns) == ["date", "value"]
+    assert len(result) == 1
+    assert result["value"].iloc[0] == pytest.approx(1100.5)
+
+
+def test_get_local_reads_cached_file_on_subsequent_call(client, tmp_path):
+    """Unchanged sync_bulk → read existing csv.gz without re-downloading."""
+    csv_path = tmp_path / "nz_cpi.csv.gz"
+    with gzip.open(csv_path, "wt", encoding="utf-8") as fh:
+        fh.write("date,value\n2023-06-01,1105.0\n")
+
+    with (
+        patch.object(client, "_info_cached", return_value=BULK_DATASET_META),
+        patch.object(client, "sync_bulk", side_effect=_sync_unchanged),
+    ):
+        result = client.get_local("nz_cpi", format="csv_gz", cache_dir=tmp_path)
+
+    assert isinstance(result, pd.DataFrame)
+    assert len(result) == 1
+    assert result["value"].iloc[0] == pytest.approx(1105.0)
+
+
+def test_get_local_expands_tilde_in_cache_dir(client, tmp_path, monkeypatch):
+    """cache_dir='~/…' is expanded to an absolute path before sync_bulk."""
+    home = tmp_path / "fakehome"
+    home.mkdir()
+    monkeypatch.setenv("HOME", str(home))
+
+    seen_path = {"path": None}
+
+    def fake_sync(name, *, path, format, **kwargs):
+        seen_path["path"] = path
+        _write_csv_gz(path)
+        return _sync_downloaded(name, path=path, format=format)
+
+    with patch.object(client, "sync_bulk", side_effect=fake_sync):
+        client.get_local("nz_cpi", format="csv_gz", cache_dir="~/.cache/eolas", meta=False)
+
+    expected = (home / ".cache" / "eolas" / "nz_cpi.csv.gz").resolve()
+    assert seen_path["path"] == expected
+    assert "~" not in str(seen_path["path"])
+
+
+def test_get_local_explicit_cache_dir_overrides_library(client, tmp_path, monkeypatch):
+    """Explicit cache_dir wins over EOLAS_LIBRARY / config resolution."""
+    import eolas_data.library as lib
+
+    env_dir = tmp_path / "from_env"
+    monkeypatch.setenv("EOLAS_LIBRARY", str(env_dir))
+    explicit = tmp_path / "explicit_cache"
+
+    def fake_sync(name, *, path, format, **kwargs):
+        _write_csv_gz(path)
+        return _sync_downloaded(name, path=path, format=format)
+
+    with patch.object(client, "sync_bulk", side_effect=fake_sync):
+        client.get_local("nz_cpi", format="csv_gz", cache_dir=explicit, meta=False)
+
+    assert (explicit / "nz_cpi.csv.gz").exists()
+    assert not (env_dir / "nz_cpi.csv.gz").exists()
+    # Sanity: library resolver would have picked env_dir
+    assert lib.resolve_library_dir(interactive=False) == env_dir.resolve()
+
+
+# ---------------------------------------------------------------------------
+# Format auto-detection
+# ---------------------------------------------------------------------------
+
+@pytest.mark.parametrize(
+    "meta,expected_fmt",
+    [
+        (GEO_DATASET_META, "geoparquet"),
+        (BULK_DATASET_META, "parquet"),
+        (
+            {
+                "name": "rbnz_b2_wholesale_rates_monthly",
+                "namespace": "rbnz",
+                "table": "rbnz_b2_wholesale_rates_monthly",
+                "geometry_type": "none",
+            },
+            "parquet",
+        ),
+    ],
+)
+def test_get_local_auto_detects_format(client, tmp_path, meta, expected_fmt):
+    """Geo metadata → geoparquet; tabular / geometry_type='none' → parquet."""
+    seen = {"format": None}
+
+    def fake_sync(name, *, path, format, **kwargs):
+        seen["format"] = format
+        raise OSError("skip read — format detection only")
+
+    with (
+        patch.object(client, "_info_cached", return_value=meta),
+        patch.object(client, "sync_bulk", side_effect=fake_sync),
+        pytest.raises(OSError, match="skip read"),
+    ):
+        client.get_local(meta["name"], cache_dir=tmp_path, meta=True)
+
+    assert seen["format"] == expected_fmt
+
+
+# ---------------------------------------------------------------------------
+# Error propagation + validation
+# ---------------------------------------------------------------------------
+
+def test_get_local_as_arrow_and_as_geo_conflict(client, tmp_path):
+    with pytest.raises(ValueError, match="mutually exclusive"):
+        client.get_local("nz_cpi", cache_dir=tmp_path, as_arrow=True, as_geo=True)
+
+
+def test_get_local_unknown_format_raises(client, tmp_path):
+    with pytest.raises(ValueError, match="Unknown format"):
+        client.get_local("nz_cpi", format="xml", cache_dir=tmp_path, meta=False)
+
+
+@pytest.mark.parametrize(
+    "exc_cls,message",
+    [
+        (BulkUpgradeRequired, "Bulk upgrade required"),
+        (BulkLicenceRestricted, "Bulk licence restricted"),
+        (BulkNotYetAvailable, "Bulk not yet available"),
+    ],
+)
+def test_get_local_propagates_bulk_errors(client, tmp_path, exc_cls, message):
+    with (
+        patch.object(client, "_info_cached", return_value=BULK_DATASET_META),
+        patch.object(client, "sync_bulk", side_effect=exc_cls(message)),
+        pytest.raises(exc_cls, match=message.split()[1]),  # partial match
+    ):
+        client.get_local("nz_cpi", cache_dir=tmp_path)
+
+
+def test_get_local_parquet_histogram_fallback_to_csv_gz(client, tmp_path):
+    """OSError mentioning 'histogram' triggers csv_gz re-sync + read."""
+    parquet_path = tmp_path / "nz_cpi.parquet"
+    csv_path = tmp_path / "nz_cpi.csv.gz"
+    calls: list[tuple[str, str]] = []
+
+    def fake_sync(name, *, path, format, **kwargs):
+        calls.append((format, path.name))
+        if format == "csv_gz":
+            _write_csv_gz(path)
+            return _sync_downloaded(name, path=path, format=format)
+        path.touch()
+        return _sync_downloaded(name, path=path, format=format)
+
+    real_read_parquet = pd.read_parquet
+
+    def flaky_read_parquet(path, *args, **kwargs):
+        if path == parquet_path:
+            raise OSError("Repetition level histogram size mismatch")
+        return real_read_parquet(path, *args, **kwargs)
+
+    with (
+        patch.object(client, "_info_cached", return_value=BULK_DATASET_META),
+        patch.object(client, "sync_bulk", side_effect=fake_sync),
+        patch("eolas_data.client.pd.read_parquet", side_effect=flaky_read_parquet),
+    ):
+        result = client.get_local("nz_cpi", cache_dir=tmp_path, meta=True)
+
+    assert isinstance(result, pd.DataFrame)
+    assert calls[0] == ("parquet", "nz_cpi.parquet")
+    assert ("csv_gz", "nz_cpi.csv.gz") in calls
+    assert csv_path.exists()
+
+
+def test_get_local_as_arrow_returns_table(client, tmp_path):
+    """as_arrow=True returns a pyarrow.Table (no pandas materialisation)."""
+    pytest.importorskip("pyarrow")
+    pytest.importorskip("pyarrow.parquet")
+    import pyarrow as pa
+    import pyarrow.parquet as pq
+
+    parquet_path = tmp_path / "nz_cpi.parquet"
+    table = pa.table({"date": ["2023-01-01"], "value": [1100.5]})
+    pq.write_table(table, parquet_path)
+
+    with patch.object(client, "sync_bulk", side_effect=_sync_unchanged):
+        result = client.get_local(
+            "nz_cpi", cache_dir=tmp_path, as_arrow=True, meta=False,
+        )
+
+    assert isinstance(result, pa.Table)
+    assert result.num_rows == 1

eolas_data-1.3.17/tests/test_source_helpers.py

@@ -0,0 +1,36 @@
+"""Parametrised smoke tests for Client source-specific fetch helpers."""
+
+import json
+from pathlib import Path
+
+import pytest
+import responses as resp_lib
+
+from eolas_data import Client
+
+BASE = "https://api.eolas.fyi"
+RECORDS = [
+    {"date": "2023-01-01", "period": "2023Q1", "value": 100.0},
+    {"date": "2023-04-01", "period": "2023Q2", "value": 101.5},
+]
+
+_SOURCE_HELPERS = json.loads(
+    (Path(__file__).parent / "fixtures" / "source_helpers.json").read_text(encoding="utf-8")
+)
+
+
+@pytest.fixture()
+def client():
+    return Client("eolas_testkey123", base_url=BASE)
+
+
+@pytest.mark.parametrize(
+    "method,expected_source",
+    sorted(_SOURCE_HELPERS.items()),
+    ids=sorted(_SOURCE_HELPERS.keys()),
+)
+@resp_lib.activate
+def test_source_helper_sets_source(client, method, expected_source):
+    resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_cpi/data", json={"data": RECORDS})
+    df = getattr(client, method)("nz_cpi")
+    assert df.eolas_source == expected_source