publicdata-au 0.1.0__tar.gz

publicdata_au-0.1.0/.gitignore

@@ -0,0 +1,24 @@
+__pycache__/
+*.egg-info/
+.venv/
+.pytest_cache/
+.ruff_cache/
+dist/
+snapshots/
+raw/
+.wrangler/
+.DS_Store
+.env
+.env.*
+store/*/*/source.*
+dist-large/
+# Lighthouse and Chrome write beside the working directory during local audits.
+lh-*.json
+C:*
+node_modules/
+
+# Written at publish time by python -m publicdata.api_text server.json.
+/server.json
+.build-cache/
+.venv*/
+clients/*/dist/

publicdata_au-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 National Digital
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

publicdata_au-0.1.0/PKG-INFO

@@ -0,0 +1,93 @@
+Metadata-Version: 2.5
+Name: publicdata-au
+Version: 0.1.0
+Summary: Query and download Australian government open data from publicdata.au.
+Project-URL: Homepage, https://publicdata.au/
+Project-URL: Documentation, https://publicdata.au/agents/
+Project-URL: Source, https://github.com/National-Digital/publicdata.au/tree/main/clients/python
+Author: National Digital
+License-Expression: MIT
+License-File: LICENSE
+Keywords: australia,government data,open data,publicdata.au
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Requires-Python: >=3.10
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: pandas
+Requires-Dist: pandas>=2.0; extra == 'pandas'
+Requires-Dist: pyarrow>=14; extra == 'pandas'
+Description-Content-Type: text/markdown
+
+# publicdata-au
+
+Query and download Australian government open data from [publicdata.au](https://publicdata.au/).
+publicdata.au republishes datasets that governments already publish under open licences, keeps
+every version at a URL that never changes, and serves each one in ten formats with a query API.
+
+This package works for every dataset the site serves, named by its slug, so a dataset added to
+the site needs no new release.
+
+```
+pip install publicdata-au            # queries and downloads, no dependencies
+pip install "publicdata-au[pandas]"  # adds read() into a pandas DataFrame
+```
+
+## Find a dataset
+
+```python
+import publicdata_au as pd_au
+
+pd_au.datasets("road crashes")  # slug, title, publisher, licence and page of each match
+pd_au.versions("au-road-deaths")  # every version kept, newest first
+```
+
+## Query rows and totals
+
+```python
+from publicdata_au import gte, in_
+
+deaths = pd_au.rows(
+    "au-road-deaths",
+    {"state": in_("QLD", "NSW"), "year": gte(2020)},
+    select=["state", "year", "road_user"],
+    order="year.desc",
+    all=True,
+)
+deaths.version  # the version the rows came from
+deaths.attribution  # the attribution the publisher's licence requires
+deaths.to_pandas()
+
+pd_au.aggregate("au-road-deaths", group="state", metric="count", where={"year": 2025})
+```
+
+A plain value must match exactly, a list matches any of its values and None matches a blank or
+suppressed cell. The filters are `eq`, `neq`, `gt`, `gte`, `lt`, `lte`, `like`, `ilike`, `in_`,
+`is_null` and `not_`.
+
+Without `version=` an answer comes from the newest version and changes when the publisher
+releases again. Pass a date from `versions()` for an answer that never changes.
+
+## Whole tables
+
+```python
+df = pd_au.read("au-road-deaths")  # needs the [pandas] extra
+df.attrs["publicdata"]  # the version, licence and attribution the file itself carries
+pd_au.download(
+    "au-road-deaths", "csv"
+)  # parquet, csv, csv.gz, json, ndjson, sqlite, xlsx, arrow, geojson, gpkg
+```
+
+Files have no rate limit. The query API allows 60 requests in 10 seconds from one address, and
+this package waits and retries when it answers 429.
+
+## Licence and attribution
+
+The data is under each publisher's own licence, which requires the attribution string that every
+answer carries. Please also name publicdata.au and link to the version you used. publicdata.au is
+an independent republication, and the publishers have not endorsed it.
+
+The package itself is under the MIT licence.

publicdata_au-0.1.0/README.md

@@ -0,0 +1,69 @@
+# publicdata-au
+
+Query and download Australian government open data from [publicdata.au](https://publicdata.au/).
+publicdata.au republishes datasets that governments already publish under open licences, keeps
+every version at a URL that never changes, and serves each one in ten formats with a query API.
+
+This package works for every dataset the site serves, named by its slug, so a dataset added to
+the site needs no new release.
+
+```
+pip install publicdata-au            # queries and downloads, no dependencies
+pip install "publicdata-au[pandas]"  # adds read() into a pandas DataFrame
+```
+
+## Find a dataset
+
+```python
+import publicdata_au as pd_au
+
+pd_au.datasets("road crashes")  # slug, title, publisher, licence and page of each match
+pd_au.versions("au-road-deaths")  # every version kept, newest first
+```
+
+## Query rows and totals
+
+```python
+from publicdata_au import gte, in_
+
+deaths = pd_au.rows(
+    "au-road-deaths",
+    {"state": in_("QLD", "NSW"), "year": gte(2020)},
+    select=["state", "year", "road_user"],
+    order="year.desc",
+    all=True,
+)
+deaths.version  # the version the rows came from
+deaths.attribution  # the attribution the publisher's licence requires
+deaths.to_pandas()
+
+pd_au.aggregate("au-road-deaths", group="state", metric="count", where={"year": 2025})
+```
+
+A plain value must match exactly, a list matches any of its values and None matches a blank or
+suppressed cell. The filters are `eq`, `neq`, `gt`, `gte`, `lt`, `lte`, `like`, `ilike`, `in_`,
+`is_null` and `not_`.
+
+Without `version=` an answer comes from the newest version and changes when the publisher
+releases again. Pass a date from `versions()` for an answer that never changes.
+
+## Whole tables
+
+```python
+df = pd_au.read("au-road-deaths")  # needs the [pandas] extra
+df.attrs["publicdata"]  # the version, licence and attribution the file itself carries
+pd_au.download(
+    "au-road-deaths", "csv"
+)  # parquet, csv, csv.gz, json, ndjson, sqlite, xlsx, arrow, geojson, gpkg
+```
+
+Files have no rate limit. The query API allows 60 requests in 10 seconds from one address, and
+this package waits and retries when it answers 429.
+
+## Licence and attribution
+
+The data is under each publisher's own licence, which requires the attribution string that every
+answer carries. Please also name publicdata.au and link to the version you used. publicdata.au is
+an independent republication, and the publishers have not endorsed it.
+
+The package itself is under the MIT licence.

publicdata_au-0.1.0/pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "publicdata-au"
+version = "0.1.0"
+description = "Query and download Australian government open data from publicdata.au."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "National Digital" }]
+keywords = ["australia", "open data", "government data", "publicdata.au"]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Science/Research",
+  "Programming Language :: Python :: 3",
+  "Topic :: Scientific/Engineering :: Information Analysis",
+]
+dependencies = []
+
+[project.optional-dependencies]
+pandas = ["pandas>=2.0", "pyarrow>=14"]
+dev = ["pytest>=8", "ruff>=0.6"]
+
+[project.urls]
+Homepage = "https://publicdata.au/"
+Documentation = "https://publicdata.au/agents/"
+Source = "https://github.com/National-Digital/publicdata.au/tree/main/clients/python"
+
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/publicdata_au"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP", "W"]
+ignore = ["E501"]

publicdata_au-0.1.0/src/publicdata_au/__init__.py

@@ -0,0 +1,436 @@
+"""Query and download Australian government open data from publicdata.au.
+
+Every function works for every dataset the site serves, named by its slug, so a dataset added to
+the site needs no new release of this package.
+
+    >>> import publicdata_au as pd_au
+    >>> pd_au.datasets("road crashes")
+    >>> pd_au.rows("au-road-deaths", {"state": "QLD", "year": pd_au.gte(2020)}, limit=5)
+    >>> pd_au.aggregate("au-road-deaths", group="state")
+    >>> pd_au.read("au-road-deaths")  # a pandas DataFrame, with the [pandas] extra
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import tempfile
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+__version__ = "0.1.0"
+__all__ = [
+    "Client",
+    "Filter",
+    "PublicDataError",
+    "Rows",
+    "aggregate",
+    "dataset",
+    "datasets",
+    "download",
+    "eq",
+    "gt",
+    "gte",
+    "ilike",
+    "in_",
+    "is_null",
+    "like",
+    "lt",
+    "lte",
+    "neq",
+    "not_",
+    "read",
+    "rows",
+    "versions",
+]
+
+SITE = "https://publicdata.au"
+FORMATS = (
+    "parquet",
+    "csv",
+    "csv.gz",
+    "json",
+    "ndjson",
+    "sqlite",
+    "xlsx",
+    "arrow",
+    "geojson",
+    "gpkg",
+)
+PAGE_MAX = 10_000
+
+
+class PublicDataError(Exception):
+    """An answer from publicdata.au that was not a success."""
+
+    def __init__(self, status: int, message: str, body: Any = None, url: str = ""):
+        super().__init__(f"{status}: {message}" + (f" ({url})" if url else ""))
+        self.status = status
+        self.body = body
+        self.url = url
+
+
+class Filter:
+    """One condition on a field, in the query API's `operator.value` form."""
+
+    def __init__(self, expr: str):
+        self.expr = expr
+
+    def __str__(self) -> str:
+        return self.expr
+
+    def __repr__(self) -> str:
+        return f"Filter({self.expr!r})"
+
+    def __eq__(self, other) -> bool:
+        return isinstance(other, Filter) and other.expr == self.expr
+
+    def __hash__(self) -> int:
+        return hash(self.expr)
+
+
+def _v(value) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    return str(value)
+
+
+def eq(value) -> Filter:
+    return Filter(f"eq.{_v(value)}")
+
+
+def neq(value) -> Filter:
+    return Filter(f"neq.{_v(value)}")
+
+
+def gt(value) -> Filter:
+    return Filter(f"gt.{_v(value)}")
+
+
+def gte(value) -> Filter:
+    return Filter(f"gte.{_v(value)}")
+
+
+def lt(value) -> Filter:
+    return Filter(f"lt.{_v(value)}")
+
+
+def lte(value) -> Filter:
+    return Filter(f"lte.{_v(value)}")
+
+
+def like(pattern: str) -> Filter:
+    """`*` stands for any run of characters. Case-sensitive."""
+    return Filter(f"like.{pattern}")
+
+
+def ilike(pattern: str) -> Filter:
+    """As `like`, ignoring case."""
+    return Filter(f"ilike.{pattern}")
+
+
+def in_(*values) -> Filter:
+    if len(values) == 1 and isinstance(values[0], (list, tuple, set, frozenset)):
+        values = tuple(values[0])
+    if any("," in _v(v) for v in values):
+        raise ValueError("a value in in_() cannot contain a comma")
+    return Filter(f"in.({','.join(_v(v) for v in values)})")
+
+
+def is_null() -> Filter:
+    """Blank in the source, or suppressed by the publisher."""
+    return Filter("is.null")
+
+
+def not_(f: Filter) -> Filter:
+    return Filter(f"not.{f.expr}")
+
+
+def _filter(value) -> str:
+    if isinstance(value, Filter):
+        return value.expr
+    if value is None:
+        return "is.null"
+    if isinstance(value, (list, tuple, set, frozenset)):
+        return in_(tuple(value)).expr
+    return eq(value).expr
+
+
+class Rows(list):
+    """A list of row dicts that also carries where they came from.
+
+    `version`, `attribution`, `cite` and `licence` come from the answer itself, so they always
+    name the version the rows were read from."""
+
+    def __init__(self, rows=(), meta: Mapping | None = None, page: Mapping | None = None):
+        super().__init__(rows)
+        self.meta = dict(meta or {})
+        self.page = dict(page or {})
+
+    @property
+    def version(self) -> str | None:
+        return self.meta.get("version")
+
+    @property
+    def attribution(self) -> str | None:
+        return self.meta.get("attribution")
+
+    @property
+    def cite(self) -> str | None:
+        return self.meta.get("cite")
+
+    @property
+    def licence(self) -> dict | None:
+        return self.meta.get("licence")
+
+    @property
+    def version_page(self) -> str | None:
+        return self.page.get("version_page")
+
+    def to_pandas(self):
+        import pandas as pd
+
+        df = pd.DataFrame(list(self))
+        df.attrs["publicdata"] = self.meta
+        return df
+
+
+class Client:
+    """Talks to one publicdata.au site. The module-level functions use a shared default."""
+
+    def __init__(
+        self,
+        site: str | None = None,
+        *,
+        timeout: float = 60,
+        retries: int = 3,
+        user_agent: str | None = None,
+    ):
+        self.site = (site or os.environ.get("PUBLICDATA_SITE") or SITE).rstrip("/")
+        self.timeout = timeout
+        self.retries = retries
+        self.user_agent = user_agent or f"publicdata-au-python/{__version__}"
+
+    def _open(self, url: str):
+        req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
+        attempt = 0
+        while True:
+            try:
+                return urllib.request.urlopen(req, timeout=self.timeout)
+            except urllib.error.HTTPError as err:
+                # The API allows a burst per address, then answers 429 with how long to wait.
+                if err.code == 429 and attempt < self.retries:
+                    attempt += 1
+                    wait = err.headers.get("Retry-After") or "10"
+                    time.sleep(min(float(wait) if wait.isdigit() else 10.0, 60.0))
+                    continue
+                raw = err.read()
+                try:
+                    body = json.loads(raw)
+                except ValueError:
+                    body = raw.decode("utf-8", "replace")
+                msg = body.get("error") if isinstance(body, dict) else str(body)[:200]
+                raise PublicDataError(err.code, msg or err.reason, body, url) from None
+            except (urllib.error.URLError, TimeoutError) as err:
+                reason = getattr(err, "reason", err)
+                raise PublicDataError(0, f"could not reach the site: {reason}", None, url) from err
+
+    def _json(self, path_or_url: str, params: Mapping | None = None):
+        url = path_or_url if "://" in path_or_url else self.site + path_or_url
+        if params:
+            q = urllib.parse.urlencode(
+                {k: v for k, v in params.items() if v is not None},
+                safe="*(),.:",
+                quote_via=urllib.parse.quote,
+            )
+            if q:
+                url += ("&" if "?" in url else "?") + q
+        with self._open(url) as r:
+            return json.loads(r.read())
+
+    def datasets(self, q: str | None = None) -> list[dict]:
+        """Datasets the site serves, each with slug, title, publisher, licence and page URL.
+        `q` searches titles, summaries, publishers, keywords and field names."""
+        return self._json("/api/v1/datasets", {"q": q} if q else None)["results"]
+
+    def dataset(self, slug: str) -> dict:
+        """The dataset's Frictionless data package: title, licence, attribution, fields and
+        every file of the newest version."""
+        return self._json(f"/d/{_slug(slug)}/datapackage.json")
+
+    def versions(self, slug: str) -> list[dict]:
+        """Every version kept, newest first, each with its date, rows, fields and source hash."""
+        return self._json(f"/d/{_slug(slug)}/versions.json")["versions"]
+
+    def _query(self, slug, kind, where, version, params) -> dict:
+        base = f"/api/v1/datasets/{_slug(slug)}/"
+        if version:
+            base += f"versions/{_date(version)}/"
+        params = dict(params)
+        for field, value in (where or {}).items():
+            params[field] = _filter(value)
+        return self._json(base + kind, params)
+
+    def rows(
+        self,
+        slug: str,
+        where: Mapping[str, Any] | None = None,
+        *,
+        select: str | list[str] | None = None,
+        order: str | list[str] | None = None,
+        limit: int | None = None,
+        offset: int | None = None,
+        version: str | None = None,
+        all: bool = False,
+    ) -> Rows:
+        """Rows of a dataset from the query API.
+
+        `where` maps a field to a value, which must match exactly, or to a filter such as
+        `gte(2020)`, `in_("QLD", "NSW")` or `is_null()`. A list means any of its values and None
+        means blank. Every condition must match.
+
+        Without `version` the answer comes from the newest version and changes when the
+        publisher releases again; with a date from `versions()` it never changes. `all=True`
+        follows every page. For a whole table, `read()` or `download()` is faster and has no
+        rate limit."""
+        if all and limit is None:
+            limit = PAGE_MAX
+        params = {
+            "select": _list(select),
+            "order": _list(order),
+            "limit": limit,
+            "offset": offset,
+        }
+        body = self._query(slug, "rows", where, version, params)
+        out = Rows(body.get("rows", ()), body.get("publicdata"), _page(body))
+        nxt = body.get("next")
+        while all and nxt:
+            body = self._json(nxt)
+            out.extend(body.get("rows", ()))
+            nxt = body.get("next")
+        out.page["next"] = nxt
+        return out
+
+    def aggregate(
+        self,
+        slug: str,
+        group: str | list[str] | None = None,
+        metric: str | list[str] = "count",
+        where: Mapping[str, Any] | None = None,
+        *,
+        version: str | None = None,
+    ) -> Rows:
+        """Counts, sums, averages, minimums and maximums by group, from the query API.
+
+        `metric` is `count`, `sum.<field>`, `avg.<field>`, `min.<field>` or `max.<field>`, or a
+        list of them. `where` works as it does for `rows()`."""
+        params = {"group": _list(group), "metric": _list(metric)}
+        body = self._query(slug, "aggregate", where, version, params)
+        return Rows(body.get("rows", ()), body.get("publicdata"), _page(body))
+
+    def file_url(self, slug: str, format: str = "parquet", version: str | None = None) -> str:
+        if format not in FORMATS:
+            raise ValueError(f"format must be one of {', '.join(FORMATS)}")
+        at = f"v/{_date(version)}" if version else "latest"
+        return f"{self.site}/d/{_slug(slug)}/{at}/data.{format}"
+
+    def _save(self, slug, format, version, path) -> tuple[Path, str]:
+        with self._open(self.file_url(slug, format, version)) as r:
+            final = r.geturl()
+            got = final.split("/v/", 1)[1].split("/", 1)[0] if "/v/" in final else "latest"
+            dest = Path(path) if path else Path(f"{slug}-{got}.{format}")
+            with open(dest, "wb") as f:
+                shutil.copyfileobj(r, f, 1 << 20)
+        return dest, got
+
+    def download(
+        self,
+        slug: str,
+        format: str = "parquet",
+        version: str | None = None,
+        path: str | os.PathLike | None = None,
+    ) -> Path:
+        """Saves one version's file, the newest by default, and returns where. Without `path`
+        the file is named `<slug>-<version>.<format>` in the working directory. Files have no
+        rate limit."""
+        return self._save(slug, format, version, path)[0]
+
+    def read(self, slug: str, version: str | None = None):
+        """The whole table as a pandas DataFrame, read from the version's Parquet file.
+        `df.attrs["publicdata"]` is the provenance header the file itself carries: version,
+        licence, attribution, citation and source."""
+        import pyarrow.parquet as pq
+
+        with tempfile.TemporaryDirectory() as d:
+            p, _ = self._save(slug, "parquet", version, Path(d) / "data.parquet")
+            table = pq.read_table(p)
+        header = (table.schema.metadata or {}).get(b"publicdata")
+        df = table.to_pandas()
+        df.attrs["publicdata"] = json.loads(header) if header else {}
+        return df
+
+
+def _slug(slug: str) -> str:
+    if not slug or not all(c.isalnum() or c == "-" for c in slug):
+        raise ValueError(f"not a dataset slug: {slug!r}")
+    return slug
+
+
+def _date(version: str) -> str:
+    v = str(version)
+    if len(v) != 10 or v[4] != "-" or v[7] != "-" or not (v[:4] + v[5:7] + v[8:]).isdigit():
+        raise ValueError(f"a version is a date such as 2026-08-07, not {version!r}")
+    return v
+
+
+def _list(value) -> str | None:
+    if value is None:
+        return None
+    return value if isinstance(value, str) else ",".join(value)
+
+
+def _page(body: Mapping) -> dict:
+    return {
+        k: body.get(k) for k in ("dataset_page", "version_page", "this_version", "manifest", "next")
+    }
+
+
+_default = Client()
+
+
+def datasets(q: str | None = None) -> list[dict]:
+    return _default.datasets(q)
+
+
+def dataset(slug: str) -> dict:
+    return _default.dataset(slug)
+
+
+def versions(slug: str) -> list[dict]:
+    return _default.versions(slug)
+
+
+def rows(slug: str, where: Mapping[str, Any] | None = None, **kw) -> Rows:
+    return _default.rows(slug, where, **kw)
+
+
+def aggregate(slug: str, group=None, metric="count", where=None, **kw) -> Rows:
+    return _default.aggregate(slug, group, metric, where, **kw)
+
+
+def download(slug: str, format: str = "parquet", version: str | None = None, path=None) -> Path:
+    return _default.download(slug, format, version, path)
+
+
+def read(slug: str, version: str | None = None):
+    return _default.read(slug, version)
+
+
+for _f in (datasets, dataset, versions, rows, aggregate, download, read):
+    _f.__doc__ = getattr(Client, _f.__name__).__doc__

publicdata_au-0.1.0/tests/test_client.py

@@ -0,0 +1,244 @@
+import io
+import json
+import os
+import threading
+import urllib.parse
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+import pytest
+
+import publicdata_au as pd_au
+
+META = {
+    "version": "2026-08-07",
+    "attribution": "Publisher, CC BY 4.0.",
+    "cite": "Cite.",
+    "licence": {"id": "CC-BY-4.0"},
+}
+
+
+class Handler(BaseHTTPRequestHandler):
+    hits: list = []
+    throttle = 0
+
+    def log_message(self, *a):
+        pass
+
+    def send(self, status, body, ctype="application/json", headers=()):
+        raw = body if isinstance(body, bytes) else json.dumps(body).encode()
+        self.send_response(status)
+        self.send_header("Content-Type", ctype)
+        for k, v in headers:
+            self.send_header(k, v)
+        self.send_header("Content-Length", str(len(raw)))
+        self.end_headers()
+        self.wfile.write(raw)
+
+    def do_GET(self):
+        u = urllib.parse.urlsplit(self.path)
+        q = dict(urllib.parse.parse_qsl(u.query))
+        Handler.hits.append((u.path, q, self.headers.get("User-Agent")))
+        base = f"http://{self.headers['Host']}"
+        if Handler.throttle:
+            Handler.throttle -= 1
+            return self.send(429, {"error": "slow down"}, headers=[("Retry-After", "0")])
+        if u.path == "/api/v1/datasets":
+            return self.send(200, {"results": [{"slug": "a", "q": q.get("q")}]})
+        if u.path in ("/api/v1/datasets/a/rows", "/api/v1/datasets/a/versions/2026-08-07/rows"):
+            off = int(q.get("offset", 0))
+            data = [{"n": i} for i in range(5)]
+            lim = int(q.get("limit", 100))
+            page = data[off : off + lim]
+            more = off + lim < len(data)
+            nxt = (
+                f"{base}{u.path}?{urllib.parse.urlencode({**q, 'offset': off + lim})}"
+                if more
+                else None
+            )
+            return self.send(
+                200, {"publicdata": META, "version_page": "vp", "rows": page, "next": nxt}
+            )
+        if u.path == "/api/v1/datasets/a/aggregate":
+            return self.send(
+                200, {"publicdata": META, "rows": [{"g": 1, "count": 2}], "next": None}
+            )
+        if u.path == "/api/v1/datasets/nope/rows":
+            return self.send(404, {"error": "No such dataset in the query API"})
+        if u.path == "/d/a/versions.json":
+            return self.send(200, {"versions": [{"version": "2026-08-07"}]})
+        if u.path == "/d/a/datapackage.json":
+            return self.send(
+                200, {"licenses": [{"name": "CC-BY-4.0"}], "publicdata:attribution": "Publisher."}
+            )
+        if u.path.startswith("/d/a/latest/"):
+            self.send_response(302)
+            self.send_header("Location", u.path.replace("/latest/", "/v/2026-08-07/"))
+            self.send_header("Content-Length", "0")
+            self.end_headers()
+            return
+        if u.path == "/d/a/v/2026-08-07/data.csv":
+            return self.send(200, b"n\n1\n", "text/csv")
+        if u.path == "/d/a/v/2026-08-07/data.parquet":
+            import pyarrow as pa
+            import pyarrow.parquet as pq
+
+            buf = io.BytesIO()
+            header = {
+                **META,
+                "version": "2026-08-07",
+                "not_endorsed": "The publisher has not endorsed this site.",
+            }
+            t = pa.table({"n": [1, 2, 3]}).replace_schema_metadata(
+                {"publicdata": json.dumps(header)}
+            )
+            pq.write_table(t, buf)
+            return self.send(200, buf.getvalue(), "application/vnd.apache.parquet")
+        self.send(404, {"error": "not here"})
+
+
+@pytest.fixture
+def client():
+    Handler.hits = []
+    Handler.throttle = 0
+    srv = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+    t = threading.Thread(target=srv.serve_forever, daemon=True)
+    t.start()
+    yield pd_au.Client(f"http://127.0.0.1:{srv.server_port}", retries=2)
+    srv.shutdown()
+
+
+def test_filters_render_in_the_apis_operator_form():
+    assert str(pd_au.gte(2020)) == "gte.2020"
+    assert str(pd_au.in_("QLD", "NSW")) == "in.(QLD,NSW)"
+    assert str(pd_au.in_(["a", "b"])) == "in.(a,b)"
+    assert str(pd_au.not_(pd_au.eq("x"))) == "not.eq.x"
+    assert str(pd_au.is_null()) == "is.null"
+    assert str(pd_au.ilike("*rider*")) == "ilike.*rider*"
+    assert str(pd_au.eq(True)) == "eq.true"
+    with pytest.raises(ValueError, match="comma"):
+        pd_au.in_("a,b")
+
+
+def test_rows_sends_where_select_and_order(client):
+    r = client.rows(
+        "a",
+        {"state": "QLD", "year": pd_au.gte(2020), "lga": None, "sex": ["F", "M"]},
+        select=["n", "m"],
+        order="n.desc",
+        limit=2,
+    )
+    path, q, ua = Handler.hits[-1]
+    assert path == "/api/v1/datasets/a/rows"
+    assert q == {
+        "state": "eq.QLD",
+        "year": "gte.2020",
+        "lga": "is.null",
+        "sex": "in.(F,M)",
+        "select": "n,m",
+        "order": "n.desc",
+        "limit": "2",
+    }
+    assert ua.startswith("publicdata-au-python/")
+    assert r == [{"n": 0}, {"n": 1}] and r.page["next"]
+
+
+def test_rows_carries_provenance(client):
+    r = client.rows("a")
+    assert (
+        r.version == "2026-08-07" and r.attribution and r.cite and r.licence == {"id": "CC-BY-4.0"}
+    )
+    assert r.version_page == "vp"
+
+
+def test_all_follows_every_page(client):
+    r = client.rows("a", all=True, limit=2)
+    assert [x["n"] for x in r] == [0, 1, 2, 3, 4] and r.page["next"] is None
+    assert len([h for h in Handler.hits if h[0].endswith("/rows")]) == 3
+
+
+def test_a_dated_version_goes_on_the_path(client):
+    client.rows("a", version="2026-08-07")
+    assert Handler.hits[-1][0] == "/api/v1/datasets/a/versions/2026-08-07/rows"
+    with pytest.raises(ValueError, match="date"):
+        client.rows("a", version="latest")
+
+
+def test_aggregate(client):
+    a = client.aggregate("a", group=["g", "h"], metric=["count", "sum.n"], where={"x": 1})
+    assert Handler.hits[-1][1] == {"group": "g,h", "metric": "count,sum.n", "x": "eq.1"}
+    assert a == [{"g": 1, "count": 2}] and a.version == "2026-08-07"
+
+
+def test_429_waits_then_retries(client):
+    Handler.throttle = 2
+    assert client.datasets("crash") == [{"slug": "a", "q": "crash"}]
+
+
+def test_429_past_the_retries_raises(client):
+    Handler.throttle = 5
+    with pytest.raises(pd_au.PublicDataError) as err:
+        client.datasets()
+    assert err.value.status == 429
+
+
+def test_an_error_names_the_status_and_the_apis_message(client):
+    with pytest.raises(pd_au.PublicDataError) as err:
+        client.rows("nope")
+    assert err.value.status == 404 and "No such dataset" in str(err.value)
+    assert err.value.body == {"error": "No such dataset in the query API"}
+
+
+def test_a_bad_slug_never_reaches_the_network(client):
+    with pytest.raises(ValueError):
+        client.rows("../etc")
+    assert Handler.hits == []
+
+
+def test_download_follows_latest_and_names_the_version(client, tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    p = client.download("a", "csv")
+    assert p.name == "a-2026-08-07.csv" and p.read_bytes() == b"n\n1\n"
+    with pytest.raises(ValueError, match="format"):
+        client.download("a", "docx")
+
+
+def test_read_takes_provenance_from_the_file_it_read(client):
+    pytest.importorskip("pandas")
+    df = client.read("a", version="2026-08-07")
+    assert list(df["n"]) == [1, 2, 3]
+    assert df.attrs["publicdata"]["version"] == "2026-08-07"
+    assert df.attrs["publicdata"]["attribution"] == "Publisher, CC BY 4.0."
+    assert "has not endorsed" in df.attrs["publicdata"]["not_endorsed"]
+    assert not any(h[0].endswith("datapackage.json") for h in Handler.hits)
+
+
+def test_an_unreachable_site_raises_the_packages_own_error():
+    import socket
+
+    s = socket.socket()
+    s.bind(("127.0.0.1", 0))
+    port = s.getsockname()[1]
+    s.close()
+    with pytest.raises(pd_au.PublicDataError) as err:
+        pd_au.Client(f"http://127.0.0.1:{port}", timeout=2).datasets()
+    assert err.value.status == 0 and "could not reach" in str(err.value)
+
+
+def test_versions_and_dataset(client):
+    assert client.versions("a") == [{"version": "2026-08-07"}]
+    assert client.dataset("a")["licenses"][0]["name"] == "CC-BY-4.0"
+
+
+def test_site_can_come_from_the_environment(monkeypatch):
+    monkeypatch.setenv("PUBLICDATA_SITE", "http://example.test/")
+    assert pd_au.Client().site == "http://example.test"
+
+
+@pytest.mark.skipif(
+    not os.environ.get("PUBLICDATA_LIVE"), reason="set PUBLICDATA_LIVE=1 to query the live site"
+)
+def test_live_site():
+    slugs = [d["slug"] for d in pd_au.datasets()]
+    assert "au-road-deaths" in slugs
+    r = pd_au.rows("au-road-deaths", {"state": "QLD"}, limit=1)
+    assert len(r) == 1 and r.attribution