publicdata-au 0.1.0__py3-none-any.whl

publicdata_au/__init__.py

@@ -0,0 +1,436 @@
+"""Query and download Australian government open data from publicdata.au.
+
+Every function works for every dataset the site serves, named by its slug, so a dataset added to
+the site needs no new release of this package.
+
+    >>> import publicdata_au as pd_au
+    >>> pd_au.datasets("road crashes")
+    >>> pd_au.rows("au-road-deaths", {"state": "QLD", "year": pd_au.gte(2020)}, limit=5)
+    >>> pd_au.aggregate("au-road-deaths", group="state")
+    >>> pd_au.read("au-road-deaths")  # a pandas DataFrame, with the [pandas] extra
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import tempfile
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+__version__ = "0.1.0"
+__all__ = [
+    "Client",
+    "Filter",
+    "PublicDataError",
+    "Rows",
+    "aggregate",
+    "dataset",
+    "datasets",
+    "download",
+    "eq",
+    "gt",
+    "gte",
+    "ilike",
+    "in_",
+    "is_null",
+    "like",
+    "lt",
+    "lte",
+    "neq",
+    "not_",
+    "read",
+    "rows",
+    "versions",
+]
+
+SITE = "https://publicdata.au"
+FORMATS = (
+    "parquet",
+    "csv",
+    "csv.gz",
+    "json",
+    "ndjson",
+    "sqlite",
+    "xlsx",
+    "arrow",
+    "geojson",
+    "gpkg",
+)
+PAGE_MAX = 10_000
+
+
+class PublicDataError(Exception):
+    """An answer from publicdata.au that was not a success."""
+
+    def __init__(self, status: int, message: str, body: Any = None, url: str = ""):
+        super().__init__(f"{status}: {message}" + (f" ({url})" if url else ""))
+        self.status = status
+        self.body = body
+        self.url = url
+
+
+class Filter:
+    """One condition on a field, in the query API's `operator.value` form."""
+
+    def __init__(self, expr: str):
+        self.expr = expr
+
+    def __str__(self) -> str:
+        return self.expr
+
+    def __repr__(self) -> str:
+        return f"Filter({self.expr!r})"
+
+    def __eq__(self, other) -> bool:
+        return isinstance(other, Filter) and other.expr == self.expr
+
+    def __hash__(self) -> int:
+        return hash(self.expr)
+
+
+def _v(value) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    return str(value)
+
+
+def eq(value) -> Filter:
+    return Filter(f"eq.{_v(value)}")
+
+
+def neq(value) -> Filter:
+    return Filter(f"neq.{_v(value)}")
+
+
+def gt(value) -> Filter:
+    return Filter(f"gt.{_v(value)}")
+
+
+def gte(value) -> Filter:
+    return Filter(f"gte.{_v(value)}")
+
+
+def lt(value) -> Filter:
+    return Filter(f"lt.{_v(value)}")
+
+
+def lte(value) -> Filter:
+    return Filter(f"lte.{_v(value)}")
+
+
+def like(pattern: str) -> Filter:
+    """`*` stands for any run of characters. Case-sensitive."""
+    return Filter(f"like.{pattern}")
+
+
+def ilike(pattern: str) -> Filter:
+    """As `like`, ignoring case."""
+    return Filter(f"ilike.{pattern}")
+
+
+def in_(*values) -> Filter:
+    if len(values) == 1 and isinstance(values[0], (list, tuple, set, frozenset)):
+        values = tuple(values[0])
+    if any("," in _v(v) for v in values):
+        raise ValueError("a value in in_() cannot contain a comma")
+    return Filter(f"in.({','.join(_v(v) for v in values)})")
+
+
+def is_null() -> Filter:
+    """Blank in the source, or suppressed by the publisher."""
+    return Filter("is.null")
+
+
+def not_(f: Filter) -> Filter:
+    return Filter(f"not.{f.expr}")
+
+
+def _filter(value) -> str:
+    if isinstance(value, Filter):
+        return value.expr
+    if value is None:
+        return "is.null"
+    if isinstance(value, (list, tuple, set, frozenset)):
+        return in_(tuple(value)).expr
+    return eq(value).expr
+
+
+class Rows(list):
+    """A list of row dicts that also carries where they came from.
+
+    `version`, `attribution`, `cite` and `licence` come from the answer itself, so they always
+    name the version the rows were read from."""
+
+    def __init__(self, rows=(), meta: Mapping | None = None, page: Mapping | None = None):
+        super().__init__(rows)
+        self.meta = dict(meta or {})
+        self.page = dict(page or {})
+
+    @property
+    def version(self) -> str | None:
+        return self.meta.get("version")
+
+    @property
+    def attribution(self) -> str | None:
+        return self.meta.get("attribution")
+
+    @property
+    def cite(self) -> str | None:
+        return self.meta.get("cite")
+
+    @property
+    def licence(self) -> dict | None:
+        return self.meta.get("licence")
+
+    @property
+    def version_page(self) -> str | None:
+        return self.page.get("version_page")
+
+    def to_pandas(self):
+        import pandas as pd
+
+        df = pd.DataFrame(list(self))
+        df.attrs["publicdata"] = self.meta
+        return df
+
+
+class Client:
+    """Talks to one publicdata.au site. The module-level functions use a shared default."""
+
+    def __init__(
+        self,
+        site: str | None = None,
+        *,
+        timeout: float = 60,
+        retries: int = 3,
+        user_agent: str | None = None,
+    ):
+        self.site = (site or os.environ.get("PUBLICDATA_SITE") or SITE).rstrip("/")
+        self.timeout = timeout
+        self.retries = retries
+        self.user_agent = user_agent or f"publicdata-au-python/{__version__}"
+
+    def _open(self, url: str):
+        req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
+        attempt = 0
+        while True:
+            try:
+                return urllib.request.urlopen(req, timeout=self.timeout)
+            except urllib.error.HTTPError as err:
+                # The API allows a burst per address, then answers 429 with how long to wait.
+                if err.code == 429 and attempt < self.retries:
+                    attempt += 1
+                    wait = err.headers.get("Retry-After") or "10"
+                    time.sleep(min(float(wait) if wait.isdigit() else 10.0, 60.0))
+                    continue
+                raw = err.read()
+                try:
+                    body = json.loads(raw)
+                except ValueError:
+                    body = raw.decode("utf-8", "replace")
+                msg = body.get("error") if isinstance(body, dict) else str(body)[:200]
+                raise PublicDataError(err.code, msg or err.reason, body, url) from None
+            except (urllib.error.URLError, TimeoutError) as err:
+                reason = getattr(err, "reason", err)
+                raise PublicDataError(0, f"could not reach the site: {reason}", None, url) from err
+
+    def _json(self, path_or_url: str, params: Mapping | None = None):
+        url = path_or_url if "://" in path_or_url else self.site + path_or_url
+        if params:
+            q = urllib.parse.urlencode(
+                {k: v for k, v in params.items() if v is not None},
+                safe="*(),.:",
+                quote_via=urllib.parse.quote,
+            )
+            if q:
+                url += ("&" if "?" in url else "?") + q
+        with self._open(url) as r:
+            return json.loads(r.read())
+
+    def datasets(self, q: str | None = None) -> list[dict]:
+        """Datasets the site serves, each with slug, title, publisher, licence and page URL.
+        `q` searches titles, summaries, publishers, keywords and field names."""
+        return self._json("/api/v1/datasets", {"q": q} if q else None)["results"]
+
+    def dataset(self, slug: str) -> dict:
+        """The dataset's Frictionless data package: title, licence, attribution, fields and
+        every file of the newest version."""
+        return self._json(f"/d/{_slug(slug)}/datapackage.json")
+
+    def versions(self, slug: str) -> list[dict]:
+        """Every version kept, newest first, each with its date, rows, fields and source hash."""
+        return self._json(f"/d/{_slug(slug)}/versions.json")["versions"]
+
+    def _query(self, slug, kind, where, version, params) -> dict:
+        base = f"/api/v1/datasets/{_slug(slug)}/"
+        if version:
+            base += f"versions/{_date(version)}/"
+        params = dict(params)
+        for field, value in (where or {}).items():
+            params[field] = _filter(value)
+        return self._json(base + kind, params)
+
+    def rows(
+        self,
+        slug: str,
+        where: Mapping[str, Any] | None = None,
+        *,
+        select: str | list[str] | None = None,
+        order: str | list[str] | None = None,
+        limit: int | None = None,
+        offset: int | None = None,
+        version: str | None = None,
+        all: bool = False,
+    ) -> Rows:
+        """Rows of a dataset from the query API.
+
+        `where` maps a field to a value, which must match exactly, or to a filter such as
+        `gte(2020)`, `in_("QLD", "NSW")` or `is_null()`. A list means any of its values and None
+        means blank. Every condition must match.
+
+        Without `version` the answer comes from the newest version and changes when the
+        publisher releases again; with a date from `versions()` it never changes. `all=True`
+        follows every page. For a whole table, `read()` or `download()` is faster and has no
+        rate limit."""
+        if all and limit is None:
+            limit = PAGE_MAX
+        params = {
+            "select": _list(select),
+            "order": _list(order),
+            "limit": limit,
+            "offset": offset,
+        }
+        body = self._query(slug, "rows", where, version, params)
+        out = Rows(body.get("rows", ()), body.get("publicdata"), _page(body))
+        nxt = body.get("next")
+        while all and nxt:
+            body = self._json(nxt)
+            out.extend(body.get("rows", ()))
+            nxt = body.get("next")
+        out.page["next"] = nxt
+        return out
+
+    def aggregate(
+        self,
+        slug: str,
+        group: str | list[str] | None = None,
+        metric: str | list[str] = "count",
+        where: Mapping[str, Any] | None = None,
+        *,
+        version: str | None = None,
+    ) -> Rows:
+        """Counts, sums, averages, minimums and maximums by group, from the query API.
+
+        `metric` is `count`, `sum.<field>`, `avg.<field>`, `min.<field>` or `max.<field>`, or a
+        list of them. `where` works as it does for `rows()`."""
+        params = {"group": _list(group), "metric": _list(metric)}
+        body = self._query(slug, "aggregate", where, version, params)
+        return Rows(body.get("rows", ()), body.get("publicdata"), _page(body))
+
+    def file_url(self, slug: str, format: str = "parquet", version: str | None = None) -> str:
+        if format not in FORMATS:
+            raise ValueError(f"format must be one of {', '.join(FORMATS)}")
+        at = f"v/{_date(version)}" if version else "latest"
+        return f"{self.site}/d/{_slug(slug)}/{at}/data.{format}"
+
+    def _save(self, slug, format, version, path) -> tuple[Path, str]:
+        with self._open(self.file_url(slug, format, version)) as r:
+            final = r.geturl()
+            got = final.split("/v/", 1)[1].split("/", 1)[0] if "/v/" in final else "latest"
+            dest = Path(path) if path else Path(f"{slug}-{got}.{format}")
+            with open(dest, "wb") as f:
+                shutil.copyfileobj(r, f, 1 << 20)
+        return dest, got
+
+    def download(
+        self,
+        slug: str,
+        format: str = "parquet",
+        version: str | None = None,
+        path: str | os.PathLike | None = None,
+    ) -> Path:
+        """Saves one version's file, the newest by default, and returns where. Without `path`
+        the file is named `<slug>-<version>.<format>` in the working directory. Files have no
+        rate limit."""
+        return self._save(slug, format, version, path)[0]
+
+    def read(self, slug: str, version: str | None = None):
+        """The whole table as a pandas DataFrame, read from the version's Parquet file.
+        `df.attrs["publicdata"]` is the provenance header the file itself carries: version,
+        licence, attribution, citation and source."""
+        import pyarrow.parquet as pq
+
+        with tempfile.TemporaryDirectory() as d:
+            p, _ = self._save(slug, "parquet", version, Path(d) / "data.parquet")
+            table = pq.read_table(p)
+        header = (table.schema.metadata or {}).get(b"publicdata")
+        df = table.to_pandas()
+        df.attrs["publicdata"] = json.loads(header) if header else {}
+        return df
+
+
+def _slug(slug: str) -> str:
+    if not slug or not all(c.isalnum() or c == "-" for c in slug):
+        raise ValueError(f"not a dataset slug: {slug!r}")
+    return slug
+
+
+def _date(version: str) -> str:
+    v = str(version)
+    if len(v) != 10 or v[4] != "-" or v[7] != "-" or not (v[:4] + v[5:7] + v[8:]).isdigit():
+        raise ValueError(f"a version is a date such as 2026-08-07, not {version!r}")
+    return v
+
+
+def _list(value) -> str | None:
+    if value is None:
+        return None
+    return value if isinstance(value, str) else ",".join(value)
+
+
+def _page(body: Mapping) -> dict:
+    return {
+        k: body.get(k) for k in ("dataset_page", "version_page", "this_version", "manifest", "next")
+    }
+
+
+_default = Client()
+
+
+def datasets(q: str | None = None) -> list[dict]:
+    return _default.datasets(q)
+
+
+def dataset(slug: str) -> dict:
+    return _default.dataset(slug)
+
+
+def versions(slug: str) -> list[dict]:
+    return _default.versions(slug)
+
+
+def rows(slug: str, where: Mapping[str, Any] | None = None, **kw) -> Rows:
+    return _default.rows(slug, where, **kw)
+
+
+def aggregate(slug: str, group=None, metric="count", where=None, **kw) -> Rows:
+    return _default.aggregate(slug, group, metric, where, **kw)
+
+
+def download(slug: str, format: str = "parquet", version: str | None = None, path=None) -> Path:
+    return _default.download(slug, format, version, path)
+
+
+def read(slug: str, version: str | None = None):
+    return _default.read(slug, version)
+
+
+for _f in (datasets, dataset, versions, rows, aggregate, download, read):
+    _f.__doc__ = getattr(Client, _f.__name__).__doc__

publicdata_au-0.1.0.dist-info/METADATA

@@ -0,0 +1,93 @@
+Metadata-Version: 2.5
+Name: publicdata-au
+Version: 0.1.0
+Summary: Query and download Australian government open data from publicdata.au.
+Project-URL: Homepage, https://publicdata.au/
+Project-URL: Documentation, https://publicdata.au/agents/
+Project-URL: Source, https://github.com/National-Digital/publicdata.au/tree/main/clients/python
+Author: National Digital
+License-Expression: MIT
+License-File: LICENSE
+Keywords: australia,government data,open data,publicdata.au
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Requires-Python: >=3.10
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: pandas
+Requires-Dist: pandas>=2.0; extra == 'pandas'
+Requires-Dist: pyarrow>=14; extra == 'pandas'
+Description-Content-Type: text/markdown
+
+# publicdata-au
+
+Query and download Australian government open data from [publicdata.au](https://publicdata.au/).
+publicdata.au republishes datasets that governments already publish under open licences, keeps
+every version at a URL that never changes, and serves each one in ten formats with a query API.
+
+This package works for every dataset the site serves, named by its slug, so a dataset added to
+the site needs no new release.
+
+```
+pip install publicdata-au            # queries and downloads, no dependencies
+pip install "publicdata-au[pandas]"  # adds read() into a pandas DataFrame
+```
+
+## Find a dataset
+
+```python
+import publicdata_au as pd_au
+
+pd_au.datasets("road crashes")  # slug, title, publisher, licence and page of each match
+pd_au.versions("au-road-deaths")  # every version kept, newest first
+```
+
+## Query rows and totals
+
+```python
+from publicdata_au import gte, in_
+
+deaths = pd_au.rows(
+    "au-road-deaths",
+    {"state": in_("QLD", "NSW"), "year": gte(2020)},
+    select=["state", "year", "road_user"],
+    order="year.desc",
+    all=True,
+)
+deaths.version  # the version the rows came from
+deaths.attribution  # the attribution the publisher's licence requires
+deaths.to_pandas()
+
+pd_au.aggregate("au-road-deaths", group="state", metric="count", where={"year": 2025})
+```
+
+A plain value must match exactly, a list matches any of its values and None matches a blank or
+suppressed cell. The filters are `eq`, `neq`, `gt`, `gte`, `lt`, `lte`, `like`, `ilike`, `in_`,
+`is_null` and `not_`.
+
+Without `version=` an answer comes from the newest version and changes when the publisher
+releases again. Pass a date from `versions()` for an answer that never changes.
+
+## Whole tables
+
+```python
+df = pd_au.read("au-road-deaths")  # needs the [pandas] extra
+df.attrs["publicdata"]  # the version, licence and attribution the file itself carries
+pd_au.download(
+    "au-road-deaths", "csv"
+)  # parquet, csv, csv.gz, json, ndjson, sqlite, xlsx, arrow, geojson, gpkg
+```
+
+Files have no rate limit. The query API allows 60 requests in 10 seconds from one address, and
+this package waits and retries when it answers 429.
+
+## Licence and attribution
+
+The data is under each publisher's own licence, which requires the attribution string that every
+answer carries. Please also name publicdata.au and link to the version you used. publicdata.au is
+an independent republication, and the publishers have not endorsed it.
+
+The package itself is under the MIT licence.

publicdata_au-0.1.0.dist-info/RECORD

@@ -0,0 +1,5 @@
+publicdata_au/__init__.py,sha256=RkLwUoPZZu4LU9a7ueZ4QFDIRaLzmuUJjeeKooQ195o,13850
+publicdata_au-0.1.0.dist-info/METADATA,sha256=964WlAFp8Di7stbZV1jNa1jmoHvd_Zb-rR9r8VpdkiY,3433
+publicdata_au-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+publicdata_au-0.1.0.dist-info/licenses/LICENSE,sha256=NTchXD9LNCpXrl0BvkqTZwLU-cPvpxJAH7kfdhrRlwI,1073
+publicdata_au-0.1.0.dist-info/RECORD,,

publicdata_au-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

publicdata_au-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 National Digital
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.