tilefetch 0.1.0__py3-none-any.whl

tilefetch/__init__.py

@@ -0,0 +1,110 @@
+"""tilefetch: download map tiles for an area, politely and correctly.
+
+Fetch XYZ / TMS / WMTS tiles for a bbox or GeoJSON polygon into MBTiles,
+PMTiles v3, a ``z/x/y`` folder, or one georeferenced image (GeoTIFF, or
+PNG/JPEG + world file + ``.prj``).
+
+>>> import tilefetch
+>>> tilefetch.tiles_for_bbox((28.97, 41.00, 28.99, 41.02), 14)[0]
+Tile(z=14, x=9510, y=6141)
+
+Respect each provider's terms of use: see ``tilefetch.list_providers()``.
+"""
+
+from ._version import __version__
+from .api import (
+    Estimate,
+    FetchResult,
+    MosaicResult,
+    estimate,
+    fetch,
+    mosaic,
+    plan_tiles,
+    resolve_source,
+)
+from .download import DEFAULT_USER_AGENT, Downloader, FetchStats
+from .georef import GeoTransform, save_georeferenced, write_geotiff
+from .pmtiles import PMTilesReader, PMTilesWriter, tileid_to_zxy, zxy_to_tileid
+from .providers import PROVIDERS, get_provider, list_providers, register_provider
+from .sources import (
+    RESTRICTED_HOSTS,
+    PolicyError,
+    TileLimitError,
+    TileSource,
+    check_url_policy,
+    parse_wmts_capabilities,
+    wmts_source,
+)
+from .storage import DirectoryStore, MBTilesStore, PMTilesStore, open_reader, open_store
+from .tiles import (
+    BBox,
+    Tile,
+    flip_y,
+    lonlat_to_mercator,
+    lonlat_to_tile,
+    mercator_to_lonlat,
+    parse_bbox,
+    parse_zoom,
+    quadkey_to_tile,
+    resolution,
+    tile_bounds_lonlat,
+    tile_bounds_mercator,
+    tile_count,
+    tile_to_quadkey,
+    tiles_for_bbox,
+    tiles_for_geometry,
+)
+
+__all__ = [
+    "__version__",
+    "fetch",
+    "mosaic",
+    "estimate",
+    "tiles_for_bbox",
+    "tiles_for_geometry",
+    "tile_count",
+    "plan_tiles",
+    "resolve_source",
+    "FetchResult",
+    "MosaicResult",
+    "Estimate",
+    "FetchStats",
+    "Downloader",
+    "DEFAULT_USER_AGENT",
+    "TileSource",
+    "wmts_source",
+    "parse_wmts_capabilities",
+    "PolicyError",
+    "TileLimitError",
+    "RESTRICTED_HOSTS",
+    "check_url_policy",
+    "PROVIDERS",
+    "get_provider",
+    "list_providers",
+    "register_provider",
+    "MBTilesStore",
+    "PMTilesStore",
+    "DirectoryStore",
+    "PMTilesReader",
+    "PMTilesWriter",
+    "open_store",
+    "open_reader",
+    "zxy_to_tileid",
+    "tileid_to_zxy",
+    "GeoTransform",
+    "save_georeferenced",
+    "write_geotiff",
+    "Tile",
+    "BBox",
+    "lonlat_to_tile",
+    "lonlat_to_mercator",
+    "mercator_to_lonlat",
+    "tile_bounds_mercator",
+    "tile_bounds_lonlat",
+    "tile_to_quadkey",
+    "quadkey_to_tile",
+    "flip_y",
+    "resolution",
+    "parse_bbox",
+    "parse_zoom",
+]

tilefetch/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

tilefetch/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

tilefetch/api.py

@@ -0,0 +1,592 @@
+"""High-level API: :func:`fetch`, :func:`mosaic`, :func:`estimate`."""
+
+from __future__ import annotations
+
+import os
+import sys
+from dataclasses import dataclass, field
+from typing import Any, Callable, Mapping
+
+from PIL import Image
+
+from ._version import __version__
+from .download import Downloader, FetchStats, TileResult
+from .georef import GeoTransform, save_georeferenced
+from .mosaic import crop_mercator, stitch, warp_to_lonlat
+from .providers import PROVIDERS
+from .sources import TileLimitError, TileSource
+from .storage import TileStore, open_reader, open_store, store_kind
+from .tiles import (
+    BBox,
+    BBoxLike,
+    Tile,
+    ZoomLike,
+    _normalise_crs,
+    bbox_to_mercator,
+    geometry_bbox,
+    load_geometry,
+    mercator_to_lonlat,
+    parse_bbox,
+    parse_zoom,
+    resolution,
+    tiles_for_bbox,
+    tiles_for_polygons,
+)
+
+__all__ = [
+    "resolve_source",
+    "plan_tiles",
+    "fetch",
+    "mosaic",
+    "estimate",
+    "FetchResult",
+    "MosaicResult",
+    "Estimate",
+]
+
+ProgressFn = Callable[[int, int], None]
+
+#: Typical tile sizes used by :func:`estimate` when no sample is taken.
+TYPICAL_TILE_BYTES = {"png": 30_000, "jpg": 20_000, "webp": 14_000, None: 25_000}
+
+
+def resolve_source(
+    source: Any, *, params: Mapping[str, str] | None = None, **overrides: Any
+) -> TileSource:
+    """Turn a provider name, URL template or :class:`TileSource` into a :class:`TileSource`.
+
+    ``params`` fills or overrides template parameters (e.g. ``{"time": "2025-07-01"}``).
+    ``overrides`` replace other fields (``attribution``, ``scheme``, ...); on
+    built-in providers only ``headers`` and ``params`` may be changed.
+    """
+    if isinstance(source, TileSource):
+        src = source
+    elif isinstance(source, str) and source in PROVIDERS:
+        src = PROVIDERS[source]
+    elif isinstance(source, str) and "{" in source and "://" in source:
+        merged = dict(params or {})
+        params = None
+        src = TileSource.from_url(source, params=merged, **overrides)
+        overrides = {}
+    else:
+        raise ValueError(
+            f"cannot interpret source {source!r}: use a provider name "
+            f"({', '.join(sorted(PROVIDERS))}), a URL template containing {{z}}/{{x}}/{{y}}, "
+            "or a TileSource (see tilefetch.wmts_source for WMTS)"
+        )
+    if params:
+        src = src.replace(params={**src.params, **params})
+    if overrides:
+        if src.builtin:
+            bad = set(overrides) - {"headers"}
+            if bad:
+                raise ValueError(f"built-in provider settings cannot be overridden: {sorted(bad)}")
+        src = src.replace(**overrides)
+    return src
+
+
+def _area(
+    bbox: BBoxLike | None, geometry: Any, zooms: list[int], bbox_crs: str
+) -> tuple[list[Tile], BBox]:
+    """Tiles to fetch and the lon/lat bounds of the request."""
+    if geometry is not None:
+        polys = load_geometry(geometry)
+        return tiles_for_polygons(polys, zooms), geometry_bbox(polys)
+    if bbox is None:
+        raise ValueError("give either bbox or geometry")
+    crs = _normalise_crs(bbox_crs)
+    b = parse_bbox(bbox)
+    tiles = tiles_for_bbox(b, zooms, crs)
+    if crs == "EPSG:3857":
+        w, s = mercator_to_lonlat(b.minx, b.miny)
+        e, n = mercator_to_lonlat(b.maxx, b.maxy)
+        b = BBox(w, s, e, n)
+    return tiles, b
+
+
+def plan_tiles(
+    source: TileSource,
+    bbox: BBoxLike | None,
+    zoom: ZoomLike,
+    *,
+    geometry: Any = None,
+    bbox_crs: str = "EPSG:4326",
+) -> tuple[list[Tile], BBox, list[int]]:
+    """Validate zooms against the source and list the tiles of a request."""
+    zooms = parse_zoom(zoom)
+    bad = [z for z in zooms if z < source.min_zoom or z > source.max_zoom]
+    if bad:
+        raise ValueError(
+            f"zoom {bad} outside the range of {source.name!r} ({source.min_zoom}-{source.max_zoom})"
+        )
+    tiles, bounds = _area(bbox, geometry, zooms, bbox_crs)
+    return tiles, bounds, zooms
+
+
+def _tile_cap(source: TileSource, max_tiles: int | None) -> int | None:
+    if source.builtin and source.max_tiles is not None:
+        return source.max_tiles if max_tiles is None else min(max_tiles, source.max_tiles)
+    return source.max_tiles if max_tiles is None else max_tiles
+
+
+def _check_cap(source: TileSource, n: int, max_tiles: int | None) -> None:
+    cap = _tile_cap(source, max_tiles)
+    if cap is not None and n > cap:
+        hint = (
+            "This limit is part of the provider's entry and cannot be raised; split the job "
+            "or lower the zoom."
+            if source.builtin
+            else "Pass max_tiles=... (CLI: --max-tiles) if you operate the server or its terms allow it."
+        )
+        raise TileLimitError(
+            f"request needs {n} tiles but {source.name!r} allows {cap} per job. {hint}"
+        )
+
+
+def _announce(source: TileSource, quiet: bool) -> None:
+    if quiet:
+        return
+    lines = []
+    if source.attribution:
+        lines.append(f"Attribution: {source.attribution}")
+    if source.terms_url:
+        lines.append(f"Terms of use: {source.terms_url}")
+    if not source.builtin and not source.attribution:
+        lines.append(
+            "Note: no attribution given for this custom source; check the provider's terms and "
+            "credit it in your outputs."
+        )
+    for line in lines:
+        print(f"[tilefetch] {line}", file=sys.stderr)
+
+
+def _metadata(
+    source: TileSource, bounds: BBox, zooms: list[int], fmt: str | None, name: str | None
+) -> dict[str, Any]:
+    w, s, e, n = bounds
+    w, e = max(-180.0, w), min(180.0, e)
+    s, n = max(-85.0511287798, s), min(85.0511287798, n)
+    if w > e:  # antimeridian: MBTiles bounds cannot express it
+        w, e = -180.0, 180.0
+    cz = zooms[0]
+    return {
+        "name": name or source.name,
+        "format": fmt or source.format or "png",
+        "bounds": [round(w, 7), round(s, 7), round(e, 7), round(n, 7)],
+        "center": [round((w + e) / 2, 7), round((s + n) / 2, 7), cz],
+        "minzoom": zooms[0],
+        "maxzoom": zooms[-1],
+        "attribution": source.attribution,
+        "description": source.description or f"Tiles from {source.name}",
+        "type": "baselayer",
+        "version": "1",
+        "generator": f"tilefetch {__version__}",
+        **({"terms_url": source.terms_url} if source.terms_url else {}),
+    }
+
+
+@dataclass
+class FetchResult:
+    """Outcome of :func:`fetch`."""
+
+    path: str
+    stats: FetchStats
+    tiles: int
+    zooms: list[int]
+    bounds: BBox
+    attribution: str
+    terms_url: str | None
+    files: list[str] = field(default_factory=list)
+
+    @property
+    def ok(self) -> bool:
+        return self.stats.failed == 0
+
+
+def _run_download(
+    dl: Downloader,
+    tiles: list[Tile],
+    store: TileStore | None,
+    resume: bool,
+    progress: ProgressFn | None,
+    sink: Callable[[TileResult], None] | None = None,
+) -> FetchStats:
+    stats = FetchStats(total=len(tiles))
+    todo = tiles
+    if store is not None and resume:
+        todo = [t for t in tiles if not store.has(t)]
+        stats.skipped = len(tiles) - len(todo)
+    done = stats.skipped
+    if progress:
+        progress(done, stats.total)
+    for r in dl.iter_fetch(todo):
+        stats.add(r)
+        if r.data is not None:
+            if store is not None:
+                store.put(r.tile, r.data)
+            if sink is not None:
+                sink(r)
+        done += 1
+        if progress:
+            progress(done, stats.total)
+    return stats
+
+
+def fetch(
+    source: Any,
+    bbox: BBoxLike | None = None,
+    zoom: ZoomLike = 12,
+    out: str | os.PathLike[str] = "tiles.mbtiles",
+    *,
+    geometry: Any = None,
+    bbox_crs: str = "EPSG:4326",
+    threads: int = 2,
+    rate: float | None = None,
+    max_tiles: int | None = None,
+    retries: int = 4,
+    backoff: float = 0.5,
+    timeout: float = 30.0,
+    user_agent: str | None = None,
+    headers: Mapping[str, str] | None = None,
+    params: Mapping[str, str] | None = None,
+    cache_dir: str | os.PathLike[str] | None = None,
+    revalidate: bool = False,
+    resume: bool = True,
+    overwrite: bool = False,
+    name: str | None = None,
+    quiet: bool = False,
+    progress: ProgressFn | None = None,
+    **mosaic_kwargs: Any,
+) -> FetchResult:
+    """Download the tiles of an area into ``out``.
+
+    The output type follows the extension: ``.mbtiles``, ``.pmtiles``, an
+    image (``.tif``, ``.png``, ``.jpg``, ``.webp``: a georeferenced mosaic of
+    the highest zoom, see :func:`mosaic`), or anything else for a ``z/x/y``
+    folder. Existing tiles in the output are kept and skipped (``resume``)
+    unless ``overwrite=True``.
+
+    ``source`` is a provider name (see :func:`tilefetch.list_providers`), a
+    URL template such as ``"https://tiles.example.org/{z}/{x}/{y}.png"``, or a
+    :class:`TileSource`. The area is ``bbox`` (``west, south, east, north`` in
+    degrees, or EPSG:3857 with ``bbox_crs``) or a GeoJSON ``geometry``.
+    """
+    src = resolve_source(source, params=params)
+    out_s = os.fspath(out)
+    if store_kind(out_s) == "image":
+        res = mosaic(
+            src,
+            bbox,
+            max(parse_zoom(zoom)),
+            out=out_s,
+            geometry=geometry,
+            bbox_crs=bbox_crs,
+            threads=threads,
+            rate=rate,
+            max_tiles=max_tiles,
+            retries=retries,
+            backoff=backoff,
+            timeout=timeout,
+            user_agent=user_agent,
+            headers=headers,
+            cache_dir=cache_dir,
+            revalidate=revalidate,
+            quiet=quiet,
+            progress=progress,
+            **mosaic_kwargs,
+        )
+        assert res.stats is not None
+        return FetchResult(
+            out_s,
+            res.stats,
+            res.stats.total,
+            [res.zoom],
+            res.request_bounds,
+            src.attribution,
+            src.terms_url,
+            res.files,
+        )
+    if mosaic_kwargs:
+        raise TypeError(f"unexpected arguments for a tile output: {sorted(mosaic_kwargs)}")
+    tiles, bounds, zooms = plan_tiles(src, bbox, zoom, geometry=geometry, bbox_crs=bbox_crs)
+    _check_cap(src, len(tiles), max_tiles)
+    dl = Downloader(
+        src,
+        threads=threads,
+        rate=rate,
+        retries=retries,
+        backoff=backoff,
+        timeout=timeout,
+        user_agent=user_agent,
+        headers=headers,
+        cache_dir=cache_dir,
+        revalidate=revalidate,
+    )
+    _announce(src, quiet)
+    store = open_store(out_s, "w" if overwrite else "a")
+    fmt: list[str | None] = [None]
+
+    def sniff(r: TileResult) -> None:
+        if fmt[0] is None and r.data:
+            from .storage import image_format
+
+            fmt[0] = image_format(r.data)
+
+    try:
+        stats = _run_download(dl, tiles, store, resume and not overwrite, progress, sniff)
+        if fmt[0] is None:
+            first = next((store.get(t) for t in tiles if store.has(t)), None)
+            if first:
+                from .storage import image_format
+
+                fmt[0] = image_format(first)
+        store.set_metadata(_metadata(src, bounds, zooms, fmt[0], name))
+    except BaseException:
+        abort = getattr(store, "abort", None)
+        if abort is not None:
+            abort()
+        else:
+            store.close()
+        raise
+    store.close()
+    return FetchResult(
+        out_s, stats, len(tiles), zooms, bounds, src.attribution, src.terms_url, [out_s]
+    )
+
+
+@dataclass
+class MosaicResult:
+    """Outcome of :func:`mosaic`."""
+
+    image: Image.Image
+    bounds: BBox
+    crs: str
+    zoom: int
+    request_bounds: BBox
+    missing: int
+    attribution: str
+    path: str | None = None
+    files: list[str] = field(default_factory=list)
+    stats: FetchStats | None = None
+
+    @property
+    def transform(self) -> GeoTransform:
+        return GeoTransform.from_bounds(self.bounds, self.image.width, self.image.height)
+
+
+def mosaic(
+    source: Any,
+    bbox: BBoxLike | None = None,
+    zoom: int = 12,
+    out: str | os.PathLike[str] | None = None,
+    *,
+    geometry: Any = None,
+    bbox_crs: str = "EPSG:4326",
+    crs: str = "EPSG:3857",
+    crop: bool | str = True,
+    resampling: str = "bilinear",
+    width: int | None = None,
+    engine: str = "pillow",
+    threads: int = 2,
+    rate: float | None = None,
+    max_tiles: int | None = None,
+    retries: int = 4,
+    backoff: float = 0.5,
+    timeout: float = 30.0,
+    user_agent: str | None = None,
+    headers: Mapping[str, str] | None = None,
+    params: Mapping[str, str] | None = None,
+    cache_dir: str | os.PathLike[str] | None = None,
+    revalidate: bool = False,
+    quiet: bool = False,
+    progress: ProgressFn | None = None,
+) -> MosaicResult:
+    """Build one georeferenced image of an area at one zoom level.
+
+    ``source`` is anything :func:`fetch` accepts, or an existing ``.mbtiles``,
+    ``.pmtiles`` or ``z/x/y`` folder (no network then).
+
+    ``crop``: ``True`` resamples to exactly the requested bbox, ``"snap"``
+    crops to the enclosing source pixels without resampling, ``False`` keeps
+    whole tiles. ``crs="EPSG:4326"`` resamples to a lon/lat grid (``width``
+    sets its pixel width). ``out`` (``.tif``, ``.png``, ``.jpg``, ``.webp``)
+    is optional; PNG/JPEG/WebP get a world file and ``.prj``, ``.tif`` is a
+    GeoTIFF (``engine="pillow"`` or ``"rasterio"``).
+    """
+    if not isinstance(zoom, int):
+        zs = parse_zoom(zoom)
+        if len(zs) != 1:
+            raise ValueError("mosaic() needs a single zoom level")
+        zoom = zs[0]
+    out_crs = _normalise_crs(crs)
+    local = isinstance(source, (str, os.PathLike)) and not (
+        isinstance(source, str) and (source in PROVIDERS or "{" in source)
+    )
+    stats: FetchStats | None = None
+    if local:
+        reader = open_reader(source)
+        try:
+            meta = getattr(reader, "metadata", {})
+            attribution = str(meta.get("attribution", "")) if isinstance(meta, dict) else ""
+            tiles, bounds = _area(bbox, geometry, [zoom], bbox_crs)
+            cache: dict[Tile, bytes] = {}
+            for t in tiles:
+                d = reader.get(t)
+                if d is not None:
+                    cache[t] = d
+        finally:
+            reader.close()
+        tile_size = 256
+    else:
+        src = resolve_source(source, params=params)
+        tiles, bounds, _ = plan_tiles(src, bbox, zoom, geometry=geometry, bbox_crs=bbox_crs)
+        _check_cap(src, len(tiles), max_tiles)
+        dl = Downloader(
+            src,
+            threads=threads,
+            rate=rate,
+            retries=retries,
+            backoff=backoff,
+            timeout=timeout,
+            user_agent=user_agent,
+            headers=headers,
+            cache_dir=cache_dir,
+            revalidate=revalidate,
+        )
+        _announce(src, quiet)
+        cache = {}
+
+        def keep(r: TileResult) -> None:
+            assert r.data is not None
+            cache[r.tile] = r.data
+
+        stats = _run_download(dl, tiles, None, False, progress, keep)
+        attribution = src.attribution
+        tile_size = src.tile_size
+    if bounds.minx > bounds.maxx:
+        raise ValueError("mosaics across the antimeridian are not supported; split the bbox")
+    image, grid, missing = stitch(tiles, cache.get, tile_size)
+    if geometry is not None or _normalise_crs(bbox_crs) == "EPSG:4326":
+        target_m = bbox_to_mercator(bounds)
+    else:
+        target_m = parse_bbox(bbox) if bbox is not None else bbox_to_mercator(bounds)
+    if out_crs == "EPSG:4326":
+        image, out_bounds = warp_to_lonlat(image, grid, bounds, resampling=resampling, width=width)
+    elif crop is False:
+        out_bounds = grid
+    else:
+        image, out_bounds = crop_mercator(
+            image, grid, target_m, resampling=resampling, snap=crop == "snap"
+        )
+    result = MosaicResult(
+        image, out_bounds, out_crs, zoom, bounds, missing, attribution, stats=stats
+    )
+    if out is not None:
+        result.path = os.fspath(out)
+        result.files = save_georeferenced(
+            result.path, image, result.transform, out_crs, attribution=attribution, engine=engine
+        )
+    return result
+
+
+@dataclass
+class Estimate:
+    """Outcome of :func:`estimate`."""
+
+    source: str
+    tiles_per_zoom: dict[int, int]
+    total_tiles: int
+    bytes_per_tile: int
+    estimated_bytes: int
+    max_tiles: int | None
+    within_limit: bool
+    rate: float | None
+    estimated_seconds: float | None
+    mosaic_size: dict[int, tuple[int, int]]
+    resolution_m: dict[int, float]
+    sampled: int = 0
+
+    def as_dict(self) -> dict[str, Any]:
+        from dataclasses import asdict
+
+        return asdict(self)
+
+
+def estimate(
+    source: Any,
+    bbox: BBoxLike | None = None,
+    zoom: ZoomLike = 12,
+    *,
+    geometry: Any = None,
+    bbox_crs: str = "EPSG:4326",
+    max_tiles: int | None = None,
+    rate: float | None = None,
+    threads: int = 2,
+    params: Mapping[str, str] | None = None,
+    sample: int = 0,
+    timeout: float = 30.0,
+    user_agent: str | None = None,
+    headers: Mapping[str, str] | None = None,
+) -> Estimate:
+    """Count tiles and estimate download size and time, without downloading.
+
+    With ``sample=N`` up to N tiles of the highest zoom are downloaded to
+    measure the real average tile size; otherwise a typical size for the
+    source's format is assumed.
+    """
+    src = resolve_source(source, params=params)
+    tiles, bounds, zooms = plan_tiles(src, bbox, zoom, geometry=geometry, bbox_crs=bbox_crs)
+    per_zoom: dict[int, int] = dict.fromkeys(zooms, 0)
+    for t in tiles:
+        per_zoom[t.z] += 1
+    bpt = TYPICAL_TILE_BYTES.get(src.format, TYPICAL_TILE_BYTES[None])
+    sampled = 0
+    if sample > 0 and tiles:
+        top = [t for t in tiles if t.z == zooms[-1]]
+        step = max(1, len(top) // sample)
+        picks = top[::step][:sample]
+        dl = Downloader(
+            src,
+            threads=threads,
+            rate=rate,
+            timeout=timeout,
+            retries=1,
+            user_agent=user_agent,
+            headers=headers,
+        )
+        sizes = [len(r.data) for r in dl.iter_fetch(picks) if r.data]
+        if sizes:
+            sampled = len(sizes)
+            bpt = int(sum(sizes) / len(sizes))
+    cap = _tile_cap(src, max_tiles)
+    eff_rate = rate if rate is not None else src.max_rate
+    if src.builtin and src.max_rate and rate is not None:
+        eff_rate = min(rate, src.max_rate)
+    sizes_px: dict[int, tuple[int, int]] = {}
+    res: dict[int, float] = {}
+    if bounds.minx <= bounds.maxx:
+        mb = bbox_to_mercator(bounds)
+        for z in zooms:
+            r = resolution(z, src.tile_size)
+            res[z] = r
+            sizes_px[z] = (
+                max(1, round((mb.maxx - mb.minx) / r)),
+                max(1, round((mb.maxy - mb.miny) / r)),
+            )
+    else:
+        for z in zooms:
+            res[z] = resolution(z, src.tile_size)
+    return Estimate(
+        source=src.name,
+        tiles_per_zoom=per_zoom,
+        total_tiles=len(tiles),
+        bytes_per_tile=bpt,
+        estimated_bytes=bpt * len(tiles),
+        max_tiles=cap,
+        within_limit=cap is None or len(tiles) <= cap,
+        rate=eff_rate,
+        estimated_seconds=(len(tiles) / eff_rate) if eff_rate else None,
+        mosaic_size=sizes_px,
+        resolution_m=res,
+        sampled=sampled,
+    )