tilearray 0.1.0__py3-none-any.whl

tilearray/fetch_presets.py

@@ -0,0 +1,77 @@
+"""Fetch-policy presets for common public tile endpoints."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ._version import __version__
+
+OSM_TILE_TEMPLATE = "https://tile.openstreetmap.org/{z}/{x}/{y}.png"
+
+EA_LIDAR_WCS_URL = (
+    "https://environment.data.gov.uk/spatialdata/"
+    "lidar-composite-digital-terrain-model-dtm-1m/wcs"
+)
+EA_LIDAR_COVERAGE_ID = (
+    "13787b9a-26a4-4775-8523-806d13af58fc__Lidar_Composite_Elevation_DTM_1m"
+)
+
+# Tiny 64 m OSGB bbox used in contract tests / EA bench (4 tiles at 32x32 chunks).
+EA_LIDAR_BENCH_BBOX = (431900.0, 382700.0, 431964.0, 382764.0)
+
+DEFAULT_CONTACT_URL = "https://github.com/MatthewJWhittle/tilearray/issues"
+
+
+def tilearray_user_agent(contact_url: str = DEFAULT_CONTACT_URL) -> str:
+    """Return a descriptive User-Agent string for public tile services."""
+
+    return f"tilearray/{__version__} (+{contact_url})"
+
+
+def osm_fetch_defaults(
+    *,
+    user_agent: str | None = None,
+    contact_url: str = DEFAULT_CONTACT_URL,
+) -> dict[str, Any]:
+    """
+    Polite fetch defaults for OpenStreetMap raster tiles (OSMF policy).
+
+    - Identifiable User-Agent (required by OSM tile usage policy)
+    - Low concurrency (2 in-flight) and 2 req/s per host
+    """
+
+    return {
+        "max_concurrent_requests": 2,
+        "adaptive_concurrency": False,
+        "rate_limit_per_second": 2.0,
+        "fetch_retries": 2,
+        "fetch_timeout": 30.0,
+        "headers": {
+            "User-Agent": user_agent or tilearray_user_agent(contact_url),
+        },
+    }
+
+
+def ea_dsp_fetch_defaults() -> dict[str, Any]:
+    """
+    Fetch defaults for Environment Agency Data Service Platform WCS endpoints.
+
+    Uses in-process AIMD concurrency (starts modest, ramps on success, backs off
+    hard on 403/408/429 / timeouts) with a bounded ceiling. No fixed per-second throttle;
+    ``Retry-After`` and transient 403/408/429/503 are handled by :class:`~tilearray.fetch.TileFetcher`.
+    """
+
+    return {
+        "max_concurrent_requests": 10,
+        "initial_concurrent_requests": 8,
+        "min_concurrent_requests": 4,
+        "multiplicative_decrease": 0.75,
+        "adaptive_concurrency": True,
+        "forbidden_circuit_breaker": True,
+        "forbidden_window_seconds": 5.0,
+        "forbidden_threshold": 6,
+        "forbidden_cooldown_seconds": 15.0,
+        "rate_limit_per_second": None,
+        "fetch_retries": 4,
+        "fetch_timeout": 60.0,
+    }

tilearray/pressure.py

@@ -0,0 +1,129 @@
+"""Shared HTTP response classification for retry and AIMD pressure."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+
+import httpx
+
+_BODY_SAMPLE_BYTES = 4096
+
+
+@dataclass(frozen=True)
+class PressureSignature:
+    """Pluggable match rule: HTTP status plus optional body substrings."""
+
+    name: str
+    status_codes: frozenset[int]
+    body_substrings: tuple[bytes, ...] = ()
+    retryable: bool = True
+    aimd_pressure: bool = True
+    circuit_breaker: bool = False
+
+
+@dataclass(frozen=True)
+class ResponseClassification:
+    """Outcome of classifying one HTTP response."""
+
+    signature: PressureSignature | None = None
+    retryable: bool = False
+    aimd_pressure: bool = False
+    circuit_breaker: bool = False
+
+
+def _gateway_status_codes() -> frozenset[int]:
+    return frozenset({403, 408, 429, 500, 502, 503, 504})
+
+
+BUILTIN_PRESSURE_SIGNATURES: tuple[PressureSignature, ...] = (
+    PressureSignature(
+        name="gateway_forbidden",
+        status_codes=frozenset({403}),
+        circuit_breaker=True,
+    ),
+    PressureSignature(
+        name="gateway_timeout",
+        status_codes=frozenset({408}),
+    ),
+    PressureSignature(
+        name="rate_limit",
+        status_codes=frozenset({429}),
+        circuit_breaker=True,
+    ),
+    PressureSignature(
+        name="upstream_error",
+        status_codes=frozenset({500, 502, 503, 504}),
+    ),
+    PressureSignature(
+        name="ogc_transient_404",
+        status_codes=frozenset({404}),
+        body_substrings=(
+            b"invalidparametervalue",
+            b"subsettingcrs",
+            b"exceptionreport",
+        ),
+    ),
+)
+
+
+@dataclass
+class PressureClassifier:
+    """Match responses against ordered signatures (first match wins)."""
+
+    signatures: tuple[PressureSignature, ...] = field(
+        default_factory=lambda: BUILTIN_PRESSURE_SIGNATURES
+    )
+
+    def register(self, signature: PressureSignature) -> None:
+        """Append a custom signature (checked after existing rules)."""
+
+        self.signatures = (*self.signatures, signature)
+
+    def classify(self, response: httpx.Response) -> ResponseClassification:
+        body = response.content[:_BODY_SAMPLE_BYTES]
+        lowered_body = body.lower() if body else b""
+
+        for signature in self.signatures:
+            if response.status_code not in signature.status_codes:
+                continue
+            if signature.body_substrings and not any(
+                needle in lowered_body for needle in signature.body_substrings
+            ):
+                continue
+            return ResponseClassification(
+                signature=signature,
+                retryable=signature.retryable,
+                aimd_pressure=signature.aimd_pressure,
+                circuit_breaker=signature.circuit_breaker,
+            )
+
+        return ResponseClassification()
+
+
+DEFAULT_PRESSURE_CLASSIFIER = PressureClassifier()
+
+
+def classify_response(
+    response: httpx.Response,
+    classifier: PressureClassifier | None = None,
+) -> ResponseClassification:
+    """Classify ``response`` for retry / AIMD / circuit-breaker handling."""
+
+    return (classifier or DEFAULT_PRESSURE_CLASSIFIER).classify(response)
+
+
+def register_pressure_signature(signature: PressureSignature) -> None:
+    """Register a signature on the process-wide default classifier."""
+
+    DEFAULT_PRESSURE_CLASSIFIER.register(signature)
+
+
+def reset_pressure_classifier(
+    signatures: Sequence[PressureSignature] | None = None,
+) -> None:
+    """Reset the default classifier (primarily for tests)."""
+
+    DEFAULT_PRESSURE_CLASSIFIER.signatures = tuple(
+        signatures if signatures is not None else BUILTIN_PRESSURE_SIGNATURES
+    )

tilearray/service/__init__.py

@@ -0,0 +1,33 @@
+"""Service abstractions and implementations for OGC-style tile services."""
+
+from .base import (
+    BaseService,
+    TileGeometry,
+    detect_service_type,
+    get_service,
+    register_service,
+)
+from .config import ServiceConfig, WCSConfig, WMSConfig, WMTSConfig, XYZConfig
+from .wcs import WCSParser, WCSService
+from .wms import WMSService
+from .wmts import WMTSParser, WMTSService
+from .xyz import XYZService
+
+__all__ = [
+    "BaseService",
+    "TileGeometry",
+    "detect_service_type",
+    "get_service",
+    "register_service",
+    "ServiceConfig",
+    "WCSConfig",
+    "WMSConfig",
+    "WMTSConfig",
+    "XYZConfig",
+    "WCSParser",
+    "WCSService",
+    "WMSService",
+    "WMTSParser",
+    "WMTSService",
+    "XYZService",
+]

tilearray/service/base.py

@@ -0,0 +1,282 @@
+"""Service registry and base abstractions for OGC tile services."""
+
+from __future__ import annotations
+
+import math
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+from typing import Any, Callable, cast
+from urllib.parse import parse_qs, urlparse
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator
+
+from ..types import CRS, BoundingBox, Format, ServiceTypeEnum, TileRequest
+from .requests import compose_tile_request as _compose_tile_request
+from .requests import merge_str_mappings
+
+__all__ = [
+    "TileGeometry",
+    "BaseService",
+    "register_service",
+    "detect_service_type",
+    "get_service",
+]
+
+
+class TileGeometry(BaseModel):
+    """Spatial description of a tile that will be fetched from a service."""
+
+    bbox: BoundingBox = Field(..., description="Geographic extent that the tile covers")
+    width: int = Field(..., gt=0, description="Tile width in output pixels")
+    height: int = Field(..., gt=0, description="Tile height in output pixels")
+    crs: CRS = Field(
+        default=CRS.EPSG_4326, description="CRS in which bounds are expressed"
+    )
+    tile_x: int | None = Field(default=None, description="XYZ tile column index")
+    tile_y: int | None = Field(default=None, description="XYZ tile row index")
+    zoom: int | None = Field(default=None, description="XYZ zoom level")
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    @field_validator("bbox")
+    @classmethod
+    def ensure_bbox_crs(
+        cls, bbox: BoundingBox, info: ValidationInfo
+    ) -> BoundingBox:  # pragma: no cover - simple validation
+        crs = info.data.get("crs")
+        if crs is not None and bbox.crs != crs:
+            raise ValueError("TileGeometry bbox CRS must match the tile CRS")
+        return bbox
+
+
+class BaseService(ABC):
+    """Abstract base class for service-specific implementations."""
+
+    service_type: ServiceTypeEnum
+
+    def __init__(self, base_url: str, **config: object) -> None:
+        self.base_url = base_url.rstrip("/?")
+        self.config = config
+
+    @classmethod
+    def from_url(cls, url: str, **config: object) -> BaseService:
+        """Factory hook for constructing a service from a URL."""
+
+        return cls(url, **config)
+
+    # ------------------------------------------------------------------
+    # Tile generation API
+    # ------------------------------------------------------------------
+    def generate_tile_requests(
+        self,
+        bbox: BoundingBox,
+        chunk_size: tuple[int, int],
+        **options: object,
+    ) -> list[TileRequest]:
+        """Generate concrete tile requests for the provided bounding box."""
+
+        tile_geoms = list(self.plan_tiles(bbox, chunk_size, **options))
+        return [
+            self.build_tile_request(tile_geom, **options) for tile_geom in tile_geoms
+        ]
+
+    def plan_tiles(
+        self,
+        bbox: BoundingBox,
+        chunk_size: tuple[int, int],
+        **options: object,
+    ) -> Iterable[TileGeometry]:
+        """Return the spatial layout of tiles for the requested area."""
+
+        width, height = chunk_size
+        resolution = options.get("resolution")
+
+        if resolution:
+            res_x, res_y = cast(tuple[float, float], resolution)
+            if res_x <= 0 or res_y <= 0:
+                raise ValueError("resolution values must be positive")
+
+            tile_width_units = width * res_x
+            tile_height_units = height * res_y
+            epsilon = min(res_x, res_y) / 10.0
+
+            current_min_y = float(bbox.min_y)
+            while current_min_y < bbox.max_y - epsilon:
+                current_max_y = min(bbox.max_y, current_min_y + tile_height_units)
+                span_y = current_max_y - current_min_y
+                pixel_height = max(1, int(math.ceil(span_y / res_y)))
+
+                current_min_x = float(bbox.min_x)
+                while current_min_x < bbox.max_x - epsilon:
+                    current_max_x = min(bbox.max_x, current_min_x + tile_width_units)
+                    span_x = current_max_x - current_min_x
+                    pixel_width = max(1, int(math.ceil(span_x / res_x)))
+
+                    yield TileGeometry(
+                        bbox=BoundingBox(
+                            min_x=current_min_x,
+                            min_y=current_min_y,
+                            max_x=current_max_x,
+                            max_y=current_max_y,
+                            crs=bbox.crs,
+                        ),
+                        width=pixel_width,
+                        height=pixel_height,
+                        crs=bbox.crs,
+                    )
+
+                    current_min_x = current_max_x
+                current_min_y = current_max_y
+            return
+
+        grid_shape_option = options.get("grid_shape")
+        if grid_shape_option is None:
+            rows, cols = 1, 1
+        elif isinstance(grid_shape_option, tuple):
+            try:
+                grid_tuple_raw = cast(tuple[Any, Any], grid_shape_option)
+                row_raw, col_raw = grid_tuple_raw
+            except ValueError as exc:  # pragma: no cover - guard
+                raise ValueError("grid_shape must be a tuple of two integers") from exc
+
+            rows, cols = int(row_raw), int(col_raw)
+        else:
+            raise ValueError("grid_shape must be a tuple of two integers")
+        if rows <= 0 or cols <= 0:
+            raise ValueError("grid_shape dimensions must be positive integers")
+
+        x_span = float(bbox.max_x - bbox.min_x)
+        y_span = float(bbox.max_y - bbox.min_y)
+
+        step_x = x_span / cols if cols > 0 else x_span
+        step_y = y_span / rows if rows > 0 else y_span
+
+        for row in range(rows):
+            min_y = float(bbox.min_y + row * step_y)
+            max_y = float(
+                bbox.min_y + (row + 1) * step_y if row < rows - 1 else bbox.max_y
+            )
+            for col in range(cols):
+                min_x = float(bbox.min_x + col * step_x)
+                max_x = float(
+                    bbox.min_x + (col + 1) * step_x if col < cols - 1 else bbox.max_x
+                )
+                yield TileGeometry(
+                    bbox=BoundingBox(
+                        min_x=min_x,
+                        min_y=min_y,
+                        max_x=max_x,
+                        max_y=max_y,
+                        crs=bbox.crs,
+                    ),
+                    width=width,
+                    height=height,
+                    crs=bbox.crs,
+                )
+
+    def request_headers(self, **options: object) -> dict[str, str]:
+        """Merge service-level and per-call HTTP headers."""
+
+        return merge_str_mappings(self.config.get("headers"), options.get("headers"))
+
+    def compose_tile_request(
+        self,
+        tile: TileGeometry,
+        *,
+        url: str,
+        params: dict[str, Any] | None = None,
+        output_format: Format | None = None,
+        crs: CRS | None = None,
+        **options: object,
+    ) -> TileRequest:
+        """Build a tile request with config headers/params applied."""
+
+        return _compose_tile_request(
+            config=self.config,
+            options=dict(options),
+            url=url,
+            params=params,
+            bbox=tile.bbox,
+            width=tile.width,
+            height=tile.height,
+            crs=crs or tile.crs,
+            output_format=output_format,
+        )
+
+    @abstractmethod
+    def build_tile_request(
+        self,
+        tile: TileGeometry,
+        **options: object,
+    ) -> TileRequest:
+        """Build the HTTP request description for a tile."""
+
+
+# ----------------------------------------------------------------------
+# Service registry utilities
+# ----------------------------------------------------------------------
+
+_SERVICE_REGISTRY: dict[ServiceTypeEnum, type[BaseService]] = {}
+
+
+def register_service(
+    service_type: ServiceTypeEnum,
+) -> Callable[[type[BaseService]], type[BaseService]]:
+    """Decorator for registering service implementations."""
+
+    def decorator(cls: type[BaseService]) -> type[BaseService]:
+        _SERVICE_REGISTRY[service_type] = cls
+        cls.service_type = service_type
+        return cls
+
+    return decorator
+
+
+def detect_service_type(
+    url: str, fallback: ServiceTypeEnum | None = None
+) -> ServiceTypeEnum:
+    """Infer the service type from the URL or query string."""
+
+    parsed = urlparse(url)
+    lower_path = parsed.path.lower()
+    query = parse_qs(parsed.query.lower())
+
+    if "service" in query:
+        try:
+            return ServiceTypeEnum(query["service"][0].upper())
+        except ValueError:
+            pass
+
+    if "wcs" in lower_path:
+        return ServiceTypeEnum.WCS
+    if "wms" in lower_path:
+        return ServiceTypeEnum.WMS
+    if "wmts" in lower_path:
+        return ServiceTypeEnum.WMTS
+
+    lower_url = url.lower()
+    if all(token in lower_url for token in ("{x}", "{y}", "{z}")):
+        return ServiceTypeEnum.XYZ
+
+    if fallback is not None:
+        return fallback
+
+    raise ValueError(f"Unable to detect service type from URL: {url}")
+
+
+def get_service(
+    url: str,
+    *,
+    service_type: ServiceTypeEnum | None = None,
+    **config: object,
+) -> BaseService:
+    """Instantiate the appropriate service implementation for `url`."""
+
+    detected_type = service_type or detect_service_type(url)
+
+    try:
+        service_cls = _SERVICE_REGISTRY[detected_type]
+    except KeyError as exc:  # pragma: no cover - defensive path
+        raise ValueError(f"No service registered for type {detected_type}") from exc
+
+    return service_cls.from_url(url, **config)