oex 0.2.0__py3-none-any.whl

oex/__init__.py

@@ -0,0 +1,32 @@
+"""oex: country-scale OSM and Overture vector exports."""
+
+from oex.config.schema import (
+    BoundaryConfig,
+    CategoryConfig,
+    DuckdbConfig,
+    HdxConfig,
+    LoggingConfig,
+    OsmSourceConfig,
+    OutputConfig,
+    OvertureSourceConfig,
+    ParallelConfig,
+    RootConfig,
+)
+from oex.exporter import Exporter, ExportResult
+
+__version__ = "0.2.0"
+__all__ = [
+    "BoundaryConfig",
+    "CategoryConfig",
+    "DuckdbConfig",
+    "ExportResult",
+    "Exporter",
+    "HdxConfig",
+    "LoggingConfig",
+    "OsmSourceConfig",
+    "OutputConfig",
+    "OvertureSourceConfig",
+    "ParallelConfig",
+    "RootConfig",
+    "__version__",
+]

oex/boundary.py

@@ -0,0 +1,120 @@
+"""Country boundary resolution: user-supplied geom or geoBoundaries ADM0."""
+
+import json
+import threading
+from dataclasses import dataclass
+from typing import Any
+
+import requests
+
+from oex.config.schema import BoundaryConfig
+from oex.logging_setup import get_logger
+
+logger = get_logger(__name__)
+
+_GEOBOUNDARIES_TPL = "https://www.geoboundaries.org/api/current/gbOpen/{iso3}/{level}/"
+
+
+@dataclass(frozen=True)
+class Boundary:
+    iso3: str
+    bbox: tuple[float, float, float, float]
+    geojson: str
+    source: str
+
+
+_lock = threading.Lock()
+_cache: dict[tuple[str, str, str], Boundary] = {}
+
+
+def _bbox_from_geometry(geometry: dict[str, Any]) -> tuple[float, float, float, float]:
+    coords: list[float] = []
+
+    def walk(node: Any) -> None:
+        if (
+            isinstance(node, list)
+            and len(node) == 2
+            and all(isinstance(v, (int, float)) for v in node)
+        ):
+            coords.extend(node)
+        elif isinstance(node, list):
+            for item in node:
+                walk(item)
+
+    walk(geometry.get("coordinates", []))
+    if not coords:
+        raise ValueError("No coordinates found in geometry")
+    xs = coords[0::2]
+    ys = coords[1::2]
+    return (min(xs), min(ys), max(xs), max(ys))
+
+
+def _featurecollection_to_geometry(fc: dict[str, Any]) -> dict[str, Any]:
+    # ST_GeomFromGeoJSON accepts a single geometry or a GeometryCollection,
+    # not a FeatureCollection.
+    features = fc.get("features", [])
+    geometries = [f["geometry"] for f in features if f.get("geometry")]
+    if len(geometries) == 1:
+        return geometries[0]
+    return {"type": "GeometryCollection", "geometries": geometries}
+
+
+def _fetch_geoboundaries(iso3: str, release: str, level: str) -> Boundary:
+    url = _GEOBOUNDARIES_TPL.format(iso3=iso3.upper(), level=level)
+    logger.info("Fetching boundary metadata: %s", url)
+    meta = requests.get(url, timeout=60)
+    meta.raise_for_status()
+    payload = meta.json()
+    geojson_url = payload.get("gjDownloadURL") or payload.get("simplifiedGeometryGeoJSON")
+    if not geojson_url:
+        raise RuntimeError(f"geoBoundaries response missing GeoJSON URL for {iso3}")
+
+    logger.info("Downloading boundary geometry: %s", geojson_url)
+    resp = requests.get(geojson_url, timeout=180)
+    resp.raise_for_status()
+    fc = resp.json()
+    geometry = _featurecollection_to_geometry(fc) if fc.get("type") == "FeatureCollection" else fc
+    bbox = _bbox_from_geometry(geometry)
+    return Boundary(
+        iso3=iso3.upper(),
+        bbox=bbox,
+        geojson=json.dumps(geometry),
+        source=f"geoBoundaries {release} {level}",
+    )
+
+
+def _from_user_geom(iso3: str, geom_str: str) -> Boundary:
+    fc = json.loads(geom_str)
+    geometry = _featurecollection_to_geometry(fc) if fc.get("type") == "FeatureCollection" else fc
+    bbox = _bbox_from_geometry(geometry)
+    return Boundary(
+        iso3=iso3.upper(),
+        bbox=bbox,
+        geojson=json.dumps(geometry),
+        source="user-provided",
+    )
+
+
+def resolve_boundary(iso3: str, cfg: BoundaryConfig) -> Boundary:
+    if not iso3:
+        raise ValueError("iso3 must be set on the config")
+    key = (iso3.upper(), cfg.geoboundaries_release, cfg.geoboundaries_level)
+    with _lock:
+        cached = _cache.get(key)
+    if cached is not None:
+        return cached
+
+    if cfg.geom:
+        boundary = _from_user_geom(iso3, cfg.geom)
+    else:
+        boundary = _fetch_geoboundaries(iso3, cfg.geoboundaries_release, cfg.geoboundaries_level)
+
+    with _lock:
+        _cache[key] = boundary
+    logger.info(
+        "Resolved boundary for %s from %s; bbox=%s",
+        boundary.iso3,
+        boundary.source,
+        boundary.bbox,
+    )
+    return boundary

oex/cli.py

@@ -0,0 +1,207 @@
+"""Typer CLI for oex."""
+
+from pathlib import Path
+
+import typer
+
+from oex.config.loader import (
+    apply_overrides,
+    iter_configs,
+    load_config,
+    select_categories,
+)
+from oex.config.schema import RootConfig
+from oex.exporter import Exporter, ExportResult
+from oex.logging_setup import get_logger, setup_logging
+from oex.osm.runner import OsmRunner
+from oex.overture.runner import OvertureRunner
+
+app = typer.Typer(
+    add_completion=False,
+    no_args_is_help=True,
+    help="Country-scale OSM and Overture vector exports.",
+)
+
+
+@app.callback()
+def _global(
+    log_level: str = typer.Option("INFO", envvar="LOG_LEVEL"),
+) -> None:
+    setup_logging(level=log_level)
+
+
+def _resolve_config(
+    iso3_or_yaml: str | None,
+    configs_dir: Path | None,
+    config: Path | None,
+) -> list[Path | None]:
+    if configs_dir is not None:
+        return list(iter_configs(configs_dir))
+    if config is not None:
+        return [config]
+    if iso3_or_yaml:
+        candidate = Path("configs") / f"{iso3_or_yaml.lower()}.yaml"
+        if candidate.exists():
+            return [candidate]
+    return [None]
+
+
+def _build_overrides(
+    iso3_or_yaml: str | None,
+    hdx_push: bool | None,
+    output_dir: Path | None,
+    osm_engine: str | None = None,
+) -> dict[str, object]:
+    overrides: dict[str, object] = {}
+    if iso3_or_yaml and len(iso3_or_yaml) <= 3 and iso3_or_yaml.isalpha():
+        overrides["iso3"] = iso3_or_yaml.upper()
+    if hdx_push is True:
+        overrides["hdx.push"] = True
+    if hdx_push is False:
+        overrides["hdx.push"] = False
+    if output_dir is not None:
+        overrides["output.dir"] = str(output_dir)
+    if osm_engine is not None:
+        overrides["source.osm.engine"] = osm_engine
+    return overrides
+
+
+def _summarise(results: list[ExportResult]) -> int:
+    log = get_logger("oex.cli")
+    total_fail = sum(r.failed for r in results)
+    for r in results:
+        log.info(
+            "%s/%s: %d ok, %d empty, %d skipped, %d failed in %.1fs",
+            r.iso3,
+            r.source_name,
+            r.succeeded,
+            r.empty,
+            r.skipped,
+            r.failed,
+            r.total_duration_s,
+        )
+    return 0 if total_fail == 0 else 1
+
+
+def _run_one(
+    yaml_path: Path | None,
+    overrides: dict[str, object],
+    theme: str | None,
+    runner_factory,
+) -> ExportResult:
+    cfg: RootConfig = load_config(yaml_path)
+    cfg = apply_overrides(cfg, overrides)
+    cfg = select_categories(cfg, theme)
+    return Exporter(cfg, runner_factory()).run()
+
+
+def _resolve_args(
+    arg1: str | None,
+    arg2: str | None,
+    configs_dir: Path | None,
+    config: Path | None,
+) -> tuple[str | None, str | None]:
+    # When --config or --configs-dir is given, the first positional is the theme,
+    # not the iso3 (iso3 comes from the YAML).
+    if configs_dir is not None or config is not None:
+        return None, arg1 if arg2 is None else arg2
+    return arg1, arg2
+
+
+@app.command("overture")
+def cmd_overture(
+    iso3_or_yaml: str | None = typer.Argument(
+        None, help="ISO3 like NPL, or name of a YAML in ./configs/"
+    ),
+    theme: str | None = typer.Argument(None, help="Optional theme override (e.g. buildings)"),
+    configs_dir: Path | None = typer.Option(
+        None, "--configs-dir", help="Run every YAML in this directory"
+    ),
+    config: Path | None = typer.Option(None, "--config", "-c", help="Explicit config YAML path"),
+    output_dir: Path | None = typer.Option(None, "--output-dir", "-o"),
+    hdx_push: bool | None = typer.Option(None, "--hdx-push/--no-hdx-push"),
+) -> None:
+    """Export Overture data."""
+    iso3_resolved, theme_resolved = _resolve_args(iso3_or_yaml, theme, configs_dir, config)
+    yamls = _resolve_config(iso3_resolved, configs_dir, config)
+    overrides = _build_overrides(iso3_resolved, hdx_push, output_dir)
+    results = [_run_one(y, overrides, theme_resolved, OvertureRunner) for y in yamls]
+    raise typer.Exit(code=_summarise(results))
+
+
+@app.command("osm")
+def cmd_osm(
+    iso3_or_yaml: str | None = typer.Argument(
+        None, help="ISO3 like NPL, or name of a YAML in ./configs/"
+    ),
+    theme: str | None = typer.Argument(None, help="Optional theme override (e.g. buildings)"),
+    configs_dir: Path | None = typer.Option(None, "--configs-dir"),
+    config: Path | None = typer.Option(None, "--config", "-c"),
+    output_dir: Path | None = typer.Option(None, "--output-dir", "-o"),
+    hdx_push: bool | None = typer.Option(None, "--hdx-push/--no-hdx-push"),
+    engine: str | None = typer.Option(
+        None,
+        "--engine",
+        help="OSM engine: 'geofabrik' (default) or 'planet_parquet'",
+    ),
+) -> None:
+    """Export OSM data via the configured engine."""
+    iso3_resolved, theme_resolved = _resolve_args(iso3_or_yaml, theme, configs_dir, config)
+    yamls = _resolve_config(iso3_resolved, configs_dir, config)
+    overrides = _build_overrides(iso3_resolved, hdx_push, output_dir, osm_engine=engine)
+    results = [_run_one(y, overrides, theme_resolved, OsmRunner) for y in yamls]
+    raise typer.Exit(code=_summarise(results))
+
+
+@app.command("osm-build-cache")
+def cmd_osm_build_cache(
+    pbf: Path | None = typer.Option(None, "--pbf", help="Local PBF path"),
+    planet: bool = typer.Option(False, "--planet", help="Download the latest planet PBF"),
+    config: Path | None = typer.Option(
+        None, "--config", "-c", help="Config to drive theme tag filters"
+    ),
+    snapshot: str | None = typer.Option(
+        None, "--snapshot", help="Snapshot label, defaults to today"
+    ),
+    themes: str | None = typer.Option(
+        None, "--themes", help="Comma-separated theme slugs to limit"
+    ),
+) -> None:
+    """Build the planet OSM PBF -> per-theme parquet cache at <cache_dir>/planet/<snapshot>/."""
+    from oex.osm.build_cache import build_cache
+    from oex.osm.fetch_planet import download_pbf
+
+    cfg: RootConfig = load_config(config)
+
+    if planet and pbf is not None:
+        raise typer.BadParameter("Pass either --planet or --pbf, not both")
+    if not planet and pbf is None:
+        raise typer.BadParameter("One of --planet or --pbf is required")
+
+    if planet:
+        src = cfg.source["osm"]
+        result = download_pbf(src.pbf_url, src.cache_dir + "/_pbf", md5_url=src.md5_url)
+        pbf_path = result.path
+    else:
+        assert pbf is not None
+        pbf_path = pbf
+
+    theme_list = [t.strip() for t in themes.split(",")] if themes else None
+    cache_root = Path(cfg.source["osm"].cache_dir) / "planet"
+    manifest = build_cache(
+        cfg,
+        pbf_path,
+        cache_root=cache_root,
+        snapshot=snapshot,
+        themes_filter=theme_list,
+    )
+    typer.echo(f"Cache snapshot: {manifest.snapshot}")
+    typer.echo(f"Themes built: {[t.theme for t in manifest.themes]}")
+
+
+def main() -> None:
+    app()
+
+
+if __name__ == "__main__":
+    main()

oex/config/__init__.py

@@ -0,0 +1,35 @@
+"""Typed configuration loading."""
+
+from oex.config.loader import (
+    ConfigError,
+    apply_overrides,
+    iter_configs,
+    load_config,
+    select_categories,
+)
+from oex.config.schema import (
+    BoundaryConfig,
+    CategoryConfig,
+    HdxConfig,
+    OsmSourceConfig,
+    OutputConfig,
+    OvertureSourceConfig,
+    ParallelConfig,
+    RootConfig,
+)
+
+__all__ = [
+    "BoundaryConfig",
+    "CategoryConfig",
+    "ConfigError",
+    "HdxConfig",
+    "OsmSourceConfig",
+    "OutputConfig",
+    "OvertureSourceConfig",
+    "ParallelConfig",
+    "RootConfig",
+    "apply_overrides",
+    "iter_configs",
+    "load_config",
+    "select_categories",
+]

oex/config/loader.py

@@ -0,0 +1,118 @@
+"""Layered YAML config: bundled defaults < user YAML < dotlist overrides."""
+
+import os
+from collections.abc import Iterator
+from importlib import resources
+from pathlib import Path
+from typing import Any, cast
+
+from omegaconf import DictConfig, ListConfig, OmegaConf
+
+from oex.config.schema import RootConfig
+
+
+class ConfigError(ValueError):
+    """Raised when a configuration is malformed."""
+
+
+def _load_yaml(source: str | os.PathLike[str]) -> DictConfig:
+    text: str
+    if isinstance(source, (str, os.PathLike)) and Path(source).exists():
+        text = Path(source).read_text(encoding="utf-8")
+    elif isinstance(source, str):
+        text = source
+    else:
+        raise ConfigError(f"Cannot load config from {source!r}")
+    cfg = OmegaConf.create(text)
+    if not isinstance(cfg, DictConfig):
+        raise ConfigError("Top-level YAML must be a mapping")
+    return cfg
+
+
+def _load_defaults() -> DictConfig:
+    pkg = resources.files("oex.defaults")
+    text = (pkg / "base.yaml").read_text(encoding="utf-8")
+    cfg = OmegaConf.create(text)
+    if not isinstance(cfg, DictConfig):
+        raise ConfigError("base.yaml is malformed")
+    return cfg
+
+
+def _load_categories_file(path: str | os.PathLike[str]) -> ListConfig:
+    raw = _load_yaml(path) if Path(str(path)).exists() else _load_yaml(str(path))
+    if "categories" in raw and isinstance(raw.categories, ListConfig):
+        return raw.categories
+    raise ConfigError(f"categories_file {path!r} must contain a top-level `categories:` list")
+
+
+def load_config(
+    user_config: str | os.PathLike[str] | None = None,
+    overrides: list[str] | None = None,
+) -> RootConfig:
+    """Build a RootConfig. categories precedence: defaults < categories_file < inline `categories:`."""
+    # Merge plain (untyped) configs first so user YAML can replace the
+    # categories list wholesale without tripping the structured-list type check.
+    merged: DictConfig = _load_defaults()
+
+    if user_config is not None:
+        user = _load_yaml(user_config)
+
+        if "categories_file" in user and user.categories_file:
+            merged.categories = _load_categories_file(str(user.categories_file))
+
+        if "categories" in user and isinstance(user.categories, ListConfig):
+            merged.categories = user.categories
+            del user["categories"]
+
+        merged = cast(DictConfig, OmegaConf.merge(merged, user))
+
+    if overrides:
+        merged = cast(DictConfig, OmegaConf.merge(merged, OmegaConf.from_dotlist(overrides)))
+
+    OmegaConf.resolve(merged)
+
+    schema = OmegaConf.structured(RootConfig)
+    typed = cast(DictConfig, OmegaConf.merge(schema, merged))
+    container: Any = OmegaConf.to_object(typed)
+    if not isinstance(container, RootConfig):
+        raise ConfigError("Merged config did not resolve to RootConfig")
+    return container
+
+
+def apply_overrides(cfg: RootConfig, overrides: dict[str, Any]) -> RootConfig:
+    """Apply a dict of dotted overrides to an already-loaded config."""
+    structured: DictConfig = cast(DictConfig, OmegaConf.structured(cfg))
+    dotlist = [f"{k}={v}" for k, v in overrides.items() if v is not None]
+    if dotlist:
+        structured = cast(DictConfig, OmegaConf.merge(structured, OmegaConf.from_dotlist(dotlist)))
+    OmegaConf.resolve(structured)
+    container: Any = OmegaConf.to_object(structured)
+    if not isinstance(container, RootConfig):
+        raise ConfigError("Override merge did not resolve to RootConfig")
+    return container
+
+
+def select_categories(cfg: RootConfig, theme: str | None) -> RootConfig:
+    """Restrict the config to a single category whose slugified name matches `theme`."""
+    if theme is None:
+        return cfg
+    needle = theme.strip().lower().replace("-", "_").replace(" ", "_")
+    kept = [c for c in cfg.categories if c.name.lower().replace(" ", "_") == needle]
+    if not kept:
+        available = ", ".join(c.name for c in cfg.categories) or "<none>"
+        raise ConfigError(f"Theme {theme!r} not found. Available: {available}")
+    new_cfg = OmegaConf.to_object(OmegaConf.structured(cfg))
+    if not isinstance(new_cfg, RootConfig):
+        raise ConfigError("select_categories failed to round-trip RootConfig")
+    new_cfg.categories = kept
+    return new_cfg
+
+
+def iter_configs(configs_dir: str | os.PathLike[str]) -> Iterator[Path]:
+    root = Path(configs_dir)
+    if not root.is_dir():
+        raise ConfigError(f"Not a directory: {root}")
+    for path in sorted(root.glob("*.yaml")):
+        yield path
+    for path in sorted(root.glob("*.yml")):
+        yield path

oex/config/schema.py

@@ -0,0 +1,143 @@
+"""Typed run configuration."""
+
+from dataclasses import dataclass, field
+from typing import Any
+
+OsmTagFilter = dict[str, Any]
+
+
+@dataclass
+class HdxConfig:
+    push: bool = False
+    site: str = "demo"
+    api_key: str | None = None
+    owner_org: str | None = None
+    maintainer: str | None = None
+    user_agent: str = "oex"
+    methodology: str = "Other"
+    methodology_other: str = "Open Source Geographic information"
+
+
+@dataclass
+class DuckdbConfig:
+    # 8 retries / 500 ms initial / 2x backoff and a 120 s timeout absorb
+    # transient S3 blips so a 200-country batch doesn't abort on one shard.
+    http_retries: int = 8
+    http_retry_wait_ms: int = 500
+    http_retry_backoff: float = 2.0
+    http_timeout_ms: int = 120_000
+    temp_dir: str = "/tmp/duckdb_temp"
+    enable_object_cache: bool = True
+
+
+@dataclass
+class LoggingConfig:
+    level: str = "INFO"
+    fmt: str | None = None
+
+
+@dataclass
+class OutputConfig:
+    dir: str = "output"
+    formats: list[str] = field(default_factory=lambda: ["gpkg", "shp"])
+    metadata: bool = False
+
+
+@dataclass
+class ParallelConfig:
+    enabled: bool = True
+    threads: int | None = None
+    memory_gb: int | None = None
+
+
+@dataclass
+class BoundaryConfig:
+    geom: str | None = None
+    geoboundaries_release: str = "CGAZ"
+    geoboundaries_level: str = "ADM0"
+
+
+@dataclass
+class OvertureSourceConfig:
+    enabled: bool = True
+    engine: str = "duckdb"
+    release: str = "latest"
+    s3_region: str = "us-west-2"
+    s3_bucket: str = "overturemaps-us-west-2"
+
+
+@dataclass
+class OsmSourceConfig:
+    enabled: bool = True
+    engine: str = "geofabrik"
+    cache_dir: str = "data/osm"
+    snapshot: str = "latest"
+    keep_pbf: bool = False
+    pbf_url: str = "https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf"
+    md5_url: str = "https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf.md5"
+    geofabrik_index_url: str = "https://download.geofabrik.de/index-v1.json"
+    geofabrik_clip_to_boundary: bool = True
+
+
+@dataclass
+class CategoryHdx:
+    title: str | None = None
+    notes: str = "Vector data export."
+    tags: list[str] = field(default_factory=lambda: ["geodata"])
+    license: str = "hdx-odc-odbl"
+    license_url: str | None = None
+    caveats: str = (
+        "Data may contain errors. Verified at the community level only; "
+        "individual features may need correction."
+    )
+
+
+@dataclass
+class CategoryOverture:
+    enabled: bool = True
+    theme: str = ""
+    feature_type: str = ""
+    select: list[str] = field(default_factory=list)
+    where: list[str] = field(default_factory=list)
+
+
+@dataclass
+class CategoryOsm:
+    # `filter` is the quackosm tag filter applied at parquet BUILD time.
+    # `where` is SQL applied at QUERY time over the already-built parquet.
+    enabled: bool = True
+    select: list[str] = field(default_factory=list)
+    where: list[str] = field(default_factory=list)
+    filter: OsmTagFilter = field(default_factory=dict)
+
+
+@dataclass
+class CategoryConfig:
+    name: str = ""
+    formats: list[str] | None = None
+    hdx: CategoryHdx = field(default_factory=CategoryHdx)
+    overture: CategoryOverture = field(default_factory=CategoryOverture)
+    osm: CategoryOsm = field(default_factory=CategoryOsm)
+
+
+@dataclass
+class RootConfig:
+    iso3: str = ""
+    key: str = ""
+    dataset_name: str | None = None
+    subnational: bool = False
+    frequency: str = "yearly"
+    categories_file: str | None = None
+    boundary: BoundaryConfig = field(default_factory=BoundaryConfig)
+    output: OutputConfig = field(default_factory=OutputConfig)
+    parallel: ParallelConfig = field(default_factory=ParallelConfig)
+    duckdb: DuckdbConfig = field(default_factory=DuckdbConfig)
+    logging: LoggingConfig = field(default_factory=LoggingConfig)
+    hdx: HdxConfig = field(default_factory=HdxConfig)
+    source: dict[str, Any] = field(
+        default_factory=lambda: {
+            "overture": OvertureSourceConfig(),
+            "osm": OsmSourceConfig(),
+        }
+    )
+    categories: list[CategoryConfig] = field(default_factory=list)

oex/defaults/__init__.py

@@ -0,0 +1 @@
+"""Bundled default YAML configuration."""