cng-datasets 0.3.0__py3-none-any.whl

cng_datasets/__init__.py

@@ -0,0 +1,13 @@
+"""
+Cloud-Native Geospatial Datasets Processing Toolkit
+
+A toolkit for processing large geospatial datasets into cloud-native formats
+(COG, GeoParquet, PMTiles) with H3 hexagonal indexing.
+"""
+
+__version__ = "0.3.0"
+
+# Lazy imports can be handled by users as needed,
+# or we can keep these commented out to allow core usage without all dependencies.
+# from . import vector, raster, k8s, storage
+# __all__ = ["vector", "raster", "k8s", "storage", "__version__"]

cng_datasets/cli.py

@@ -0,0 +1,463 @@
+"""
+Command-line interface for cng-datasets toolkit.
+"""
+
+import argparse
+import sys
+
+from cng_datasets.raster.cog import VALID_HEX_REDUCERS
+
+
+def main():
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description="Cloud-native geospatial dataset processing toolkit",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # Vector processing command
+    vector_parser = subparsers.add_parser("vector", help="Process vector datasets")
+    vector_parser.add_argument("--input", required=True, help="Input file URL")
+    vector_parser.add_argument("--output", required=True, help="Output directory URL")
+    vector_res_group = vector_parser.add_mutually_exclusive_group()
+    vector_res_group.add_argument("--resolution", type=int, default=10, help="H3 resolution")
+    vector_res_group.add_argument(
+        "--resolution-by-area", type=str, default=None,
+        help="Variable resolution by polygon area (issue #98). Comma-separated "
+             "'threshold:resolution' bins (planar deg2) plus a trailing catch-all "
+             "resolution, e.g. '12:8,600:6,5' (area<=12 -> res 8, <=600 -> res 6, "
+             "else res 5). Output carries a union schema + native_res column. "
+             "Include 0 in --parent-resolutions so the h0 partition column exists.")
+    vector_parser.add_argument("--chunk-size", type=int, default=500, help="Number of rows to process in pass 1 (geometry to H3 arrays)")
+    vector_parser.add_argument("--intermediate-chunk-size", type=int, default=10, help="Number of rows to process in pass 2 (unnesting arrays) - reduce if hitting OOM")
+    vector_parser.add_argument("--chunk-id", type=int, help="Process specific chunk")
+    vector_parser.add_argument("--parent-resolutions", type=str, default="9,8,0", help="Comma-separated parent H3 resolutions (default: '9,8,0')")
+    vector_parser.add_argument("--id-column", help="ID column name (auto-detected if not specified)")
+
+
+    # Raster processing command
+    raster_parser = subparsers.add_parser("raster", help="Process raster datasets")
+    raster_parser.add_argument("--input", required=True, action="append", dest="inputs",
+                               help="Input raster file (local or /vsicurl/ URL). Repeat for multiple tiles to mosaic.")
+    raster_parser.add_argument("--output-cog", help="Output COG file path")
+    raster_parser.add_argument("--output-parquet", help="Output parquet directory (e.g., s3://bucket/dataset/hex/)")
+    raster_parser.add_argument("--resolution", type=int, help="H3 resolution (auto-detected if not specified)")
+    raster_parser.add_argument("--parent-resolutions", type=str, default="0", help="Comma-separated parent H3 resolutions (default: '0')")
+    raster_parser.add_argument("--h0-index", type=int, help="Process specific h0 region (0-121), or omit to process all")
+    raster_parser.add_argument("--value-column", default="value", help="Name for raster value column (default: 'value')")
+    raster_parser.add_argument("--nodata", type=str,
+                               help="NoData value(s) to exclude. Accepts a single value or a "
+                                    "comma-separated list for categorical products with multiple "
+                                    "fill codes, e.g. '-9999,-1111,32767' (all collapsed to the "
+                                    "first in the COG and excluded from hex tiling).")
+    raster_parser.add_argument("--compression", default="deflate", help="COG compression (deflate, lzw, zstd)")
+    raster_parser.add_argument("--blocksize", type=int, default=512, help="COG block size (default: 512)")
+    raster_parser.add_argument("--resampling", default="nearest", help="Resampling method for COG creation (default: nearest)")
+    raster_parser.add_argument("--hex-resampling", default="mean",
+                               help="Reducer for aggregating source pixels into each "
+                                    "H3 cell. With --method=exact-extract (default), "
+                                    "one of: sum/mean/mode/fractions/max/min (max/min for "
+                                    "peak/richness rasters; 'fractions' emits per-class "
+                                    "coverage rows for categorical area accounting, #142). "
+                                    "With --method=warp-centroid, any GDAL resampleAlg "
+                                    "(average, sum, mode, near, bilinear, cubic, ...). "
+                                    "Default: mean.")
+    raster_parser.add_argument("--method", default="exact-extract",
+                               choices=("exact-extract", "warp-centroid"),
+                               help="Raster→hex algorithm. 'exact-extract' (default): "
+                                    "area-weighted per-cell, one row per cell, mass-conserving. "
+                                    "'warp-centroid': older gdal.Warp→XYZ→centroid path; "
+                                    "fast and low-memory but emits one row per warped pixel "
+                                    "(consumers GROUP BY h<res>) and is mass-conserving only "
+                                    "when hex pitch is finer than source pixel pitch (see #84).")
+    raster_parser.add_argument("--target-crs", default="EPSG:4326", help="Output CRS for mosaic (default: EPSG:4326)")
+    raster_parser.add_argument("--target-extent", help="Clip bbox 'xmin,ymin,xmax,ymax' in target CRS (mosaic only)")
+    raster_parser.add_argument("--target-resolution", type=float, help="Output pixel size in target CRS units (mosaic only)")
+    raster_parser.add_argument("--band", type=int, help="Extract single band from multi-band sources, 1-indexed (mosaic only)")
+    raster_parser.add_argument("--local-cache-dir", default="/tmp/cng-raster-cache",
+                               help="Directory to copy remote input rasters into before processing "
+                                    "(default: /tmp/cng-raster-cache). Reading a remote COG via "
+                                    "/vsis3/ pays per-pixel HTTP latency that dominates wall time "
+                                    "on dense h0 cells — local-cache gives ~12x speedup at the "
+                                    "cost of one upfront copy. Use --no-local-cache to stream.")
+    raster_parser.add_argument("--no-local-cache", dest="local_cache_dir", action="store_const",
+                               const=None, help="Stream the input via /vsis3/ instead of "
+                                                "copying to local disk first.")
+
+    # Repartition command
+    repartition_parser = subparsers.add_parser("repartition", help="Repartition chunks by h0")
+    repartition_parser.add_argument("--chunks-dir", required=True, help="Input chunks directory URL")
+    repartition_parser.add_argument("--output-dir", required=True, help="Output directory URL")
+    repartition_parser.add_argument("--source-parquet", required=True, help="Source parquet with full attributes")
+    repartition_parser.add_argument("--cleanup", action="store_true", default=True, help="Remove chunks after repartitioning")
+    repartition_parser.add_argument("--memory-limit", type=str, default=None, help="DuckDB memory limit (e.g. '27GiB'). Overrides DUCKDB_MEMORY_LIMIT env var.")
+
+    # K8s job generation command
+    k8s_parser = subparsers.add_parser("k8s", help="Generate Kubernetes job")
+    k8s_parser.add_argument("--job-name", required=True, help="Job name")
+    k8s_parser.add_argument("--cmd", nargs="+", required=True, help="Container command", dest="container_command")
+    k8s_parser.add_argument("--output", default="job.yaml", help="Output YAML file")
+    k8s_parser.add_argument("--chunks", type=int, help="Number of chunks for indexed job")
+    k8s_parser.add_argument("--namespace", default="biodiversity", help="Kubernetes namespace (default: biodiversity)")
+
+    # Workflow generation command
+    workflow_parser = subparsers.add_parser("workflow", help="Generate complete dataset workflow")
+    workflow_parser.add_argument("--dataset", required=True, help="Dataset name (e.g., redlining)")
+    workflow_parser.add_argument("--source-url", action="append", required=True, dest="source_urls", help="Source data URL (can be specified multiple times for multiple inputs)")
+    workflow_parser.add_argument("--bucket", required=True, help="S3 bucket for outputs")
+    workflow_parser.add_argument("--output-dir", default="k8s", help="Output directory for YAML files")
+    workflow_parser.add_argument("--namespace", default="biodiversity", help="Kubernetes namespace (default: biodiversity)")
+    workflow_parser.add_argument("--h3-resolution", type=int, default=None, help="Target H3 resolution (default: auto — 10 for polygons/points, 8 for lines)")
+    workflow_parser.add_argument(
+        "--resolution-by-area", type=str, default=None,
+        help="Variable resolution by polygon area (issue #98), e.g. '12:8,600:6,5'. "
+             "Mutually exclusive with --h3-resolution; emits the same flag into the "
+             "hex job. Include 0 in --parent-resolutions for the h0 partition column.")
+    workflow_parser.add_argument("--parent-resolutions", type=str, default="9,8,0", help="Comma-separated parent H3 resolutions (default: '9,8,0')")
+    workflow_parser.add_argument("--id-column", help="ID column name (auto-detected if not specified)")
+    workflow_parser.add_argument("--layer", help="Layer name for multi-layer datasets (e.g., GDB files)")
+    workflow_parser.add_argument("--hex-memory", type=str, default="8Gi", help="Memory per hex job pod (default: 8Gi)")
+    workflow_parser.add_argument("--max-parallelism", type=int, default=50, help="Maximum parallel hex jobs (default: 50)")
+    workflow_parser.add_argument("--max-completions", type=int, default=200, help="Maximum hex job completions (default: 200, increase to reduce chunk size/memory)")
+    workflow_parser.add_argument("--intermediate-chunk-size", type=int, default=10, help="Number of rows to process in pass 2 (unnesting arrays) - reduce if hitting OOM")
+    workflow_parser.add_argument("--row-group-size", type=int, default=100000, help="Number of rows per group in convert job (default: 100000)")
+    workflow_parser.add_argument("--hex-storage", type=str, default="10Gi", help="Ephemeral storage request/limit per hex job pod (default: 10Gi)")
+    workflow_parser.add_argument("--repartition-storage", type=str, default="50Gi", help="Ephemeral storage request/limit for repartition job pod (default: 50Gi)")
+    workflow_parser.add_argument("--repartition-memory", type=str, default="32Gi", help="Memory request/limit for repartition job pod (default: 32Gi)")
+    workflow_parser.add_argument("--backend", choices=["k8s", "armada"], default="k8s", help="Job backend: 'k8s' for standard Kubernetes Jobs (default), 'armada' for Armada queue submission")
+    # Cluster/storage configuration flags
+    workflow_parser.add_argument("--profile", default=None, metavar="NAME_OR_PATH", help="Cluster profile name (e.g. 'nrp') or path to a YAML profile file. Explicit flags below override profile values.")
+    workflow_parser.add_argument("--s3-endpoint", default=None, metavar="HOST", help="Internal S3 endpoint for jobs (default from profile, or rook-ceph-rgw-nautiluss3.rook)")
+    workflow_parser.add_argument("--s3-public-endpoint", default=None, metavar="HOST", help="Public S3 endpoint (default from profile, or s3-west.nrp-nautilus.io)")
+    workflow_parser.add_argument("--s3-secret-name", default=None, metavar="SECRET", help="Kubernetes secret name for S3 credentials (default from profile, or 'aws')")
+    workflow_parser.add_argument("--rclone-secret-name", default=None, metavar="SECRET", help="Kubernetes secret name for rclone config (default from profile, or 'rclone-config')")
+    workflow_parser.add_argument("--rclone-remote", default=None, metavar="REMOTE", help="Rclone remote name for setup-bucket and pmtiles (default from profile, or 'nrp')")
+    workflow_parser.add_argument("--priority-class", default=None, metavar="CLASS", help="Kubernetes priorityClassName; '' to omit (default from profile, or 'opportunistic')")
+    workflow_parser.add_argument("--node-affinity", default=None, choices=["gpu-avoid", "none"], help="Node affinity: 'gpu-avoid' (NRP GPU avoidance) or 'none' to omit (default from profile)")
+
+    # Raster workflow generation command
+    raster_workflow_parser = subparsers.add_parser("raster-workflow", help="Generate complete raster dataset workflow")
+    raster_workflow_parser.add_argument("--dataset", required=True, help="Dataset name")
+    raster_workflow_parser.add_argument("--source-url", required=True, action="append", dest="source_urls",
+                                        help="Source raster URL. Repeat for multiple tiles to mosaic.")
+    raster_workflow_parser.add_argument("--bucket", required=True, help="S3 bucket for outputs")
+    raster_workflow_parser.add_argument("--output-dir", default="k8s", help="Output directory for YAML files")
+    raster_workflow_parser.add_argument("--namespace", default="biodiversity", help="Kubernetes namespace")
+    raster_workflow_parser.add_argument("--h3-resolution", type=int, default=8, help="Target H3 resolution (default: 8)")
+    raster_workflow_parser.add_argument("--parent-resolutions", type=str, default="0", help="Comma-separated parent H3 resolutions (default: '0')")
+    raster_workflow_parser.add_argument("--value-column", default="value", help="Name for raster value column")
+    raster_workflow_parser.add_argument("--nodata", type=str,
+                                        help="NoData value(s) to exclude. Accepts a single value or "
+                                             "a comma-separated list for categorical products with "
+                                             "multiple fill codes, e.g. '-9999,-1111,32767'.")
+    raster_workflow_parser.add_argument("--hex-resampling", default="mean",
+                                        choices=VALID_HEX_REDUCERS,
+                                        help="Reducer for H3 aggregation. 'sum' for "
+                                             "counts (population, carbon); 'mean' for intensities; "
+                                             "'mode' for categorical (single dominant class); "
+                                             "'fractions' for categorical area accounting (one "
+                                             "(value, frac) row per class per cell); 'max'/'min' "
+                                             "for peak/extremum (species richness). Default: mean.")
+    raster_workflow_parser.add_argument("--hex-memory", type=str, default="32Gi", help="Memory per hex job pod (default: 32Gi)")
+    raster_workflow_parser.add_argument("--max-parallelism", type=int, default=61, help="Maximum parallel hex jobs (default: 61)")
+    raster_workflow_parser.add_argument("--hex-storage", type=str, default="20Gi", help="Ephemeral storage request/limit per hex job pod (default: 20Gi)")
+    raster_workflow_parser.add_argument("--cog-storage", type=str, default="50Gi", help="Ephemeral storage request/limit for COG preprocess job pod (default: 50Gi)")
+    raster_workflow_parser.add_argument("--target-extent", help="Clip bbox 'xmin,ymin,xmax,ymax' in EPSG:4326 (multi-tile only)")
+    raster_workflow_parser.add_argument("--target-resolution", type=float, help="Output pixel size in degrees (multi-tile only)")
+    raster_workflow_parser.add_argument("--band", type=int, help="Extract single band from multi-band sources, 1-indexed (multi-tile only)")
+    raster_workflow_parser.add_argument("--output-cog-name", help="S3 key for intermediate COG (default: {dataset}-cog.tif)")
+    raster_workflow_parser.add_argument("--backend", choices=["k8s", "armada"], default="k8s", help="Job backend: 'k8s' for standard Kubernetes Jobs (default), 'armada' for Armada queue submission")
+    # Cluster/storage configuration flags
+    raster_workflow_parser.add_argument("--profile", default=None, metavar="NAME_OR_PATH", help="Cluster profile name (e.g. 'nrp') or path to a YAML profile file. Explicit flags below override profile values.")
+    raster_workflow_parser.add_argument("--s3-endpoint", default=None, metavar="HOST", help="Internal S3 endpoint for jobs (default from profile, or rook-ceph-rgw-nautiluss3.rook)")
+    raster_workflow_parser.add_argument("--s3-public-endpoint", default=None, metavar="HOST", help="Public S3 endpoint (default from profile, or s3-west.nrp-nautilus.io)")
+    raster_workflow_parser.add_argument("--s3-secret-name", default=None, metavar="SECRET", help="Kubernetes secret name for S3 credentials (default from profile, or 'aws')")
+    raster_workflow_parser.add_argument("--rclone-secret-name", default=None, metavar="SECRET", help="Kubernetes secret name for rclone config (default from profile, or 'rclone-config')")
+    raster_workflow_parser.add_argument("--rclone-remote", default=None, metavar="REMOTE", help="Rclone remote name for setup-bucket (default from profile, or 'nrp')")
+    raster_workflow_parser.add_argument("--priority-class", default=None, metavar="CLASS", help="Kubernetes priorityClassName; '' to omit (default from profile, or 'opportunistic')")
+    raster_workflow_parser.add_argument("--node-affinity", default=None, choices=["gpu-avoid", "none"], help="Node affinity: 'gpu-avoid' (NRP GPU avoidance) or 'none' to omit (default from profile)")
+
+    # Sync job generation command
+    sync_job_parser = subparsers.add_parser("sync-job", help="Generate Kubernetes job for syncing between S3 locations")
+    sync_job_parser.add_argument("--job-name", required=True, help="Job name")
+    sync_job_parser.add_argument("--source", required=True, help="Source path (e.g., 'remote1:bucket/path')")
+    sync_job_parser.add_argument("--destination", required=True, help="Destination path (e.g., 'remote2:bucket/path')")
+    sync_job_parser.add_argument("--output", default="sync-job.yaml", help="Output YAML file (default: sync-job.yaml)")
+    sync_job_parser.add_argument("--namespace", default="biodiversity", help="Kubernetes namespace (default: biodiversity)")
+    sync_job_parser.add_argument("--cpu", default="2", help="CPU request/limit (default: 2)")
+    sync_job_parser.add_argument("--memory", default="4Gi", help="Memory request/limit (default: 4Gi)")
+    sync_job_parser.add_argument("--dry-run", action="store_true", help="Dry run mode (show what would be synced)")
+
+    # Storage management command
+    storage_parser = subparsers.add_parser("storage", help="Manage cloud storage")
+    storage_subparsers = storage_parser.add_subparsers(dest="storage_command")
+
+    cors_parser = storage_subparsers.add_parser("cors", help="Configure bucket CORS")
+    cors_parser.add_argument("--bucket", required=True, help="Bucket name")
+    cors_parser.add_argument("--endpoint", help="S3 endpoint URL")
+
+    sync_parser = storage_subparsers.add_parser("sync", help="Sync with rclone")
+    sync_parser.add_argument("--source", required=True, help="Source path")
+    sync_parser.add_argument("--destination", required=True, help="Destination path")
+    sync_parser.add_argument("--dry-run", action="store_true", help="Dry run mode")
+
+    setup_bucket_parser = storage_subparsers.add_parser("setup-bucket", help="Setup public bucket with CORS")
+    setup_bucket_parser.add_argument("--bucket", required=True, help="Bucket name")
+    setup_bucket_parser.add_argument("--remote", default="nrp", help="Rclone remote name (default: nrp)")
+    setup_bucket_parser.add_argument("--endpoint", help="S3 endpoint URL (defaults to AWS_PUBLIC_ENDPOINT env var)")
+    setup_bucket_parser.add_argument("--no-cors", action="store_true", help="Skip CORS configuration")
+    setup_bucket_parser.add_argument("--verify", action="store_true", help="Verify bucket configuration after setup")
+
+    args = parser.parse_args()
+
+    if not args.command:
+        parser.print_help()
+        sys.exit(1)
+
+    try:
+        _dispatch(args)
+    except ValueError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def _dispatch(args):
+    if args.command == "vector":
+        from .vector import process_vector_chunks
+        from .vector.h3_tiling import parse_resolution_by_area
+        # Parse parent resolutions from comma-separated string
+        parent_res = [int(x.strip()) for x in args.parent_resolutions.split(',') if x.strip()]
+        resolution_by_area = (
+            parse_resolution_by_area(args.resolution_by_area)
+            if args.resolution_by_area else None
+        )
+        process_vector_chunks(
+            input_url=args.input,
+            output_url=args.output,
+            chunk_id=args.chunk_id,
+            h3_resolution=args.resolution,
+            parent_resolutions=parent_res,
+            chunk_size=args.chunk_size,
+            intermediate_chunk_size=args.intermediate_chunk_size,
+            id_column=args.id_column,
+            resolution_by_area=resolution_by_area,
+        )
+
+    elif args.command == "raster":
+        from .raster import RasterProcessor, create_mosaic_cog
+
+        # Parse parent resolutions
+        parent_res = [int(x.strip()) for x in args.parent_resolutions.split(',') if x.strip()]
+
+        # Parse optional mosaic parameters
+        target_extent = None
+        if getattr(args, 'target_extent', None):
+            parts = [float(x) for x in args.target_extent.split(',')]
+            target_extent = tuple(parts)
+
+        input_path = args.inputs if len(args.inputs) > 1 else args.inputs[0]
+
+        # If multiple inputs and only --output-cog requested, use create_mosaic_cog directly
+        if isinstance(input_path, list) and args.output_cog and not args.output_parquet:
+            # Categorical sources (--hex-resampling mode/fractions) must not
+            # average class codes in the COG overviews (issue #108).
+            overview_resampling = (
+                "mode" if args.hex_resampling in ("mode", "fractions") else "average"
+            )
+            create_mosaic_cog(
+                source_urls=input_path,
+                output_path=args.output_cog,
+                target_crs=getattr(args, 'target_crs', 'EPSG:4326'),
+                target_extent=target_extent,
+                target_resolution=getattr(args, 'target_resolution', None),
+                band=getattr(args, 'band', None),
+                nodata=args.nodata,
+                resampling=args.resampling,
+                compression=args.compression,
+                overview_resampling=overview_resampling,
+            )
+        else:
+            processor = RasterProcessor(
+                input_path=input_path,
+                output_cog_path=args.output_cog,
+                output_parquet_path=args.output_parquet,
+                h3_resolution=args.resolution,
+                parent_resolutions=parent_res,
+                h0_index=args.h0_index,
+                value_column=args.value_column,
+                nodata_value=args.nodata,
+                compression=args.compression,
+                blocksize=args.blocksize,
+                resampling=args.resampling,
+                hex_resampling=args.hex_resampling,
+                method=getattr(args, 'method', 'exact-extract'),
+                target_crs=getattr(args, 'target_crs', 'EPSG:4326'),
+                target_extent=target_extent,
+                target_resolution=getattr(args, 'target_resolution', None),
+                band=getattr(args, 'band', None),
+                local_cache_dir=getattr(args, 'local_cache_dir', '/tmp/cng-raster-cache'),
+            )
+
+            if args.output_cog:
+                processor.create_cog()
+
+            if args.output_parquet:
+                if args.h0_index is not None:
+                    processor.process_h0_region()
+                else:
+                    processor.process_all_h0_regions()
+
+    elif args.command == "repartition":
+        from .vector import repartition_by_h0
+        repartition_by_h0(
+            chunks_dir=args.chunks_dir,
+            output_dir=args.output_dir,
+            source_parquet=args.source_parquet,
+            cleanup=args.cleanup,
+            memory_limit=args.memory_limit,
+        )
+
+    elif args.command == "k8s":
+        from .k8s import K8sJobManager
+        manager = K8sJobManager(namespace=getattr(args, 'namespace', 'biodiversity'))
+        if args.chunks:
+            job_spec = manager.generate_chunked_job(
+                job_name=args.job_name,
+                script_path=args.container_command[0],
+                num_chunks=args.chunks,
+            )
+        else:
+            job_spec = manager.generate_job_yaml(
+                job_name=args.job_name,
+                command=args.container_command,
+            )
+        manager.save_job_yaml(job_spec, args.output)
+
+    elif args.command == "sync-job":
+        from .k8s import generate_sync_job
+        generate_sync_job(
+            job_name=args.job_name,
+            source=args.source,
+            destination=args.destination,
+            output_file=args.output,
+            namespace=args.namespace,
+            cpu=args.cpu,
+            memory=args.memory,
+            dry_run=args.dry_run,
+        )
+
+    elif args.command == "workflow":
+        from .k8s import generate_dataset_workflow
+        from .vector.h3_tiling import parse_resolution_by_area
+        # Parse parent resolutions from comma-separated string
+        parent_res = [int(x.strip()) for x in args.parent_resolutions.split(',') if x.strip()]
+        if args.resolution_by_area and args.h3_resolution is not None:
+            raise ValueError("--resolution-by-area and --h3-resolution are mutually exclusive")
+        # Validate the spec early so workflow generation fails fast on a bad bin.
+        if args.resolution_by_area:
+            parse_resolution_by_area(args.resolution_by_area)
+        generate_dataset_workflow(
+            dataset_name=args.dataset,
+            source_urls=args.source_urls,
+            bucket=args.bucket,
+            output_dir=args.output_dir,
+            namespace=args.namespace,
+            h3_resolution=args.h3_resolution,
+            resolution_by_area=args.resolution_by_area,
+            parent_resolutions=parent_res,
+            id_column=args.id_column,
+            layer=args.layer,
+            hex_memory=args.hex_memory,
+            max_parallelism=args.max_parallelism,
+            max_completions=args.max_completions,
+            intermediate_chunk_size=args.intermediate_chunk_size,
+            row_group_size=args.row_group_size,
+            backend=args.backend,
+            hex_storage=args.hex_storage,
+            repartition_storage=args.repartition_storage,
+            repartition_memory=args.repartition_memory,
+            profile=args.profile,
+            s3_endpoint=args.s3_endpoint,
+            s3_public_endpoint=args.s3_public_endpoint,
+            s3_secret_name=args.s3_secret_name,
+            rclone_secret_name=args.rclone_secret_name,
+            rclone_remote=args.rclone_remote,
+            priority_class=args.priority_class,
+            node_affinity=args.node_affinity,
+        )
+
+    elif args.command == "raster-workflow":
+        from .k8s import generate_raster_workflow
+        # Parse parent resolutions
+        parent_res = [int(x.strip()) for x in args.parent_resolutions.split(',') if x.strip()]
+        # Parse optional mosaic parameters
+        target_extent = None
+        if getattr(args, 'target_extent', None):
+            parts = [float(x) for x in args.target_extent.split(',')]
+            target_extent = tuple(parts)
+        generate_raster_workflow(
+            dataset_name=args.dataset,
+            source_urls=args.source_urls,
+            bucket=args.bucket,
+            output_dir=args.output_dir,
+            namespace=args.namespace,
+            h3_resolution=args.h3_resolution,
+            parent_resolutions=parent_res,
+            value_column=args.value_column,
+            nodata_value=args.nodata,
+            hex_resampling=args.hex_resampling,
+            hex_memory=args.hex_memory,
+            max_parallelism=args.max_parallelism,
+            hex_storage=args.hex_storage,
+            cog_storage=args.cog_storage,
+            target_extent=target_extent,
+            target_resolution=getattr(args, 'target_resolution', None),
+            band=getattr(args, 'band', None),
+            output_cog_name=getattr(args, 'output_cog_name', None),
+            backend=args.backend,
+            profile=args.profile,
+            s3_endpoint=args.s3_endpoint,
+            s3_public_endpoint=args.s3_public_endpoint,
+            s3_secret_name=args.s3_secret_name,
+            rclone_secret_name=args.rclone_secret_name,
+            rclone_remote=args.rclone_remote,
+            priority_class=args.priority_class,
+            node_affinity=args.node_affinity,
+        )
+
+    elif args.command == "storage":
+        if args.storage_command == "cors":
+            from .storage import configure_bucket_cors
+            configure_bucket_cors(
+                bucket_name=args.bucket,
+                endpoint_url=args.endpoint,
+            )
+        elif args.storage_command == "sync":
+            from .storage import RcloneSync
+            syncer = RcloneSync(dry_run=args.dry_run)
+            syncer.sync(args.source, args.destination)
+        elif args.storage_command == "setup-bucket":
+            from .storage import setup_public_bucket
+            from .storage.setup_bucket import verify_bucket_config
+            import json
+
+            success = setup_public_bucket(
+                bucket_name=args.bucket,
+                remote=args.remote,
+                endpoint=args.endpoint,
+                set_cors=not args.no_cors,
+                verbose=True
+            )
+
+            if success and args.verify:
+                print("\nVerifying configuration...")
+                results = verify_bucket_config(args.bucket, args.endpoint)
+                print(json.dumps(results, indent=2))
+
+            sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()

cng_datasets/hex_checks.py

@@ -0,0 +1,63 @@
+"""Post-build invariants for H3-indexed (hex) datasets.
+
+These are schema-only assertions run after the hex write step (vector
+repartition and raster hex) to fail fast on a corrupt build rather than
+silently shipping a dataset that breaks downstream consumers.
+"""
+
+from typing import Callable, List, Tuple
+
+
+def assert_h3_columns_unsigned(
+    fetch: Callable[[str], List[Tuple]],
+    hex_glob: str,
+) -> None:
+    """Assert every physical H3 index column ``h{N>=1}`` reads back as ``UBIGINT``.
+
+    Native-cell columns (``h3_polygon_wkt_to_cells`` / ``h3_latlng_to_cell``)
+    are always ``UBIGINT``, but ``h3_cell_to_parent`` changed its return type
+    from signed ``BIGINT`` to ``UBIGINT`` in a newer h3 community-extension
+    release. Because both the pipeline and the MCP do *unpinned*
+    ``INSTALL h3 FROM community``, a dataset's parent-column sign would
+    otherwise be a fossil of the extension version on its build date. We keep
+    tracking the latest h3, so this is an assertion (fail the build) rather
+    than a version pin — it also catches a bespoke ingest that casts the native
+    cell column (e.g. a ``BIGINT`` native ``h6``).
+
+    ``h0`` is exempt: it is the Hive partition key, so DuckDB infers its type
+    from the directory string and always reads it back as signed ``BIGINT``
+    regardless of the physical type. The convention is therefore: physical
+    h-cols ``UBIGINT``; ``h0`` the signed hive key.
+
+    The check is schema-only (``DESCRIBE``, no data scan) and is evaluated
+    through the same path the consumer reads (the hive glob), so it sees the
+    types consumers actually get.
+
+    Args:
+        fetch: Callable that runs a SQL string and returns the result rows as a
+            list of tuples. Pass ``lambda sql: con.raw_sql(sql).fetchall()`` for
+            an ibis DuckDB connection, or ``lambda sql: con.execute(sql).fetchall()``
+            for a raw ``duckdb`` connection.
+        hex_glob: A ``read_parquet``-compatible path/glob for the written hex
+            output (e.g. ``s3://bucket/foo/hex/h0=*/data_0.parquet``).
+
+    Raises:
+        RuntimeError: if any ``h{N>=1}`` column is not ``UBIGINT``. See issue #102.
+    """
+    sql = (
+        "SELECT column_name, column_type FROM "
+        f"(DESCRIBE SELECT * FROM read_parquet('{hex_glob}')) "
+        "WHERE column_name SIMILAR TO 'h[1-9][0-9]*' "
+        "AND column_type <> 'UBIGINT'"
+    )
+    offenders = fetch(sql)
+    if offenders:
+        cols = ", ".join(f"{name} ({typ})" for name, typ in offenders)
+        raise RuntimeError(
+            "H3 index columns must be UBIGINT after the hex build "
+            "(h0 is exempt as the signed hive key), but found non-UBIGINT "
+            f"columns: {cols}. This usually means the h3 community extension "
+            "emitted signed BIGINT parents (an older h3_cell_to_parent) or an "
+            "ingest cast the native cell column. Rebuild with a current h3 "
+            "extension or recast the offending columns to UBIGINT. See issue #102."
+        )

cng_datasets/k8s/__init__.py

@@ -0,0 +1,25 @@
+"""Kubernetes job generation and management utilities."""
+
+from .jobs import K8sJobManager, generate_job_yaml, submit_job
+from .workflows import (
+    generate_dataset_workflow,
+    generate_raster_workflow,
+    generate_sync_job,
+    ClusterConfig,
+    load_profile,
+    cluster_config_from_args,
+)
+from .armada import (
+    k8s_job_to_armada,
+    k8s_indexed_job_to_armada,
+    convert_workflow_to_armada,
+    save_armada_yaml,
+)
+
+__all__ = [
+    "K8sJobManager", "generate_job_yaml", "submit_job",
+    "generate_dataset_workflow", "generate_raster_workflow", "generate_sync_job",
+    "ClusterConfig", "load_profile", "cluster_config_from_args",
+    "k8s_job_to_armada", "k8s_indexed_job_to_armada",
+    "convert_workflow_to_armada", "save_armada_yaml",
+]