mapchete-hub-cli 2025.8.0__py2.py3-none-any.whl

mapchete_hub_cli/__init__.py

@@ -0,0 +1,22 @@
+from mapchete_hub_cli.client import (
+    COMMANDS,
+    DEFAULT_TIMEOUT,
+    JOB_STATUSES,
+    MHUB_CLI_ZONES_WAIT_TILES_COUNT,
+    MHUB_CLI_ZONES_WAIT_TIME_SECONDS,
+    Client,
+    Job,
+    load_mapchete_config,
+)
+
+__all__ = [
+    "Client",
+    "Job",
+    "COMMANDS",
+    "DEFAULT_TIMEOUT",
+    "JOB_STATUSES",
+    "MHUB_CLI_ZONES_WAIT_TILES_COUNT",
+    "MHUB_CLI_ZONES_WAIT_TIME_SECONDS",
+    "load_mapchete_config",
+]
+__version__ = "2025.8.0"

mapchete_hub_cli/cli/__init__.py

@@ -0,0 +1,4 @@
+from mapchete_hub_cli.cli.main import mhub
+
+
+__all__ = ["mhub"]

mapchete_hub_cli/cli/cancel.py

@@ -0,0 +1,55 @@
+import logging
+
+import click
+
+from mapchete_hub_cli.cli import options
+from mapchete_hub_cli.client import Client, Jobs
+
+logger = logging.getLogger(__name__)
+
+
+@click.command(short_help="Cancel jobs.")
+@options.opt_job_ids
+@options.opt_output_path
+@options.opt_status
+@options.opt_command
+@options.opt_since_no_default
+@options.opt_until
+@options.opt_job_name
+@options.opt_force
+@options.opt_debug
+@click.pass_context
+def cancel(ctx, job_ids, debug=False, force=False, **kwargs):
+    """Cancel jobs and their follow-up jobs if batch was submitted."""
+    client = Client(**ctx.obj)
+    try:
+        kwargs.update(from_date=kwargs.pop("since"), to_date=kwargs.pop("until"))
+
+        if job_ids:
+            jobs = Jobs.from_jobs([client.job(job_id) for job_id in job_ids])
+
+        else:
+            if all([v is None for v in kwargs.values()]):  # pragma: no cover
+                click.echo(ctx.get_help())
+                raise click.UsageError(
+                    "Please either provide one or more job IDs or other search values."
+                )
+            jobs = client.jobs(**kwargs)
+
+        unfinished_jobs = jobs.unfinished_jobs(msg_writer=click.echo)
+
+        if not unfinished_jobs:  # pragma: no cover
+            click.echo("No revokable jobs found.")
+            return
+
+        for job in unfinished_jobs:
+            click.echo(job.job_id)
+        if force or click.confirm(
+            f"Do you really want to cancel {len(unfinished_jobs)} job(s)?", abort=True
+        ):
+            unfinished_jobs.cancel(msg_writer=click.echo)
+
+    except Exception as e:  # pragma: no cover
+        if debug:
+            raise
+        raise click.ClickException(e)

mapchete_hub_cli/cli/clean.py

@@ -0,0 +1,79 @@
+import logging
+
+import click
+
+from mapchete_hub_cli.cli import options
+from mapchete_hub_cli.client import Client
+
+logger = logging.getLogger(__name__)
+
+
+@click.command(short_help="Abort stalled jobs.")
+@click.pass_context
+@click.option(
+    "--inactive-since",
+    type=click.STRING,
+    default="5h",
+    help="Time since jobs have been inactive.",
+    show_default=True,
+)
+@click.option(
+    "--pending-since",
+    type=click.STRING,
+    default="3d",
+    help="Time since jobs have been pending.",
+    show_default=True,
+)
+@click.option(
+    "--skip-dashboard-check", is_flag=True, help="Skip dashboard availability check."
+)
+@click.option("--retry", is_flag=True, help="Retry instead of cancel stalled jobs.")
+@options.opt_use_old_image
+@options.opt_force
+@options.opt_debug
+def clean(
+    ctx: click.Context,
+    inactive_since: str = "5h",
+    pending_since: str = "3d",
+    skip_dashboard_check: bool = False,
+    retry: bool = False,
+    use_old_image: bool = False,
+    force: bool = False,
+    debug: bool = False,
+):
+    """
+    Checks for probably stalled jobs and offers to cancel or retry them.
+
+    The check looks at three properties:\n
+    - jobs which are pending for too long\n
+    - jobs which are parsing|initializing|running but have been inactive for too long\n
+    - jobs which are running, have a scheduler but scheduler dashboard is not available\n
+    """
+    try:
+        stalled_jobs = Client(**ctx.obj).stalled_jobs(
+            inactive_since=inactive_since,
+            pending_since=pending_since,
+            check_inactive_dashboard=not skip_dashboard_check,
+            msg_writer=click.echo,
+        )
+        if stalled_jobs:  # pragma: no cover
+            click.echo(f"found {len(stalled_jobs)} potentially stalled jobs:")
+            for job in stalled_jobs:
+                click.echo(job.job_id)
+            if force or click.confirm(
+                f"Do you really want to cancel {'and retry ' if retry else ''}{len(stalled_jobs)} job(s)?",
+                abort=True,
+            ):
+                if retry:
+                    stalled_jobs.cancel_and_retry(
+                        msg_writer=click.echo, use_old_image=use_old_image
+                    )
+                else:
+                    stalled_jobs.cancel(msg_writer=click.echo)
+        else:
+            click.echo("No stalled jobs found.")
+
+    except Exception as exc:  # pragma: no cover
+        if debug:
+            raise
+        raise click.ClickException(str(exc))

mapchete_hub_cli/cli/execute.py

@@ -0,0 +1,186 @@
+from time import sleep
+from typing import List, Optional, Tuple
+
+import click
+
+from mapchete_hub_cli.cli import options
+from mapchete_hub_cli.cli.progress import show_progress_bar
+from mapchete_hub_cli.client import Client
+from mapchete_hub_cli.parser import load_mapchete_config
+
+
+@click.command(help="Execute a process.")
+@options.arg_mapchete_files
+@options.opt_zoom
+@options.opt_area
+@options.opt_area_crs
+@options.opt_zones_within_area
+@options.opt_bounds
+@options.opt_point
+@options.opt_tile
+@options.opt_overwrite
+@options.opt_verbose
+@options.opt_dask_specs
+@options.opt_dask_max_submitted_tasks
+@options.opt_dask_chunksize
+@options.opt_dask_no_task_graph
+@options.opt_debug
+@options.opt_job_name
+@options.opt_make_zones
+@options.opt_full_zones
+@options.opt_zones_wait_count
+@options.opt_zones_wait_seconds
+@options.opt_zone
+@options.opt_force
+@click.pass_context
+def execute(
+    ctx: click.Context,
+    mapchete_files: List[str],
+    bounds: Optional[Tuple[float, float, float, float]] = None,
+    overwrite: bool = False,
+    verbose: bool = False,
+    debug: bool = False,
+    dask_no_task_graph: bool = False,
+    dask_max_submitted_tasks: int = 1000,
+    dask_chunksize: int = 100,
+    make_zones_on_zoom: Optional[int] = None,
+    full_zones: int = False,
+    zones_wait_count: int = 100,
+    zones_wait_seconds: float = 1.0,
+    job_name: Optional[str] = None,
+    zone: Optional[str] = None,
+    force: bool = False,
+    area: Optional[str] = None,
+    zones_within_area: bool = False,
+    **kwargs,
+):
+    """Execute a process."""
+    dask_settings = dict(
+        process_graph=not dask_no_task_graph,
+        max_submitted_tasks=dask_max_submitted_tasks,
+        chunksize=dask_chunksize,
+    )
+    client = Client(**ctx.obj)
+
+    for mapchete_file in mapchete_files:
+        try:
+            if make_zones_on_zoom is not None and (
+                bounds is None and area is None
+            ):  # pragma: no cover
+                raise click.UsageError(
+                    "--make-zones-on-zoom requires --bounds and/or --area"
+                )
+            elif make_zones_on_zoom is not None or zone is not None:
+                try:
+                    from mapchete.config.parse import guess_geometry
+                    from mapchete.tile import BufferedTilePyramid, BufferedTile
+                except ImportError:  # pragma: no cover
+                    raise ImportError(
+                        "please install mapchete_hub_cli[zones] extra for this feature."
+                    )
+                tp = BufferedTilePyramid(
+                    load_mapchete_config(mapchete_file)["pyramid"]["grid"]
+                )
+                zones: List[BufferedTile] = []
+                if zone:
+                    zones = [tp.tile(*zone)]
+                else:
+                    if bounds:
+                        zones = [
+                            tile_zone
+                            for tile_zone in tp.tiles_from_bounds(
+                                bounds, make_zones_on_zoom
+                            )
+                        ]
+                    if area:
+                        geometry, crs = guess_geometry(area)
+                        if crs != tp.crs:  # pragma: no cover
+                            raise ValueError(
+                                f"area CRS ({crs}) must be the same as BufferedTilePyramid CRS ({tp.crs})"
+                            )
+                        zones = zones or [
+                            tile_zone
+                            for tile_zone in tp.tiles_from_geom(
+                                geometry, make_zones_on_zoom
+                            )
+                        ]
+
+                        if zones_within_area:
+                            zones = [
+                                tile_zone
+                                for tile_zone in zones
+                                if tile_zone.bbox.within(geometry)
+                            ]
+                        else:
+                            zones = [
+                                tile_zone
+                                for tile_zone in zones
+                                if tile_zone.bbox.intersects(geometry)
+                            ]
+                if (
+                    force
+                    or len(zones) == 1
+                    or click.confirm(
+                        f"Do you really want to submit {len(zones)} jobs?", abort=True
+                    )
+                ):
+                    for tile_zone in zones:
+                        zone_job_name = (
+                            f"{job_name}-{tile_zone.zoom}-{tile_zone.row}-{tile_zone.col}"
+                            if job_name
+                            else None
+                        )
+                        process_bounds = (
+                            bounds_intersection(bounds, tile_zone.bounds)
+                            if (bounds and not full_zones)
+                            else tile_zone.bounds
+                        )
+                        if len(zones) >= zones_wait_count:  # pragma: no cover
+                            sleep(zones_wait_seconds)
+                        job = client.start_job(
+                            command="execute",
+                            config=mapchete_file,
+                            params=dict(
+                                kwargs,
+                                bounds=process_bounds,
+                                mode="overwrite" if overwrite else "continue",
+                                dask_settings=dask_settings,
+                                job_name=zone_job_name,
+                            ),
+                        )
+                        click.echo(job.job_id)
+            else:
+                job = client.start_job(
+                    command="execute",
+                    config=mapchete_file,
+                    params=dict(
+                        kwargs,
+                        bounds=bounds,
+                        mode="overwrite" if overwrite else "continue",
+                        dask_settings=dask_settings,
+                        job_name=job_name,
+                    ),
+                )
+                if verbose:  # pragma: no cover
+                    click.echo(f"job {job.job_id} {job.status}")
+                    if job.properties.get("dask_dashboard_link"):
+                        click.echo(
+                            f"dask dashboard: {job.properties.get('dask_dashboard_link')}"
+                        )
+                    show_progress_bar(job, disable=debug)
+                else:
+                    click.echo(job.job_id)
+
+        except Exception as e:  # pragma: no cover
+            if debug:
+                raise
+            raise click.ClickException(e)
+
+
+def bounds_intersection(bounds1, bounds2):
+    return (
+        max([bounds1[0], bounds2[0]]),
+        max([bounds1[1], bounds2[1]]),
+        min([bounds1[2], bounds2[2]]),
+        min([bounds1[3], bounds2[3]]),
+    )

mapchete_hub_cli/cli/job.py

@@ -0,0 +1,146 @@
+from typing import List, Optional
+
+import click
+import oyaml as yaml
+
+from mapchete_hub_cli.cli import options
+from mapchete_hub_cli.client import Client, Job
+from mapchete_hub_cli.enums import Status
+from mapchete_hub_cli.time import pretty_time, pretty_time_since, str_to_date
+
+
+@click.command(short_help="Show job status.")
+@click.argument("job_id", type=click.STRING)
+@options.opt_geojson
+@options.opt_metadata_items
+@click.option("--traceback", is_flag=True, help="Print only traceback if available.")
+@click.option("--show-config", is_flag=True, help="Print Mapchete config.")
+@click.option("--show-params", is_flag=True, help="Print Mapchete parameters.")
+@click.option("--show-process", is_flag=True, help="Print Mapchete process.")
+@options.opt_debug
+@click.pass_context
+def job(
+    ctx,
+    job_id,
+    geojson=False,
+    show_config=False,
+    show_params=False,
+    show_process=False,
+    traceback=False,
+    debug=False,
+    metadata_items=None,
+    **_,
+):
+    """
+    Show job status.
+
+    JOB_ID can either be a valid job ID or :last:, in which case the CLI will automatically
+    determine the most recently updated job.
+    """
+    try:
+        client = Client(**ctx.obj)
+        job = client.job(job_id)
+        if geojson:  # pragma: no cover
+            click.echo(job.to_json())
+            return
+        elif show_config:
+            click.echo(yaml.dump(job.properties["mapchete"]["config"], indent=2))
+            return
+        elif show_params:
+            for k, v in job.properties["mapchete"]["params"].items():
+                click.echo(
+                    f"{k}: {', '.join(map(str, v)) if v else None}"
+                ) if isinstance(v, list) else click.echo(f"{k}: {v}")
+            return
+        elif show_process:
+            process = job.properties["mapchete"]["config"].get("process")
+            process = process if isinstance(process, list) else [process]
+            for line in process:
+                click.echo(line)
+            return
+        elif traceback:  # pragma: no cover
+            click.echo(job.properties.get("exception"))
+            click.echo(job.properties.get("traceback"))
+        print_job_details(job, metadata_items=metadata_items, verbose=True)
+    except Exception as e:  # pragma: no cover
+        if debug:
+            raise
+        raise click.ClickException(e)
+
+
+status_colors = {
+    Status.pending: "blue",
+    Status.parsing: "yellow",
+    Status.initializing: "yellow",
+    Status.running: "yellow",
+    Status.retrying: "yellow",
+    Status.done: "green",
+    Status.failed: "red",
+    Status.cancelled: "magenta",
+}
+
+
+def print_job_details(
+    job: Job, metadata_items: Optional[List[str]] = None, verbose: bool = False
+):
+    try:
+        color = status_colors[job.status]
+    except KeyError:  # pragma: no cover
+        color = "white"
+
+    mapchete_config = job.properties.get("mapchete", {}).get("config", {})
+
+    # job ID and job status
+    click.echo(click.style(f"{job.job_id}", fg=color, bold=True))
+
+    if verbose:
+        # job name
+        click.echo(f"job name: {job.properties.get('job_name')}")
+
+        # status
+        click.echo(click.style(f"status: {job.status}"))
+
+        # exception
+        click.echo(click.style(f"exception: {job.properties.get('exception')}"))
+
+        # progress
+        current = job.properties.get("current_progress", 0)
+        total = job.properties.get("total_progress", 100)
+        progress = round(100 * current / total, 2) if total else 0.0
+        click.echo(f"progress: {progress}%")
+
+        # dask_dashboard_link
+        click.echo(f"dask dashboard: {job.properties.get('dask_dashboard_link')}")
+
+        # command
+        click.echo(f"command: {job.properties.get('command')}")
+
+        # output path
+        click.echo(f"output path: {mapchete_config.get('output', {}).get('path')}")
+
+        # bounds
+        click.echo(f"bounds: {job.bounds}")
+
+        # started, updated, finished time
+        for time_property in ["started", "updated", "finished"]:
+            click.echo(
+                f"{time_property}: {job.properties.get(time_property, 'unknown')}"
+            )
+
+        # runtime
+        runtime = job.properties.get("runtime", "unknown")
+        click.echo(f"runtime: {pretty_time(runtime) if runtime else None}")
+
+        # last received update
+        last_update = job.properties.get("updated", "unknown")
+        click.echo(
+            f"last received update: {pretty_time_since(str_to_date(last_update))} ago"
+        )
+
+    if metadata_items:
+        for i in metadata_items:
+            click.echo(f"{i}: {job.properties.get(i)}")
+
+    if verbose or metadata_items:
+        # append newline
+        click.echo("")

mapchete_hub_cli/cli/jobs.py

@@ -0,0 +1,80 @@
+import logging
+
+import click
+
+from mapchete_hub_cli.cli import options
+from mapchete_hub_cli.cli.job import print_job_details
+from mapchete_hub_cli.client import Client
+
+logger = logging.getLogger(__name__)
+
+
+@click.command(short_help="Show current jobs.")
+@options.opt_output_path
+@options.opt_status
+@options.opt_command
+@options.opt_since
+@options.opt_until
+@options.opt_job_name
+@options.opt_unique_by_job_name
+@options.opt_sort_by
+@options.opt_bounds
+@options.opt_geojson
+@options.opt_metadata_items
+@options.opt_verbose
+@options.opt_debug
+@click.pass_context
+def jobs(
+    ctx,
+    geojson=False,
+    verbose=False,
+    sort_by=None,
+    debug=False,
+    metadata_items=None,
+    **kwargs,
+):
+    """Show current jobs."""
+
+    def _sort_jobs(jobs, sort_by=None):
+        if sort_by == "status":
+            return list(
+                sorted(
+                    jobs,
+                    key=lambda x: (
+                        x.to_dict()["properties"]["status"],
+                        x.to_dict()["properties"]["updated"],
+                    ),
+                )
+            )
+        elif sort_by in ["started", "runtime"]:
+            return list(
+                sorted(jobs, key=lambda x: x.to_dict()["properties"][sort_by] or 0.0)
+            )
+        elif sort_by == "progress":
+
+            def _get_progress(job):
+                properties = job.to_dict().get("properties", {})
+                current = properties.get("current_progress")
+                total = properties.get("total_progress")
+                return 100 * current / total if total else 0.0
+
+            return list(sorted(jobs, key=lambda x: _get_progress(x)))
+
+    kwargs.update(from_date=kwargs.pop("since"), to_date=kwargs.pop("until"))
+    try:
+        client = Client(**ctx.obj)
+        jobs = client.jobs(**kwargs)
+        if geojson:
+            click.echo(jobs.to_json())
+        else:
+            # sort by status and then by timestamp
+            jobs = _sort_jobs(jobs, sort_by=sort_by)
+            logger.debug(jobs)
+            if verbose:
+                click.echo(f"{len(jobs)} jobs found. \n")
+            for i in jobs:
+                print_job_details(i, metadata_items=metadata_items, verbose=verbose)
+    except Exception as e:  # pragma: no cover
+        if debug:
+            raise
+        raise click.ClickException(e)

mapchete_hub_cli/cli/main.py

@@ -0,0 +1,51 @@
+from importlib import metadata
+
+import click
+from click_plugins import with_plugins
+
+from mapchete_hub_cli import DEFAULT_TIMEOUT, __version__
+
+
+entry_points = metadata.entry_points()
+commands = entry_points.select(group="mapchete_hub_cli.cli.commands")
+
+host_options = dict(host_ip="0.0.0.0", port=5000)
+
+
+@with_plugins(commands)
+@click.version_option(version=__version__, message="%(version)s")
+@click.group(help="Process control on Mapchete Hub.")
+@click.option(
+    "--host",
+    "-h",
+    type=click.STRING,
+    nargs=1,
+    default=f"{host_options['host_ip']}:{host_options['port']}",
+    help="Address and port of mhub endpoint",
+    show_default=True,
+)
+@click.option(
+    "--timeout",
+    type=click.INT,
+    default=DEFAULT_TIMEOUT,
+    help="Time in seconds to wait for server response.",
+    show_default=True,
+)
+@click.option(
+    "--user",
+    "-u",
+    type=click.STRING,
+    help="Username for basic auth. (Or set MHUB_USER env variable.)",
+)
+@click.option(
+    "--password",
+    "-p",
+    type=click.STRING,
+    help="Password for basic auth. (Or set MHUB_PASSWORD env variable.)",
+)
+@click.pass_context
+def mhub(ctx, host, **kwargs):
+    """Main command group."""
+    host = host if host.startswith("http") else f"http://{host}"
+    host = host if host.endswith("/") else f"{host}/"
+    ctx.obj = dict(host=host, **kwargs)