geodeploy 1.3.0__py3-none-any.whl

geodeploy/cli/commands/upload.py

@@ -0,0 +1,154 @@
+"""`geodeploy upload` — one command for every kind of file, and any number of them.
+
+The route (through the API, direct-to-storage, chunked, CSV, raster) is worked out per file by
+`geodeploy.uploads.plan`; `--dry-run` prints those decisions without moving a byte, which is the
+honest way to find out that a 300 MB GeoPackage is going to become a GeoParquet layer rather than
+a PostGIS table.
+"""
+from __future__ import annotations
+
+import os
+from typing import Any, Dict, List
+
+from ...errors import GeoDeployError
+from ...uploads import LARGE_UPLOAD_THRESHOLD
+from ..main import add_command
+from ..output import EXIT_GENERIC, EXIT_OK, human_size
+from ._common import confirm  # noqa: F401  (kept for symmetry with other command modules)
+
+
+def register(subparsers) -> None:
+    parser = add_command(
+        subparsers, "upload", cmd_upload,
+        "upload one or more files and register them as layers",
+        epilog="""\
+examples:
+  geodeploy upload roads.gpkg                       one file
+  geodeploy upload *.gpkg *.tif --wait              a whole directory's worth, waiting for ingest
+  geodeploy upload sites.csv --x lon --y lat        CSV points (columns guessed if omitted)
+  geodeploy upload plots.csv --wkt geometry         CSV with WKT geometry of any type
+  geodeploy upload big.parquet --name "Parcels"     GeoParquet, direct to storage, chunked
+  geodeploy upload data/*.tif --dry-run             what would happen, without doing it
+
+Files at or over {threshold} bypass the API and upload straight to object storage in parts —
+that is what makes a multi-gigabyte upload survive a proxy in front of the instance.
+""".format(threshold=human_size(LARGE_UPLOAD_THRESHOLD)))
+
+    parser.add_argument("files", nargs="+", help="files to upload")
+    parser.add_argument("--type", dest="layer_type", choices=["vector", "raster"],
+                        help="force the layer type (default: from the extension)")
+    parser.add_argument("--name", help="layer name (only sensible with a single file)")
+    parser.add_argument("--wait", action="store_true",
+                        help="wait for ingest to finish, and fail if it does not")
+    parser.add_argument("--dry-run", action="store_true",
+                        help="show the route each file would take, and stop")
+    parser.add_argument("--concurrency", type=int, default=1,
+                        help="files uploaded at once (default 1; each large file already uses "
+                             "four parallel parts)")
+    parser.add_argument("--stop-on-error", action="store_true",
+                        help="stop at the first failure instead of continuing")
+    parser.add_argument("--public", action="store_true",
+                        help="share each layer publicly once it is ready (STAC + OGC + raw asset)")
+
+    csv_group = parser.add_argument_group("CSV geometry")
+    csv_group.add_argument("--x", dest="x_column", help="longitude/easting column")
+    csv_group.add_argument("--y", dest="y_column", help="latitude/northing column")
+    csv_group.add_argument("--wkt", dest="wkt_column", help="WKT geometry column (any geometry type)")
+    csv_group.add_argument("--srid", type=int, default=4326, help="CRS of the coordinates (default 4326)")
+    csv_group.add_argument("--delimiter", choices=["comma", "semicolon", "tab", "pipe", "space"],
+                           help="field delimiter (default: sniffed from the file)")
+    csv_group.add_argument("--no-guess", action="store_true",
+                           help="do not guess geometry columns — fail instead")
+
+
+def cmd_upload(ctx, args) -> int:
+    client = ctx.client()
+    out = ctx.out
+
+    if args.name and len(args.files) > 1:
+        out.error("--name applies to a single file; upload them one at a time to name each.")
+        return EXIT_GENERIC
+
+    plans = []
+    for path in args.files:
+        plans.append(client.uploads.plan(
+            path, layer_type=args.layer_type, name=args.name, x_column=args.x_column,
+            y_column=args.y_column, wkt_column=args.wkt_column, srid=args.srid,
+            delimiter=args.delimiter, guess_csv=not args.no_guess))
+
+    for plan in plans:
+        if plan.csv_opts and plan.csv_opts.get("guessed"):
+            geometry = (("WKT column {0}".format(plan.csv_opts["wkt_column"]))
+                        if plan.csv_opts.get("wkt_column")
+                        else "x={0}, y={1}".format(plan.csv_opts.get("x_column"),
+                                                   plan.csv_opts.get("y_column")))
+            out.warn("{0}: guessed geometry from the header ({1}). Pass --x/--y or --wkt to be "
+                     "explicit.".format(os.path.basename(plan.path), geometry))
+
+    if args.dry_run:
+        ctx.out.render([dict(p.as_dict(), size_h=human_size(p.size)) for p in plans],
+                       ["path", "layer_type", "route", "name", "size_h", "chunked", "reason"])
+        return EXIT_OK
+
+    results = []  # type: List[Any]
+    failures = 0
+    for plan in plans:
+        progress = out.progress(os.path.basename(plan.path), plan.size)
+        out.info("{0} → {1} ({2}, {3})".format(os.path.basename(plan.path), plan.name,
+                                               plan.route, human_size(plan.size)))
+        try:
+            result = client.uploads.upload(
+                plan.path, plan=plan, wait=False,
+                on_progress=lambda done, total: progress.update(done, total))
+            progress.finish()
+            out.info("  queued as layer {0} (job {1})".format(result.layer_id, result.job_id))
+            results.append(result)
+        except GeoDeployError as exc:
+            progress.finish()
+            failures += 1
+            out.error("{0}: {1}".format(os.path.basename(plan.path), exc))
+            if args.stop_on_error:
+                break
+
+    if args.wait:
+        for result in results:
+            failures += _wait_for(ctx, result)
+
+    if args.public:
+        for result in results:
+            if result.final is None or (result.final or {}).get("status") in ("ready", "completed"):
+                try:
+                    client.layers.api(result.plan.layer_type).share(result.layer_id,
+                                                                    visibility="public")
+                    out.info("  {0} is now public.".format(result.plan.name))
+                except GeoDeployError as exc:
+                    out.warn("Could not share {0}: {1}".format(result.plan.name, exc))
+
+    payload = [r.as_dict() for r in results]
+    if out.json_mode:
+        out.json({"ok": failures == 0, "uploaded": payload, "failed": failures})
+    else:
+        out.table(payload, ["file", "name", "layer_type", "layer_id", "job_id"]) if payload else None
+        if not failures:
+            out.success("{0} file(s) uploaded.{1}".format(
+                len(results), "" if args.wait else " Ingest continues in the background — "
+                                                   "`geodeploy layers list` shows progress."))
+    return EXIT_GENERIC if failures else EXIT_OK
+
+
+def _wait_for(ctx, result) -> int:
+    """Follow one ingest job to the end. Returns 1 if it failed, 0 otherwise."""
+    out = ctx.out
+    label = os.path.basename(result.plan.path)
+
+    def on_progress(status: Dict[str, Any]) -> None:
+        out.info("  {0}: {1:3d}%  {2}".format(label, status.get("progress") or 0,
+                                              status.get("current_step") or status.get("status")))
+    try:
+        result.final = ctx.client().jobs.wait(result.job_id, result.plan.layer_type,
+                                              on_progress=on_progress)
+        out.success("{0} ready (layer {1}).".format(label, result.layer_id))
+        return 0
+    except GeoDeployError as exc:
+        out.error("{0}: {1}".format(label, exc))
+        return 1

geodeploy/cli/main.py

@@ -0,0 +1,263 @@
+"""`geodeploy` — the entry point: global options, dispatch, and the one place errors become exits.
+
+Argparse rather than a framework, for the no-dependency rule. The shape is conventional so it needs
+no learning: `geodeploy <group> <command> [args]`, `--json` anywhere, `-h` at every level.
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+from typing import Any, List, Optional
+
+from .. import __version__
+from ..config import Config, resolve
+from ..errors import (APIError, AuthError, ConfigError, GeoDeployError, PermissionError_,
+                      ServerError, TransportError, ValidationError)
+from ..jobs import JobFailed, JobTimeout
+from . import output
+from .output import (EXIT_AUTH, EXIT_GENERIC, EXIT_NETWORK, EXIT_OK, EXIT_SERVER, EXIT_USAGE,
+                     Formatter)
+
+EPILOG = """\
+examples:
+  geodeploy login https://geodeploy.example.org      log in and remember the instance
+  geodeploy upload roads.gpkg sites.csv --wait       upload several files and wait for ingest
+  geodeploy layers list --type vector                what is on the instance
+  geodeploy portals add-layer 3 roads --color '#e11d48' --marker star
+  geodeploy portals style 3 roads --color-field pop --classify quantile --classes 5
+  geodeploy portals publish 3                        make the edits live
+
+Every command takes --json for machine-readable output. Exit codes: 0 ok, 1 failed, 2 bad usage,
+3 authentication, 4 network, 5 server error.
+"""
+
+
+class Context(object):
+    """Everything a command handler needs: the formatter, the resolved instance, a lazy client."""
+
+    def __init__(self, args: argparse.Namespace, fmt: Formatter):
+        self.args = args
+        self.out = fmt
+        self.config = Config.load()
+        self._client = None  # type: Optional[Any]
+        self._resolved = None  # type: Optional[Any]
+
+    @property
+    def resolved(self):
+        if self._resolved is None:
+            self._resolved = resolve(url=getattr(self.args, "url", None),
+                                     token=getattr(self.args, "token", None),
+                                     profile=getattr(self.args, "profile", None),
+                                     config=self.config)
+        return self._resolved
+
+    def client(self, auth_required: bool = True, session: bool = False):
+        """The API client for this invocation, built once.
+
+        `auth_required=False` is for the public surfaces (STAC, OGC, templates): those work with a
+        URL alone, and demanding a token to read what the whole internet can read would be silly.
+
+        `session=True` marks a command that hits a route which REJECTS API tokens by design
+        (`/admin/*`, `/tokens`, ownership transfer). It prefers a stored password session; with
+        only a token available it still sends it, so the user gets the server's own 403 — which
+        `admin.py` rewrites into "run `geodeploy login --password`" — rather than a client-side
+        guess about what the instance would have said.
+        """
+        if self._client is None:
+            info = self.resolved
+            if not info.url:
+                raise AuthError(401, "No instance configured.", "")
+            token, jwt = info.token, info.jwt
+            if session and jwt:
+                token = None           # a session outranks a token for these routes
+            if auth_required and not token and not jwt:
+                raise AuthError(401, "No credentials for {0}.".format(info.url), "")
+            from ..client import Client
+            self._client = Client(
+                info.url, token=token, jwt=jwt,
+                timeout=getattr(self.args, "timeout", None) or 120.0,
+                verify_tls=not getattr(self.args, "insecure", False),
+                on_request=(lambda method, url: self.out.debug("{0} {1}".format(method, url)))
+                if self.out.verbose else None)
+            self.out.debug("instance {0} (from {1}), credential from {2}".format(
+                info.url, info.source_url, info.source_token))
+        return self._client
+
+
+def _global_flags() -> argparse.ArgumentParser:
+    """The flags every command accepts, as a PARENT parser.
+
+    Attached to the root *and* to every leaf command, because `geodeploy layers list --json` is
+    what people type — argparse would otherwise only accept `geodeploy --json layers list`, which
+    nobody does twice. `SUPPRESS` as the default is what makes that safe: an unmentioned flag on
+    the subparser leaves the root's value alone instead of resetting it.
+    """
+    parent = argparse.ArgumentParser(add_help=False)
+    common = parent.add_argument_group("connection")
+    common.add_argument("-p", "--profile", default=argparse.SUPPRESS,
+                        help="use a saved profile (see `geodeploy profile`)")
+    common.add_argument("--url", default=argparse.SUPPRESS,
+                        help="instance URL, overriding the profile and GEODEPLOY_URL")
+    common.add_argument("--token", default=argparse.SUPPRESS,
+                        help="API token, overriding the stored one and GEODEPLOY_TOKEN")
+    common.add_argument("--timeout", type=float, default=argparse.SUPPRESS,
+                        help="seconds to wait for an API call (default 120)")
+    common.add_argument("--insecure", action="store_true", default=argparse.SUPPRESS,
+                        help="skip TLS verification (self-signed instances only)")
+
+    fmt = parent.add_argument_group("output")
+    fmt.add_argument("--json", action="store_true", dest="json_mode", default=argparse.SUPPRESS,
+                     help="machine-readable JSON on stdout, and nothing else")
+    fmt.add_argument("-q", "--quiet", action="store_true", default=argparse.SUPPRESS,
+                     help="only errors")
+    fmt.add_argument("-v", "--verbose", action="store_true", default=argparse.SUPPRESS,
+                     help="log each request to stderr")
+    return parent
+
+
+GLOBAL_FLAGS = _global_flags()
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="geodeploy",
+        description="Upload data, build portals and operate a GeoDeploy instance from the shell.",
+        epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter,
+        parents=[GLOBAL_FLAGS])
+    parser.add_argument("--version", action="version", version="geodeploy {0}".format(__version__))
+    # NO `set_defaults` for the global flags here. `parents=` shares ACTION OBJECTS between this
+    # parser and every leaf, and `set_defaults` rewrites `action.default` in place — which would
+    # replace the SUPPRESS above with a real default on every leaf, so `geodeploy --json layers
+    # list` would have its --json reset to False by the leaf parser. Callers read these with
+    # `getattr(args, name, default)` instead.
+
+    subparsers = parser.add_subparsers(dest="_group", metavar="<command>")
+
+    from .commands import (admin, auth, browse, catalog, imports, jobs, layers, portals,
+                           sources, upload)
+    for module in (auth, browse, upload, layers, portals, sources, imports, jobs,
+                   catalog, admin):
+        module.register(subparsers)
+    return parser
+
+
+def _tolerate_legacy_console() -> None:
+    """Never let a code-page limitation turn help text into a traceback.
+
+    `Formatter` degrades typographic characters it knows about, but argparse writes help and usage
+    straight to the stream, and an em dash in a `--help` epilog would raise UnicodeEncodeError on a
+    Windows console still running code page 437. Mojibake is a cosmetic problem; a crash printing
+    help is not.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)
+        if reconfigure is not None:
+            try:
+                reconfigure(errors="replace")
+            except (ValueError, OSError):  # pragma: no cover - a stream that cannot be reconfigured
+                pass
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    _tolerate_legacy_console()
+    parser = build_parser()
+    args = parser.parse_args(argv if argv is not None else sys.argv[1:])
+    fmt = Formatter(json_mode=getattr(args, "json_mode", False),
+                    quiet=getattr(args, "quiet", False),
+                    verbose=getattr(args, "verbose", False))
+
+    handler = getattr(args, "_handler", None)
+    if handler is None:
+        # A bare `geodeploy`, or a group with no command: show that group's help, not a traceback.
+        sub = getattr(args, "_subparser", None)
+        (sub or parser).print_help(sys.stderr)
+        return EXIT_USAGE
+
+    ctx = Context(args, fmt)
+    try:
+        return handler(ctx, args) or EXIT_OK
+    except KeyboardInterrupt:
+        fmt.error("Interrupted.")
+        return EXIT_GENERIC
+    except AuthError as exc:
+        fmt.error(exc.detail or "Authentication failed.", hint=_auth_hint(ctx, exc))
+        return EXIT_AUTH
+    except PermissionError_ as exc:
+        scope = exc.missing_scope
+        fmt.error(exc.detail or "Not allowed.",
+                  hint=("Mint a token with the {0} scope (Settings → API tokens)."
+                        .format(scope) if scope else
+                        "Your role may be too low for this — an editor cannot administer an "
+                        "instance, and administration is session-only anyway."))
+        return EXIT_AUTH
+    except ServerError as exc:
+        fmt.error(exc.detail or "The instance returned an error.",
+                  hint="Check `geodeploy admin health`, or the service logs.")
+        return EXIT_SERVER
+    except TransportError as exc:
+        fmt.error(str(exc), hint="If the host is right, check the instance is up and reachable.")
+        return EXIT_NETWORK
+    except JobTimeout as exc:
+        fmt.error(str(exc))
+        return EXIT_GENERIC
+    except JobFailed as exc:
+        fmt.error("Ingest failed: {0}".format(exc),
+                  hint="`geodeploy layers reprocess <layer>` restarts it without re-uploading.")
+        return EXIT_GENERIC
+    except ValidationError as exc:
+        # A ValidationError raised before any request has no URL: that is the CLI rejecting the
+        # arguments, which is a usage error (2). One that came back from the instance is a failed
+        # operation (1) — the arguments were fine, the data or the state was not.
+        fmt.error(exc.detail or str(exc))
+        return EXIT_GENERIC if exc.url else EXIT_USAGE
+    except APIError as exc:
+        fmt.error(exc.detail or str(exc))
+        return EXIT_GENERIC
+    except ConfigError as exc:
+        fmt.error(str(exc))
+        return EXIT_USAGE
+    except GeoDeployError as exc:
+        fmt.error(str(exc))
+        return EXIT_GENERIC
+    except BrokenPipeError:  # `geodeploy … | head` — not an error worth a message
+        return EXIT_OK
+    except OSError as exc:
+        fmt.error(str(exc))
+        return EXIT_GENERIC
+
+
+def _auth_hint(ctx: Context, exc: AuthError) -> str:
+    info = ctx.resolved
+    if not info.url:
+        return "Run `geodeploy login <instance-url>`, or set GEODEPLOY_URL."
+    if not info.token:
+        return ("Run `geodeploy login {0}` with a token from Settings → API tokens, "
+                "or set GEODEPLOY_TOKEN.".format(info.url))
+    return ("The credential for {0} was refused — it may be revoked or expired. "
+            "`geodeploy login {0}` stores a new one.".format(info.url))
+
+
+def add_command(group, name: str, handler, help_text: str, aliases=(), epilog: Optional[str] = None):
+    """Register one command on a group's subparser set, wiring its handler and help."""
+    parser = group.add_parser(name, help=help_text, description=help_text, aliases=list(aliases),
+                              epilog=epilog, parents=[GLOBAL_FLAGS],
+                              formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.set_defaults(_handler=handler)
+    return parser
+
+
+def group_parser(subparsers, name: str, help_text: str, aliases=()):
+    """Register a command GROUP (`geodeploy layers …`) and return its own subparser set.
+
+    `_subparser` is stashed so that `geodeploy layers` with no command prints the group's help
+    rather than the root's — the root's help is a wall, and the user has already narrowed down.
+    """
+    parser = subparsers.add_parser(name, help=help_text, description=help_text,
+                                   aliases=list(aliases),
+                                   formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.set_defaults(_subparser=parser)
+    return parser.add_subparsers(dest="_command", metavar="<subcommand>")
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())