nace-cli 0.1.0__py3-none-any.whl

nace_cli/_docops.py

@@ -0,0 +1,414 @@
+"""Document operations (``parse``, ``split``, ``classify``, ``extract``, ``ground``) and uploads."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import uuid
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from nace_sdk import (
+    MAX_WAIT_SECONDS,
+    DocumentJob,
+    NaceAPIError,
+    NaceClient,
+    NaceError,
+    OnConflict,
+    ParseResultSource,
+    Source,
+    UploadedFile,
+    UrlSource,
+    WorkspaceFileSource,
+)
+from nace_sdk._core.constants import MAX_IDEMPOTENCY_KEY_LENGTH
+from nace_sdk._core.endpoints import url_file_name
+
+from nace_cli._inputs import page_ranges, parse_options, read_json, read_secret
+from nace_cli._jobs import add_wait_args, submit_and_report
+from nace_cli._output import CommandFailed, add_job_output_args, to_json
+
+SOURCE_HELP = """\
+SOURCE is one of:
+  https://...                   a public link the document service downloads
+                                (--file-name names the file when the link doesn't end in one)
+  a local file                  uploaded into your workspace first, then used as a workspace file
+  ws:<workspace_id>/<file_id>   a file already in your workspace (what `nace upload` prints)
+  job:<job_id>                  a finished parse job's result, so it is not parsed again
+                                (not for classify)
+"""
+
+ON_CONFLICT_CHOICES = ("reject", "new_version")
+
+
+def add_parsers(sub: argparse._SubParsersAction, common: argparse.ArgumentParser) -> None:
+    upload = sub.add_parser("upload", parents=[common], help="Upload a file into your workspace; prints its ws: source.")
+    upload.add_argument("file", help="Local file.")
+    upload.add_argument("--path", default=None, help="Path in the workspace (default: the file name).")
+    _add_on_conflict(upload)
+    upload.set_defaults(run=run_upload, family="client")
+
+    grant = sub.add_parser("upload-grant", parents=[common], help="Mint a one-use token another client can upload one file with.")
+    grant.add_argument("--path", default=None, help="Pin the workspace path the upload must use.")
+    grant.add_argument("--max-bytes", type=int, default=None, metavar="N", help="Largest file the grant accepts.")
+    grant.add_argument("--total-size-bytes", type=int, default=None, metavar="N", help="Exact size the file must have.")
+    grant.add_argument("--ttl-seconds", type=int, default=None, metavar="N", help="How long the grant stays valid.")
+    grant.set_defaults(run=run_upload_grant, family="client")
+
+    session = sub.add_parser("upload-session", parents=[common], help="Open a resumable upload in parts for a large file.")
+    session.add_argument("--path", required=True, help="Path of the file in the workspace.")
+    session.add_argument("--total-size-bytes", type=int, required=True, metavar="N", help="The file's exact size.")
+    session.add_argument("--ttl-seconds", type=int, default=None, metavar="N", help="How long the session accepts parts.")
+    _add_on_conflict(session)
+    session.add_argument("--idempotency-key", default=None, metavar="KEY", help="Retry-safe key; the same key returns the same session.")
+    session.set_defaults(run=run_upload_session, family="client")
+
+    parse = _operation(sub, common, "parse", "Convert a document to Markdown, text or layout blocks.")
+    parse.add_argument(
+        "--format",
+        dest="formats",
+        default=None,
+        help="Comma list of markdown,text,blocks (default: the API's markdown,blocks; stdout shows the Markdown).",
+    )
+    parse.add_argument(
+        "--password",
+        default=None,
+        metavar="VALUE",
+        help="An encrypted PDF's password: the text, @FILE, or - for stdin, typed hidden at a terminal (keeps it out of shell history).",
+    )
+    parse.set_defaults(run=run_parse)
+
+    split = _operation(sub, common, "split", "Split a multi-document file into the documents it contains.")
+    _add_class_args(split)
+    split.set_defaults(run=run_split)
+
+    classify = _operation(sub, common, "classify", "Classify a document, or each page, into your classes.")
+    _add_class_args(classify)
+    classify.add_argument("--granularity", choices=("document", "page"), default=None, help="One label for the file, or one per page.")
+    classify.set_defaults(run=run_classify)
+
+    extract = _operation(sub, common, "extract", "Extract structured fields with a JSON Schema, inline or saved.")
+    schema = extract.add_mutually_exclusive_group()
+    schema.add_argument("-s", "--schema", default=None, help="JSON Schema: a file path, @FILE, or inline JSON.")
+    schema.add_argument("--schema-id", default=None, metavar="ID", help="A saved schema (sch_...), at its latest version.")
+    extract.add_argument("--schema-version", type=int, default=None, metavar="N", help="With --schema-id, pin this version.")
+    extract.add_argument("--instructions", default=None, help="Extra guidance for the extraction.")
+    extract.set_defaults(run=run_extract)
+
+    ground = _operation(sub, common, "ground", "Find where each target text appears in a document.", pages=False)
+    ground.add_argument("--target", action="append", dest="targets", default=[], metavar="[ID=]TEXT", help="Repeatable; ids default to t1, t2, ...")
+    ground.add_argument("--targets", dest="targets_file", default=None, metavar="JSON", help="JSON array of targets.")
+    ground.add_argument(
+        "--semantic",
+        action="store_true",
+        help="Deprecated: match every target by meaning, unless it sets semantic itself. Fails on PDF, Word and PowerPoint.",
+    )
+    ground.set_defaults(run=run_ground)
+
+
+def _add_on_conflict(parser: argparse.ArgumentParser) -> None:
+    # Unset means the document service's reject, for a local SOURCE too: a job reads its workspace file's
+    # current version when it starts, not when it is created, so new_version changes the input of jobs
+    # still queued on that file.
+    parser.add_argument(
+        "--on-conflict",
+        choices=ON_CONFLICT_CHOICES,
+        default=None,
+        help=(
+            "When a different file has the path: reject (default) fails with path_conflict; new_version adds a version, "
+            "which jobs on that file not yet started then read."
+        ),
+    )
+
+
+def _operation(sub: argparse._SubParsersAction, common: argparse.ArgumentParser, name: str, help_text: str, *, pages: bool = True):
+    parser = sub.add_parser(
+        name, parents=[common], help=help_text, description=help_text, epilog=SOURCE_HELP, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument("source", help="https:// URL, local file, ws:<workspace>/<file>, or job:<parse job id>.")
+    parser.add_argument("--file-name", default=None, metavar="NAME", help="For an https:// SOURCE: the file's name; its extension picks the parser.")
+    parser.add_argument("--upload-path", default=None, metavar="PATH", help="For a local SOURCE: its path in the workspace (default: its name).")
+    _add_on_conflict(parser)
+    if pages:
+        # Ground reads whole documents: it has no page selection.
+        parser.add_argument("-p", "--pages", default=None, metavar="SPEC", help="1-based pages, e.g. 3 or 1-5,8 (not with a job: SOURCE).")
+        parser.add_argument(
+            "--mode",
+            dest="parse_mode",
+            choices=("low", "medium", "high"),
+            default=None,
+            help="Parse effort (default low); medium or high needs an original SOURCE, not job:.",
+        )
+    parser.add_argument(
+        "--option",
+        action="append",
+        dest="option_pairs",
+        default=[],
+        metavar="KEY=VALUE",
+        help="Any other request field; dotted keys nest, JSON values parse (output.formats='[\"text\"]').",
+    )
+    parser.add_argument("--options", default=None, metavar="JSON", help="Request fields as a JSON object (file, @FILE, inline); flags win.")
+    parser.add_argument("--wait-seconds", type=int, default=None, metavar="N", help="Hold the create open up to N seconds (max 60).")
+    parser.add_argument("--idempotency-key", default=None, metavar="KEY", help="Retry-safe key; the same key and body return the same job.")
+    add_wait_args(parser)
+    add_job_output_args(parser)
+    parser.set_defaults(family="client", operation=name)
+    return parser
+
+
+def _add_class_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--class",
+        action="append",
+        dest="class_specs",
+        default=[],
+        metavar="ID:LABEL[:DESCRIPTION]",
+        help="One class; repeatable.",
+    )
+    parser.add_argument("--classes", dest="classes_file", default=None, metavar="JSON", help="JSON array of classes.")
+
+
+def _check_source_flags(args: argparse.Namespace) -> None:
+    """Refuse flag and SOURCE combinations the document service would refuse, before any upload or request."""
+    spec: str = args.source
+    if spec.lower().startswith("http://"):
+        raise ValueError("SOURCE must be an https:// URL: the document service refuses http://")
+    if args.file_name is not None and not spec.startswith("https://"):
+        raise ValueError("--file-name names the file at an https:// SOURCE; it doesn't apply here")
+    if (args.upload_path is not None or args.on_conflict is not None) and spec.startswith(("https://", "ws:", "job:")):
+        raise ValueError("--upload-path and --on-conflict apply to a local-file SOURCE only")
+    if not spec.startswith("job:"):
+        return
+    operation = args.operation
+    if operation == "classify":
+        raise ValueError("classify reads the original file: pass a URL, a local file or ws:, not job:")
+    if getattr(args, "pages", None) and operation in ("parse", "extract"):
+        raise ValueError(f"{operation} can't select pages of a job: SOURCE; drop -p, or pass the original file")
+    if getattr(args, "parse_mode", None) not in (None, "low"):
+        raise ValueError("--mode medium or high parses again, so it needs the original file, not job:")
+    if getattr(args, "password", None) is not None:
+        raise ValueError("--password is for the original file; a job: SOURCE is already parsed")
+
+
+def resolve_source(client: NaceClient, args: argparse.Namespace) -> Source:
+    """``SOURCE`` as a request source, uploading a local file first."""
+    spec: str = args.source
+    if spec.startswith("https://"):
+        file_name = args.file_name
+        if file_name is None:
+            try:
+                file_name = url_file_name(spec)
+            except NaceError as exc:
+                raise NaceError(f"{exc} (or pass --file-name)") from None
+        return UrlSource(url=spec, file_name=file_name)
+    if spec.startswith("job:"):
+        return ParseResultSource(job_id=_uuid(spec[4:], spec))
+    if spec.startswith("ws:"):
+        workspace_id, sep, file_id = spec[3:].partition("/")
+        if not sep:
+            raise ValueError(f"expected ws:<workspace_id>/<file_id>, got {spec!r}")
+        return WorkspaceFileSource(workspace_id=_uuid(workspace_id, spec), file_id=_uuid(file_id, spec))
+    path = Path(spec)
+    if not path.is_file():
+        raise ValueError(f"no such file, and not a URL, ws: or job: source: {spec}")
+    print(f"uploading {path.name}", file=sys.stderr)
+    uploaded = upload_file(client, path, path=args.upload_path, on_conflict=args.on_conflict)
+    print(f"uploaded {workspace_source(uploaded)}", file=sys.stderr)
+    return uploaded.as_source()
+
+
+def upload_file(client: NaceClient, file: Path, *, path: str | None, on_conflict: OnConflict | None) -> UploadedFile:
+    try:
+        return client.documents.upload(file, path=path, on_conflict=on_conflict)
+    except NaceAPIError:
+        raise
+    except NaceError as exc:
+        # Every request has gone out by now: the upload failed or is still assembling.
+        raise CommandFailed(str(exc)) from None
+
+
+def _uuid(value: str, spec: str) -> str:
+    try:
+        return str(uuid.UUID(value))
+    except ValueError:
+        raise ValueError(f"not an id in {spec!r}: {value!r}") from None
+
+
+def workspace_source(uploaded: UploadedFile) -> str:
+    return f"ws:{uploaded.workspace_id}/{uploaded.file_id}"
+
+
+def _body(args: argparse.Namespace) -> dict[str, Any]:
+    """The request fields from ``--options``, then ``--option``, then the flags every operation shares."""
+    _check_source_flags(args)
+    base = read_json(args.options, what="--options") if args.options else {}
+    if not isinstance(base, dict):
+        raise ValueError("--options must be a JSON object")
+    body = parse_options(args.option_pairs, base)
+    if "source" in body:
+        raise ValueError("pass the source as SOURCE (with --file-name for a URL), not as a request field")
+    ranges = page_ranges(getattr(args, "pages", None))
+    if ranges is not None:
+        body["page_ranges"] = ranges
+    if getattr(args, "parse_mode", None):
+        body["parse_mode"] = args.parse_mode
+    return body
+
+
+def _take(body: dict[str, Any], key: str, flag: Any, kind: type) -> Any:
+    """A value the SDK takes as an argument: the flag's, else the body field's (which is removed either way)."""
+    value = body.pop(key, None)
+    if flag is not None:
+        return flag
+    if value is not None and (not isinstance(value, kind) or isinstance(value, bool)):
+        raise ValueError(f"{key} must be a {kind.__name__}")
+    return value
+
+
+def _classes(args: argparse.Namespace, body: dict[str, Any]) -> Any:
+    classes: list[dict[str, Any]] = []
+    if args.classes_file:
+        loaded = read_json(args.classes_file, what="--classes")
+        if not isinstance(loaded, list):
+            raise ValueError("--classes must be a JSON array")
+        classes.extend(loaded)
+    for spec in args.class_specs:
+        parts = spec.split(":", 2)
+        if len(parts) < 2 or not parts[0] or not parts[1]:
+            raise ValueError(f"--class needs ID:LABEL[:DESCRIPTION], got {spec!r}")
+        entry = {"id": parts[0], "label": parts[1]}
+        if len(parts) == 3 and parts[2]:
+            entry["description"] = parts[2]
+        classes.append(entry)
+    from_body = body.pop("classes", None)
+    if classes:
+        return classes
+    if not from_body:
+        raise ValueError("give at least one --class or --classes")
+    return from_body
+
+
+def _submit(client: NaceClient, args: argparse.Namespace, body: dict[str, Any], **typed: Any) -> int:
+    """Send the operation: ``typed`` as the SDK's own arguments, every other field of ``body`` as given.
+
+    The run functions have already moved their typed fields (classes, schema, targets...) out of
+    ``body``, so ``extra_body``, which the SDK merges last, can't override them or the source.
+    """
+    wait_seconds = _take(body, "wait_seconds", args.wait_seconds, int)
+    idempotency_key = _take(body, "idempotency_key", args.idempotency_key, str)
+    # The SDK checks these too, but only at the create, after a local SOURCE was uploaded.
+    if wait_seconds is not None and not 0 <= wait_seconds <= MAX_WAIT_SECONDS:
+        raise ValueError(f"wait_seconds must be between 0 and {MAX_WAIT_SECONDS}")
+    if idempotency_key is not None and not 0 < len(idempotency_key) <= MAX_IDEMPOTENCY_KEY_LENGTH:
+        raise ValueError(f"the idempotency key must be 1 to {MAX_IDEMPOTENCY_KEY_LENGTH} characters")
+    source = resolve_source(client, args)
+    create: Callable[..., DocumentJob] = getattr(client.documents, args.operation)
+    job = create(source, wait_seconds=wait_seconds, idempotency_key=idempotency_key, extra_body=body or None, **typed)
+    return submit_and_report(client, job, args)
+
+
+def run_upload(client: NaceClient, args: argparse.Namespace) -> int:
+    path = Path(args.file)
+    if not path.is_file():
+        raise ValueError(f"no such file: {args.file}")
+    uploaded = upload_file(client, path, path=args.path, on_conflict=args.on_conflict)
+    if args.json:
+        print(uploaded.model_dump_json(indent=2, exclude_none=True))
+    else:
+        print(workspace_source(uploaded))
+    return 0
+
+
+def run_upload_grant(client: NaceClient, args: argparse.Namespace) -> int:
+    grant = client.documents.create_upload_grant(
+        path=args.path, max_bytes=args.max_bytes, total_size_bytes=args.total_size_bytes, ttl_seconds=args.ttl_seconds
+    )
+    print(grant.model_dump_json(indent=2))
+    print("the token uploads one file into your workspace: hand it only to whoever uploads that file", file=sys.stderr)
+    return 0
+
+
+def run_upload_session(client: NaceClient, args: argparse.Namespace) -> int:
+    session = client.documents.create_upload_session(
+        path=args.path,
+        total_size_bytes=args.total_size_bytes,
+        ttl_seconds=args.ttl_seconds,
+        on_conflict=args.on_conflict,
+        idempotency_key=args.idempotency_key,
+    )
+    print(to_json(session.model_dump(mode="json")))
+    print("session_token writes into your workspace: hand it only to whoever uploads the parts", file=sys.stderr)
+    return 0
+
+
+def run_parse(client: NaceClient, args: argparse.Namespace) -> int:
+    body = _body(args)
+    if args.formats:
+        output = body.get("output")
+        formats = [item.strip() for item in args.formats.split(",") if item.strip()]
+        body["output"] = {**(output if isinstance(output, dict) else {}), "formats": formats}
+    if args.password is not None:
+        body["password"] = read_secret(args.password, prompt="PDF password: ")
+    return _submit(client, args, body)
+
+
+def run_split(client: NaceClient, args: argparse.Namespace) -> int:
+    body = _body(args)
+    return _submit(client, args, body, classes=_classes(args, body))
+
+
+def run_classify(client: NaceClient, args: argparse.Namespace) -> int:
+    body = _body(args)
+    if args.granularity:
+        body["granularity"] = args.granularity
+    return _submit(client, args, body, classes=_classes(args, body))
+
+
+def run_extract(client: NaceClient, args: argparse.Namespace) -> int:
+    body = _body(args)
+    if args.schema_version is not None and args.schema_id is None:
+        raise ValueError("--schema-version pins a version of a saved schema, so it needs --schema-id")
+    if args.schema is not None or args.schema_id is not None:
+        # A schema flag replaces whichever schema --options/--option named.
+        for key in ("schema", "schema_id", "schema_version"):
+            body.pop(key, None)
+        schema = read_json(args.schema, what="--schema") if args.schema is not None else None
+        if schema is not None and not isinstance(schema, dict):
+            raise ValueError("--schema must be a JSON object")
+        fields = {"schema": schema, "schema_id": args.schema_id, "schema_version": args.schema_version}
+    else:
+        fields = {key: body.pop(key, None) for key in ("schema", "schema_id", "schema_version")}
+        if fields["schema"] is None and fields["schema_id"] is None:
+            raise ValueError("give -s/--schema or --schema-id")
+        # As for the flags, refused here rather than by the SDK after a local SOURCE was uploaded.
+        if fields["schema"] is not None and fields["schema_id"] is not None:
+            raise ValueError("the options name both schema and schema_id; keep one")
+        if fields["schema_version"] is not None and fields["schema_id"] is None:
+            raise ValueError("schema_version pins a version of a saved schema, so it needs schema_id")
+    instructions = _take(body, "instructions", args.instructions, str)
+    return _submit(client, args, body, instructions=instructions, **fields)
+
+
+def run_ground(client: NaceClient, args: argparse.Namespace) -> int:
+    body = _body(args)
+    targets: list[Any] = []
+    if args.targets_file:
+        loaded = read_json(args.targets_file, what="--targets")
+        if not isinstance(loaded, list):
+            raise ValueError("--targets must be a JSON array")
+        targets.extend(loaded)
+    for index, spec in enumerate(args.targets, start=1):
+        target_id, sep, text = spec.partition("=")
+        if not sep:
+            target_id, text = f"t{index}", spec
+        targets.append({"id": target_id.strip(), "text": text})
+    from_body = body.pop("targets", None)
+    if not targets:
+        if not from_body:
+            raise ValueError("give at least one --target or --targets")
+        targets = from_body
+    if args.semantic and isinstance(targets, list):
+        # NDI matches by meaning per target; there is no request-wide switch.
+        targets = [{"semantic": True, **target} if isinstance(target, dict) else target for target in targets]
+    return _submit(client, args, body, targets=targets)

nace_cli/_entrypoint.py

@@ -0,0 +1,24 @@
+"""Bootstrap the ``nace`` executable: operating system trust store and UTF-8 stdio."""
+
+from __future__ import annotations
+
+import io
+import sys
+
+import truststore
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Enable native certificate verification and UTF-8 stdio before loading the CLI and SDK."""
+    truststore.inject_into_ssl()
+    # Piped stdio otherwise follows the Windows ANSI code page, which cannot encode every character a result may hold.
+    for stream in (sys.stdout, sys.stderr):
+        if isinstance(stream, io.TextIOWrapper):
+            stream.reconfigure(encoding="utf-8")
+    # The same for text piped in (`--state -`); `-sig` drops the BOM some Windows tools put first.
+    if isinstance(sys.stdin, io.TextIOWrapper):
+        sys.stdin.reconfigure(encoding="utf-8-sig")
+
+    from nace_cli._cli import main as cli_main  # noqa: PLC0415
+
+    return cli_main(argv)

nace_cli/_inputs.py

@@ -0,0 +1,94 @@
+"""Reading command-line values: ``@file`` / ``-`` text, JSON, ``key=value`` options, page specs, ids."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import uuid
+from getpass import getpass
+from pathlib import Path
+from typing import Any
+
+
+def read_text(value: str) -> str:
+    """``-`` reads stdin, ``@PATH`` reads a file, anything else is the text itself."""
+    if value == "-":
+        return sys.stdin.read()
+    if value.startswith("@"):
+        return Path(value[1:]).read_text(encoding="utf-8")
+    return value
+
+
+def read_secret(value: str, *, prompt: str) -> str:
+    """Like ``read_text``, without the one trailing newline a file or a pipe ends with.
+
+    ``-`` at a terminal asks with ``prompt`` and doesn't echo what is typed.
+    """
+    if value == "-" and sys.stdin.isatty():
+        return getpass(prompt)
+    text = read_text(value)
+    return text[:-1].removesuffix("\r") if text.endswith("\n") else text
+
+
+def job_id(value: str) -> str:
+    """An argparse ``type``: a job id, refused as a usage error before any request when it is not a UUID."""
+    try:
+        return str(uuid.UUID(value.strip()))
+    except ValueError:
+        raise argparse.ArgumentTypeError(f"not a job id: {value!r}") from None
+
+
+def read_json(value: str, *, what: str) -> Any:
+    """Inline JSON, ``@PATH``, ``-``, or a path to a JSON file."""
+    if not value.startswith(("@", "-", "{", "[")) and Path(value).is_file():
+        value = f"@{value}"
+    try:
+        return json.loads(read_text(value))
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"{what} is not valid JSON: {exc.msg} (line {exc.lineno})") from None
+
+
+def parse_options(pairs: list[str] | None, base: dict[str, Any] | None = None) -> dict[str, Any]:
+    """``key=value`` pairs into a request body. Dotted keys nest; a JSON value is parsed.
+
+    ``output.formats=["markdown"]`` becomes ``{"output": {"formats": ["markdown"]}}``;
+    ``parse_mode=high`` stays a string.
+    """
+    options: dict[str, Any] = dict(base or {})
+    for pair in pairs or []:
+        key, sep, raw = pair.partition("=")
+        if not sep or not key:
+            raise ValueError(f"--option needs KEY=VALUE, got {pair!r}")
+        try:
+            value: Any = json.loads(raw)
+        except json.JSONDecodeError:
+            value = raw
+        target = options
+        *parents, leaf = key.split(".")
+        for part in parents:
+            child = target.get(part)
+            if not isinstance(child, dict):
+                child = target[part] = {}
+            target = child
+        target[leaf] = value
+    return options
+
+
+def page_ranges(spec: str | None) -> list[dict[str, int]] | None:
+    """``1-3,5`` into ``[{"start": 1, "end": 3}, {"start": 5, "end": 5}]`` (1-based, inclusive)."""
+    if not spec:
+        return None
+    ranges: list[dict[str, int]] = []
+    for part in spec.split(","):
+        part = part.strip()
+        start_text, sep, end_text = part.partition("-")
+        try:
+            start = int(start_text)
+            end = int(end_text) if sep else start
+        except ValueError:
+            raise ValueError(f"--pages must look like 3 or 1-5,8, got {spec!r}") from None
+        if start < 1 or end < start:
+            raise ValueError(f"--pages must look like 3 or 1-5,8, got {spec!r}")
+        ranges.append({"start": start, "end": end})
+    return ranges