nace-cli 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nace_cli-0.1.0 → nace_cli-0.2.0}/PKG-INFO +22 -6
- {nace_cli-0.1.0 → nace_cli-0.2.0}/README.md +20 -4
- {nace_cli-0.1.0 → nace_cli-0.2.0}/pyproject.toml +2 -2
- {nace_cli-0.1.0 → nace_cli-0.2.0}/pyproject.toml.orig +2 -2
- {nace_cli-0.1.0 → nace_cli-0.2.0}/src/nace_cli/__init__.py +1 -1
- {nace_cli-0.1.0 → nace_cli-0.2.0}/src/nace_cli/_cli.py +26 -2
- {nace_cli-0.1.0 → nace_cli-0.2.0}/src/nace_cli/_decide.py +74 -4
- {nace_cli-0.1.0 → nace_cli-0.2.0}/src/nace_cli/_docops.py +114 -25
- {nace_cli-0.1.0 → nace_cli-0.2.0}/src/nace_cli/_inputs.py +25 -0
- {nace_cli-0.1.0 → nace_cli-0.2.0}/src/nace_cli/_jobs.py +63 -22
- {nace_cli-0.1.0 → nace_cli-0.2.0}/src/nace_cli/_login.py +7 -4
- {nace_cli-0.1.0 → nace_cli-0.2.0}/src/nace_cli/_output.py +36 -6
- {nace_cli-0.1.0 → nace_cli-0.2.0}/src/nace_cli/_schemas.py +31 -7
- nace_cli-0.2.0/src/nace_cli/_search.py +370 -0
- nace_cli-0.2.0/src/nace_cli/_upload.py +263 -0
- nace_cli-0.2.0/src/nace_cli/_workspaces.py +604 -0
- {nace_cli-0.1.0 → nace_cli-0.2.0}/LICENSE +0 -0
- {nace_cli-0.1.0 → nace_cli-0.2.0}/src/nace_cli/_entrypoint.py +0 -0
- {nace_cli-0.1.0 → nace_cli-0.2.0}/src/nace_cli/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nace-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Nace CLI: calibrated decisions and document jobs from the shell
|
|
5
5
|
Keywords: drex,nace,cli,document-intelligence
|
|
6
6
|
Author: Nace AI
|
|
@@ -16,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.13
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.14
|
|
18
18
|
Classifier: Typing :: Typed
|
|
19
|
-
Requires-Dist: nace-sdk>=0.
|
|
19
|
+
Requires-Dist: nace-sdk>=0.2,<1
|
|
20
20
|
Requires-Dist: truststore>=0.10.4,<1
|
|
21
21
|
Requires-Python: >=3.11
|
|
22
22
|
Project-URL: Homepage, https://console.nace.ai
|
|
@@ -83,7 +83,7 @@ nace upload-grant --path inbox/scan.pdf --max-bytes 10000000 # a one-use uploa
|
|
|
83
83
|
nace upload-session --path big.csv --total-size-bytes 734003200
|
|
84
84
|
```
|
|
85
85
|
|
|
86
|
-
`upload-grant` and `upload-session` print JSON for another client to upload with: the CLI doesn't send parts or complete a session itself, and `nace upload` (or a local-file `SOURCE`) does every step for you.
|
|
86
|
+
`nace upload DIR` uploads every file under a folder, keeping its path inside the folder under `--path` (default: the folder's name), `-j` at a time (default 8). Hidden files and folders are left out, and a file the document service refuses as `unsupported_file_type` is skipped: stderr lists them when the run ends. `--ingest` (with `--workspace`) then ingests what was uploaded, 1000 files per ingestion. `upload-grant` and `upload-session` print JSON for another client to upload with: the CLI doesn't send parts or complete a session itself, and `nace upload` (or a local-file `SOURCE`) does every step for you.
|
|
87
87
|
|
|
88
88
|
## Jobs and their files
|
|
89
89
|
|
|
@@ -111,13 +111,29 @@ nace schema sch_... > invoice.schema.json # the JSON Schema itself; --json for t
|
|
|
111
111
|
nace schema-version sch_... 1
|
|
112
112
|
```
|
|
113
113
|
|
|
114
|
+
## Workspaces and search
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
nace workspace-create fy25-audit --global-context "Entity: Acme Holdings Ltd."
|
|
118
|
+
nace workspace-use <workspace_id> # later commands use it; --workspace ID or $NACE_WORKSPACE_ID win
|
|
119
|
+
nace upload ./report.pdf --workspace <workspace_id> --path reports/2025/report.pdf
|
|
120
|
+
nace upload ./fy25 --workspace <workspace_id> --ingest # every file under a folder, 8 at a time, then ingested
|
|
121
|
+
nace ingest --path-prefix reports/ # makes the files searchable
|
|
122
|
+
nace workspace-files --status failed
|
|
123
|
+
nace search "What were total liabilities at year end?"
|
|
124
|
+
nace qa "What was total freight spend in FY2024?" # stderr: session <id>; follow up with --session
|
|
125
|
+
nace kg-build
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
`search` and `qa` print the answer as the ndi CLI did: the answer, `evidences (N):` with each evidence as `1. path, page N` and its quote indented under it, then the Ground jobs to poll, the tables the answer refers to and any catalog rows; the route goes to stderr. Lists (`jobs`, `workspaces`, `workspace-files`) print a `<what> (N):` header, an indented line per item and `next: <cursor>` when more pages follow. Uploads go to a workspace only with `--workspace`; without it, to the server. `nace workspace-delete` takes the workspace only from `--workspace`.
|
|
129
|
+
|
|
114
130
|
## Output and exit codes
|
|
115
131
|
|
|
116
|
-
Stdout carries the payload and stderr the status, so `nace parse doc.pdf > doc.md` captures only the document. Exit `0` on success, `1` on an API or job failure (the message names the error type, code, request id and up to five fields that failed validation), `2` on a usage or configuration error.
|
|
132
|
+
Stdout carries the payload and stderr the status, so `nace parse doc.pdf > doc.md` captures only the document. A wait that runs out prints the job id on stdout. Exit `0` on success, `1` on an API or job failure (including an ingestion that failed a file, from `nace ingest`, `nace upload --ingest` or `nace job`) (the message names the error type, code, request id and up to five fields that failed validation), `2` on a usage or configuration error.
|
|
117
133
|
|
|
118
|
-
Requests are retried up to twice on a `408`, `429`, `5xx`, dropped connection or timeout, but only while the retry can start within 30 s of the first attempt, so a request that fails after 30 s, such as one that hit `--timeout` (120 s) or a slow `529`, is not retried. `schema-create
|
|
134
|
+
Requests are retried up to twice on a `408`, `429`, `5xx`, dropped connection or timeout, but only while the retry can start within 30 s of the first attempt, so a request that fails after 30 s, such as one that hit `--timeout` (120 s) or a slow `529`, is not retried. `schema-create`, `schema-version-add` and `workspace-create` are never retried.
|
|
119
135
|
|
|
120
|
-
Full guide: https://console.nace.ai/docs/developer-tools/cli (install, sign-in, `decide`, documents, uploads, jobs, schemas, exit codes).
|
|
136
|
+
Full guide: https://console.nace.ai/docs/developer-tools/cli (install, sign-in, `decide`, documents, uploads, jobs, schemas, workspaces, search, exit codes).
|
|
121
137
|
|
|
122
138
|
## License
|
|
123
139
|
|
|
@@ -58,7 +58,7 @@ nace upload-grant --path inbox/scan.pdf --max-bytes 10000000 # a one-use uploa
|
|
|
58
58
|
nace upload-session --path big.csv --total-size-bytes 734003200
|
|
59
59
|
```
|
|
60
60
|
|
|
61
|
-
`upload-grant` and `upload-session` print JSON for another client to upload with: the CLI doesn't send parts or complete a session itself, and `nace upload` (or a local-file `SOURCE`) does every step for you.
|
|
61
|
+
`nace upload DIR` uploads every file under a folder, keeping its path inside the folder under `--path` (default: the folder's name), `-j` at a time (default 8). Hidden files and folders are left out, and a file the document service refuses as `unsupported_file_type` is skipped: stderr lists them when the run ends. `--ingest` (with `--workspace`) then ingests what was uploaded, 1000 files per ingestion. `upload-grant` and `upload-session` print JSON for another client to upload with: the CLI doesn't send parts or complete a session itself, and `nace upload` (or a local-file `SOURCE`) does every step for you.
|
|
62
62
|
|
|
63
63
|
## Jobs and their files
|
|
64
64
|
|
|
@@ -86,13 +86,29 @@ nace schema sch_... > invoice.schema.json # the JSON Schema itself; --json for t
|
|
|
86
86
|
nace schema-version sch_... 1
|
|
87
87
|
```
|
|
88
88
|
|
|
89
|
+
## Workspaces and search
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
nace workspace-create fy25-audit --global-context "Entity: Acme Holdings Ltd."
|
|
93
|
+
nace workspace-use <workspace_id> # later commands use it; --workspace ID or $NACE_WORKSPACE_ID win
|
|
94
|
+
nace upload ./report.pdf --workspace <workspace_id> --path reports/2025/report.pdf
|
|
95
|
+
nace upload ./fy25 --workspace <workspace_id> --ingest # every file under a folder, 8 at a time, then ingested
|
|
96
|
+
nace ingest --path-prefix reports/ # makes the files searchable
|
|
97
|
+
nace workspace-files --status failed
|
|
98
|
+
nace search "What were total liabilities at year end?"
|
|
99
|
+
nace qa "What was total freight spend in FY2024?" # stderr: session <id>; follow up with --session
|
|
100
|
+
nace kg-build
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
`search` and `qa` print the answer as the ndi CLI did: the answer, `evidences (N):` with each evidence as `1. path, page N` and its quote indented under it, then the Ground jobs to poll, the tables the answer refers to and any catalog rows; the route goes to stderr. Lists (`jobs`, `workspaces`, `workspace-files`) print a `<what> (N):` header, an indented line per item and `next: <cursor>` when more pages follow. Uploads go to a workspace only with `--workspace`; without it, to the server. `nace workspace-delete` takes the workspace only from `--workspace`.
|
|
104
|
+
|
|
89
105
|
## Output and exit codes
|
|
90
106
|
|
|
91
|
-
Stdout carries the payload and stderr the status, so `nace parse doc.pdf > doc.md` captures only the document. Exit `0` on success, `1` on an API or job failure (the message names the error type, code, request id and up to five fields that failed validation), `2` on a usage or configuration error.
|
|
107
|
+
Stdout carries the payload and stderr the status, so `nace parse doc.pdf > doc.md` captures only the document. A wait that runs out prints the job id on stdout. Exit `0` on success, `1` on an API or job failure (including an ingestion that failed a file, from `nace ingest`, `nace upload --ingest` or `nace job`) (the message names the error type, code, request id and up to five fields that failed validation), `2` on a usage or configuration error.
|
|
92
108
|
|
|
93
|
-
Requests are retried up to twice on a `408`, `429`, `5xx`, dropped connection or timeout, but only while the retry can start within 30 s of the first attempt, so a request that fails after 30 s, such as one that hit `--timeout` (120 s) or a slow `529`, is not retried. `schema-create
|
|
109
|
+
Requests are retried up to twice on a `408`, `429`, `5xx`, dropped connection or timeout, but only while the retry can start within 30 s of the first attempt, so a request that fails after 30 s, such as one that hit `--timeout` (120 s) or a slow `529`, is not retried. `schema-create`, `schema-version-add` and `workspace-create` are never retried.
|
|
94
110
|
|
|
95
|
-
Full guide: https://console.nace.ai/docs/developer-tools/cli (install, sign-in, `decide`, documents, uploads, jobs, schemas, exit codes).
|
|
111
|
+
Full guide: https://console.nace.ai/docs/developer-tools/cli (install, sign-in, `decide`, documents, uploads, jobs, schemas, workspaces, search, exit codes).
|
|
96
112
|
|
|
97
113
|
## License
|
|
98
114
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "nace-cli"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.2.0"
|
|
4
4
|
description = "Nace CLI: calibrated decisions and document jobs from the shell"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -24,7 +24,7 @@ classifiers = [
|
|
|
24
24
|
"Typing :: Typed",
|
|
25
25
|
]
|
|
26
26
|
dependencies = [
|
|
27
|
-
"nace-sdk>=0.
|
|
27
|
+
"nace-sdk>=0.2,<1",
|
|
28
28
|
"truststore>=0.10.4,<1",
|
|
29
29
|
]
|
|
30
30
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "nace-cli"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.2.0"
|
|
4
4
|
description = "Nace CLI: calibrated decisions and document jobs from the shell"
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Nace AI", email = "engineering@nace.ai" }
|
|
@@ -27,7 +27,7 @@ classifiers = [
|
|
|
27
27
|
"Typing :: Typed",
|
|
28
28
|
]
|
|
29
29
|
dependencies = [
|
|
30
|
-
"nace-sdk>=0.
|
|
30
|
+
"nace-sdk>=0.2,<1",
|
|
31
31
|
"truststore>=0.10.4,<1",
|
|
32
32
|
]
|
|
33
33
|
|
|
@@ -16,7 +16,8 @@ from nace_sdk import NaceAPIConnectionError, NaceAPIError, NaceClient, NaceError
|
|
|
16
16
|
from nace_sdk.constants import LOG_LEVEL_ENV, read_env
|
|
17
17
|
from nace_sdk.credentials import CredentialsError, resolve_credentials
|
|
18
18
|
|
|
19
|
-
from nace_cli import _decide, _docops, _jobs, _login, _schemas
|
|
19
|
+
from nace_cli import _decide, _docops, _jobs, _login, _schemas, _search, _upload, _workspaces
|
|
20
|
+
from nace_cli._inputs import check_one_stdin_flag
|
|
20
21
|
from nace_cli._output import CommandFailed
|
|
21
22
|
|
|
22
23
|
#: Long enough for a decision (Drex holds one up to 60 s) and for a document
|
|
@@ -46,6 +47,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
46
47
|
return 2
|
|
47
48
|
_log_to_stderr()
|
|
48
49
|
try:
|
|
50
|
+
check_one_stdin_flag(args)
|
|
49
51
|
if args.family == "standalone":
|
|
50
52
|
return args.run(args)
|
|
51
53
|
credentials = resolve_credentials()
|
|
@@ -93,6 +95,15 @@ def _validation_lines(exc: NaceAPIError) -> list[dict[str, Any]]:
|
|
|
93
95
|
return lines
|
|
94
96
|
|
|
95
97
|
|
|
98
|
+
#: What to do next for the workspace and search errors, by code.
|
|
99
|
+
_WORKSPACE_HINTS = {
|
|
100
|
+
"workspace_not_found": "`nace workspaces` lists the workspaces your key can use; `nace workspace-use ID` saves one.",
|
|
101
|
+
"session_busy": "The conversation's last turn is still running: wait for it with `nace job <id> --wait`, then ask again.",
|
|
102
|
+
"rate_limited": "Automatic searches and QA turns share a limit of 60 per account every 10 minutes.",
|
|
103
|
+
"access_denied": "Your key can use the workspace but not do this: ask an account owner or admin, or whoever created the workspace.",
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
|
|
96
107
|
def report_status_error(exc: NaceAPIError) -> None:
|
|
97
108
|
"""One line a person can act on, plus where a request failed validation."""
|
|
98
109
|
label = "/".join(str(part) for part in (exc.type, exc.code) if part)
|
|
@@ -112,6 +123,16 @@ def report_status_error(exc: NaceAPIError) -> None:
|
|
|
112
123
|
print(f" Pay the unpaid invoice at {billing}", file=sys.stderr)
|
|
113
124
|
else:
|
|
114
125
|
print(f" Top up or add a card at {billing}", file=sys.stderr)
|
|
126
|
+
# An older nace-sdk (>=0.1) has no `idempotency_key` on its errors.
|
|
127
|
+
resume_key = getattr(exc, "idempotency_key", None)
|
|
128
|
+
if exc.code == "job_pending" and resume_key:
|
|
129
|
+
# The job may have started and be billed; the same key and body resume it instead of creating another.
|
|
130
|
+
print(f" The job may still be running. Rerun the same command with --idempotency-key {resume_key} to resume it.", file=sys.stderr)
|
|
131
|
+
hint = _WORKSPACE_HINTS.get(exc.code or "")
|
|
132
|
+
if hint:
|
|
133
|
+
print(f" {hint}", file=sys.stderr)
|
|
134
|
+
if exc.code == "rate_limited" and (wait := exc.headers.get("retry-after")):
|
|
135
|
+
print(f" Run it again in {wait}s.", file=sys.stderr)
|
|
115
136
|
if exc.code == "path_conflict":
|
|
116
137
|
print(
|
|
117
138
|
" Upload to another path (--upload-path, or --path on `nace upload`), or pass --on-conflict new_version to add a version of that file.",
|
|
@@ -123,7 +144,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
123
144
|
parser = argparse.ArgumentParser(
|
|
124
145
|
prog="nace",
|
|
125
146
|
description=(
|
|
126
|
-
"Nace CLI: calibrated decisions
|
|
147
|
+
"Nace CLI: calibrated decisions, document jobs, workspaces and search. Credentials come from "
|
|
127
148
|
"$NACE_API_KEY / $NACE_BASE_URL or ~/.nace/config.toml (`nace login`). "
|
|
128
149
|
"Set $NACE_LOG_LEVEL=debug to log each request to stderr."
|
|
129
150
|
),
|
|
@@ -141,9 +162,12 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
141
162
|
sub = parser.add_subparsers(dest="command")
|
|
142
163
|
_login.add_parsers(sub)
|
|
143
164
|
_decide.add_parsers(sub, common)
|
|
165
|
+
_upload.add_parsers(sub, common)
|
|
144
166
|
_docops.add_parsers(sub, common)
|
|
145
167
|
_jobs.add_parsers(sub, common)
|
|
146
168
|
_schemas.add_parsers(sub, common)
|
|
169
|
+
_workspaces.add_parsers(sub, common)
|
|
170
|
+
_search.add_parsers(sub, common)
|
|
147
171
|
return parser
|
|
148
172
|
|
|
149
173
|
|
|
@@ -3,14 +3,27 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import argparse
|
|
6
|
+
import hashlib
|
|
6
7
|
import sys
|
|
8
|
+
from pathlib import Path
|
|
7
9
|
from typing import Any
|
|
8
10
|
|
|
9
|
-
from nace_sdk import Choice, ChoiceAnswer, ListModelsResponse, NaceClient, Noul, NoulAnswer, Score, ScoreAnswer, SystemOneResponse
|
|
11
|
+
from nace_sdk import Choice, ChoiceAnswer, ListModelsResponse, NaceClient, Noul, NoulAnswer, Score, ScoreAnswer, SystemOneResponse, image_input
|
|
12
|
+
from nace_sdk.images import _media_type
|
|
10
13
|
|
|
14
|
+
from nace_cli._docops import parse_ws_source, upload_file, workspace_source
|
|
11
15
|
from nace_cli._inputs import read_json, read_text
|
|
12
16
|
from nace_cli._output import to_json
|
|
13
17
|
|
|
18
|
+
#: The only model that reads images, and the one `--image` picks when no model is named.
|
|
19
|
+
IMAGE_MODEL = "drex-max"
|
|
20
|
+
#: Drex Max reads at most this many images per request.
|
|
21
|
+
MAX_IMAGES = 8
|
|
22
|
+
#: Local images up to this size are sent inline; larger ones are uploaded first (8 inline fit the 4 MiB body).
|
|
23
|
+
INLINE_IMAGE_BYTES = 256 * 1024
|
|
24
|
+
#: Where `--image` uploads larger images, named by their content.
|
|
25
|
+
IMAGE_UPLOAD_FOLDER = "drex-max-images"
|
|
26
|
+
|
|
14
27
|
|
|
15
28
|
def add_parsers(sub: argparse._SubParsersAction, common: argparse.ArgumentParser) -> None:
|
|
16
29
|
decide = sub.add_parser(
|
|
@@ -31,6 +44,18 @@ def add_parsers(sub: argparse._SubParsersAction, common: argparse.ArgumentParser
|
|
|
31
44
|
decide.add_argument("--score", action="append", default=[], metavar="NAME=LOW,...,HIGH[: INSTRUCTIONS]", help="Ordered levels.")
|
|
32
45
|
decide.add_argument("--questions", default=None, metavar="JSON", help="Questions object: inline JSON, @FILE or a path.")
|
|
33
46
|
decide.add_argument("--body", default=None, metavar="JSON", help="A whole request body; other flags override its fields.")
|
|
47
|
+
decide.add_argument(
|
|
48
|
+
"--image",
|
|
49
|
+
dest="images",
|
|
50
|
+
action="append",
|
|
51
|
+
default=[],
|
|
52
|
+
metavar="IMAGE",
|
|
53
|
+
help=(
|
|
54
|
+
f"An image for Drex Max (PNG, JPEG or WebP); repeatable, up to {MAX_IMAGES}, read in order as Image 1, Image 2... "
|
|
55
|
+
f"A local file up to {INLINE_IMAGE_BYTES // 1024} KiB is sent inline; a larger one is uploaded first. "
|
|
56
|
+
f"Sets --model {IMAGE_MODEL} unless you name one."
|
|
57
|
+
),
|
|
58
|
+
)
|
|
34
59
|
decide.add_argument("-m", "--model", default=None, help="Model id (default drex-latest). See `nace models`.")
|
|
35
60
|
decide.set_defaults(run=run_decide, family="client")
|
|
36
61
|
|
|
@@ -66,10 +91,52 @@ def build_request(args: argparse.Namespace) -> dict[str, Any]:
|
|
|
66
91
|
raise ValueError("--state is required (text, @FILE, or - for stdin)")
|
|
67
92
|
if args.model:
|
|
68
93
|
body["model"] = args.model
|
|
94
|
+
elif args.images and "model" not in body:
|
|
95
|
+
body["model"] = IMAGE_MODEL
|
|
96
|
+
if len(args.images) > MAX_IMAGES:
|
|
97
|
+
raise ValueError(f"at most {MAX_IMAGES} --image per request, got {len(args.images)}")
|
|
98
|
+
for spec in args.images:
|
|
99
|
+
_check_image(spec)
|
|
69
100
|
body["questions"] = questions
|
|
70
101
|
return body
|
|
71
102
|
|
|
72
103
|
|
|
104
|
+
def _check_image(spec: str) -> None:
|
|
105
|
+
"""Refuse a bad ``--image`` before anything is uploaded or sent."""
|
|
106
|
+
if spec.startswith("ws:"):
|
|
107
|
+
parse_ws_source(spec)
|
|
108
|
+
return
|
|
109
|
+
path = Path(spec)
|
|
110
|
+
if not path.is_file():
|
|
111
|
+
raise ValueError(f"--image: no such file, and not ws:<workspace>/<file>: {spec}")
|
|
112
|
+
with path.open("rb") as handle:
|
|
113
|
+
head = handle.read(12)
|
|
114
|
+
if _media_type(head) is None:
|
|
115
|
+
raise ValueError(f"--image: {spec} is not a PNG, JPEG or WebP image")
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def resolve_images(client: NaceClient, specs: list[str]) -> list[Any]:
|
|
119
|
+
"""Each ``--image`` as an ``images`` entry: ws: as a file reference, a small file inline, a larger one uploaded."""
|
|
120
|
+
images: list[Any] = []
|
|
121
|
+
for spec in specs:
|
|
122
|
+
if spec.startswith("ws:"):
|
|
123
|
+
workspace_id, file_id = parse_ws_source(spec)
|
|
124
|
+
images.append({"type": "workspace_file", "workspace_id": workspace_id, "file_id": file_id})
|
|
125
|
+
continue
|
|
126
|
+
path = Path(spec)
|
|
127
|
+
data = path.read_bytes()
|
|
128
|
+
if len(data) <= INLINE_IMAGE_BYTES:
|
|
129
|
+
images.append(image_input(data))
|
|
130
|
+
continue
|
|
131
|
+
# Named by content, so the same image again returns the file already there.
|
|
132
|
+
target = f"{IMAGE_UPLOAD_FOLDER}/{hashlib.sha256(data).hexdigest()[:16]}{path.suffix.lower()}"
|
|
133
|
+
print(f"uploading {path.name}", file=sys.stderr)
|
|
134
|
+
uploaded = upload_file(client, path, path=target, on_conflict=None)
|
|
135
|
+
print(f"uploaded {workspace_source(uploaded)}", file=sys.stderr)
|
|
136
|
+
images.append(image_input(uploaded))
|
|
137
|
+
return images
|
|
138
|
+
|
|
139
|
+
|
|
73
140
|
def _state(text: str) -> Any:
|
|
74
141
|
"""A state that parses as a JSON object or array is sent as JSON; anything else as text."""
|
|
75
142
|
stripped = text.strip()
|
|
@@ -99,14 +166,17 @@ def _labelled(spec: str, flag: str) -> tuple[str, list[str], str | None]:
|
|
|
99
166
|
|
|
100
167
|
def run_decide(client: NaceClient, args: argparse.Namespace) -> int:
|
|
101
168
|
body = build_request(args)
|
|
102
|
-
|
|
103
|
-
|
|
169
|
+
if args.images:
|
|
170
|
+
body["images"] = resolve_images(client, args.images)
|
|
171
|
+
extra = {key: value for key, value in body.items() if key not in ("state", "questions", "model", "images")}
|
|
172
|
+
response = client.system_one(body["state"], body["questions"], model=body.get("model"), images=body.get("images"), extra_body=extra or None)
|
|
104
173
|
if args.json:
|
|
105
174
|
print(response_json(response))
|
|
106
175
|
return 0
|
|
107
176
|
print(render_answers(response))
|
|
108
177
|
timing = f" · {response.evaluation_time_ms:.0f} ms" if response.evaluation_time_ms is not None else ""
|
|
109
|
-
|
|
178
|
+
image_tokens = f" ({response.usage.image_tokens} image)" if response.usage.image_tokens else ""
|
|
179
|
+
print(f"{response.model} · {response.usage.input_tokens} input tokens{image_tokens}{timing} · {response.request_id}", file=sys.stderr)
|
|
110
180
|
return 0
|
|
111
181
|
|
|
112
182
|
|
|
@@ -33,37 +33,40 @@ SOURCE_HELP = """\
|
|
|
33
33
|
SOURCE is one of:
|
|
34
34
|
https://... a public link the document service downloads
|
|
35
35
|
(--file-name names the file when the link doesn't end in one)
|
|
36
|
-
a local file uploaded
|
|
37
|
-
ws:<workspace_id>/<file_id> a file already
|
|
36
|
+
a local file uploaded to the server first, then used from there
|
|
37
|
+
ws:<workspace_id>/<file_id> a file already uploaded (what `nace upload` prints)
|
|
38
38
|
job:<job_id> a finished parse job's result, so it is not parsed again
|
|
39
39
|
(not for classify)
|
|
40
40
|
"""
|
|
41
41
|
|
|
42
42
|
ON_CONFLICT_CHOICES = ("reject", "new_version")
|
|
43
|
+
#: The most locations Ground returns per target.
|
|
44
|
+
MAX_MATCHES = 50
|
|
43
45
|
|
|
44
46
|
|
|
45
47
|
def add_parsers(sub: argparse._SubParsersAction, common: argparse.ArgumentParser) -> None:
|
|
46
|
-
|
|
47
|
-
upload.add_argument("file", help="Local file.")
|
|
48
|
-
upload.add_argument("--path", default=None, help="Path in the workspace (default: the file name).")
|
|
49
|
-
_add_on_conflict(upload)
|
|
50
|
-
upload.set_defaults(run=run_upload, family="client")
|
|
51
|
-
|
|
48
|
+
# `nace upload` itself is in _upload, which registers it just before these.
|
|
52
49
|
grant = sub.add_parser("upload-grant", parents=[common], help="Mint a one-use token another client can upload one file with.")
|
|
53
50
|
grant.add_argument("--path", default=None, help="Pin the workspace path the upload must use.")
|
|
54
51
|
grant.add_argument("--max-bytes", type=int, default=None, metavar="N", help="Largest file the grant accepts.")
|
|
55
52
|
grant.add_argument("--total-size-bytes", type=int, default=None, metavar="N", help="Exact size the file must have.")
|
|
56
53
|
grant.add_argument("--ttl-seconds", type=int, default=None, metavar="N", help="How long the grant stays valid.")
|
|
54
|
+
add_upload_workspace(grant, "The workspace the upload goes into")
|
|
57
55
|
grant.set_defaults(run=run_upload_grant, family="client")
|
|
58
56
|
|
|
59
57
|
session = sub.add_parser("upload-session", parents=[common], help="Open a resumable upload in parts for a large file.")
|
|
60
58
|
session.add_argument("--path", required=True, help="Path of the file in the workspace.")
|
|
61
59
|
session.add_argument("--total-size-bytes", type=int, required=True, metavar="N", help="The file's exact size.")
|
|
62
60
|
session.add_argument("--ttl-seconds", type=int, default=None, metavar="N", help="How long the session accepts parts.")
|
|
63
|
-
|
|
61
|
+
add_on_conflict(session)
|
|
64
62
|
session.add_argument("--idempotency-key", default=None, metavar="KEY", help="Retry-safe key; the same key returns the same session.")
|
|
63
|
+
add_upload_workspace(session, "The workspace the file goes into")
|
|
65
64
|
session.set_defaults(run=run_upload_session, family="client")
|
|
66
65
|
|
|
66
|
+
delete = sub.add_parser("upload-delete", parents=[common], help="Delete an uploaded file and everything derived from it.")
|
|
67
|
+
delete.add_argument("workspace_file", metavar="ws:<workspace_id>/<file_id>", help="The file, as `nace upload` printed it.")
|
|
68
|
+
delete.set_defaults(run=run_upload_delete, family="client")
|
|
69
|
+
|
|
67
70
|
parse = _operation(sub, common, "parse", "Convert a document to Markdown, text or layout blocks.")
|
|
68
71
|
parse.add_argument(
|
|
69
72
|
"--format",
|
|
@@ -71,6 +74,19 @@ def add_parsers(sub: argparse._SubParsersAction, common: argparse.ArgumentParser
|
|
|
71
74
|
default=None,
|
|
72
75
|
help="Comma list of markdown,text,blocks (default: the API's markdown,blocks; stdout shows the Markdown).",
|
|
73
76
|
)
|
|
77
|
+
parse.add_argument(
|
|
78
|
+
"--table-format", choices=("html", "markdown"), default=None, help="How tables are written in the Markdown and text (default html)."
|
|
79
|
+
)
|
|
80
|
+
parse.add_argument(
|
|
81
|
+
"--chunk", choices=("none", "page", "section"), default=None, help="Split the document into chunks for retrieval (default none)."
|
|
82
|
+
)
|
|
83
|
+
parse.add_argument(
|
|
84
|
+
"--figures",
|
|
85
|
+
choices=("omit", "include", "describe"),
|
|
86
|
+
default=None,
|
|
87
|
+
help="Drop figures, keep them, or keep and caption them (default include).",
|
|
88
|
+
)
|
|
89
|
+
parse.add_argument("--sheets", default=None, metavar="NAMES", help="A workbook's sheets to parse, a comma list of names (default: every sheet).")
|
|
74
90
|
parse.add_argument(
|
|
75
91
|
"--password",
|
|
76
92
|
default=None,
|
|
@@ -81,11 +97,21 @@ def add_parsers(sub: argparse._SubParsersAction, common: argparse.ArgumentParser
|
|
|
81
97
|
|
|
82
98
|
split = _operation(sub, common, "split", "Split a multi-document file into the documents it contains.")
|
|
83
99
|
_add_class_args(split)
|
|
100
|
+
split.add_argument(
|
|
101
|
+
"--unknown",
|
|
102
|
+
choices=("include", "force", "error"),
|
|
103
|
+
default=None,
|
|
104
|
+
help="Pages no class matches: keep them unclassified, give them the best class, or fail (default include).",
|
|
105
|
+
)
|
|
106
|
+
split.add_argument("--include-content", action="store_true", help="Give each segment its Markdown.")
|
|
84
107
|
split.set_defaults(run=run_split)
|
|
85
108
|
|
|
86
109
|
classify = _operation(sub, common, "classify", "Classify a document, or each page, into your classes.")
|
|
87
110
|
_add_class_args(classify)
|
|
88
111
|
classify.add_argument("--granularity", choices=("document", "page"), default=None, help="One label for the file, or one per page.")
|
|
112
|
+
classify.add_argument(
|
|
113
|
+
"--unknown", choices=("allow", "force_best"), default=None, help="Allow an unknown label, or force the closest class (default allow)."
|
|
114
|
+
)
|
|
89
115
|
classify.set_defaults(run=run_classify)
|
|
90
116
|
|
|
91
117
|
extract = _operation(sub, common, "extract", "Extract structured fields with a JSON Schema, inline or saved.")
|
|
@@ -94,11 +120,14 @@ def add_parsers(sub: argparse._SubParsersAction, common: argparse.ArgumentParser
|
|
|
94
120
|
schema.add_argument("--schema-id", default=None, metavar="ID", help="A saved schema (sch_...), at its latest version.")
|
|
95
121
|
extract.add_argument("--schema-version", type=int, default=None, metavar="N", help="With --schema-id, pin this version.")
|
|
96
122
|
extract.add_argument("--instructions", default=None, help="Extra guidance for the extraction.")
|
|
123
|
+
extract.add_argument("--no-citations", action="store_true", help="Don't say where each field was read (citations.enabled false).")
|
|
97
124
|
extract.set_defaults(run=run_extract)
|
|
98
125
|
|
|
99
126
|
ground = _operation(sub, common, "ground", "Find where each target text appears in a document.", pages=False)
|
|
100
127
|
ground.add_argument("--target", action="append", dest="targets", default=[], metavar="[ID=]TEXT", help="Repeatable; ids default to t1, t2, ...")
|
|
101
128
|
ground.add_argument("--targets", dest="targets_file", default=None, metavar="JSON", help="JSON array of targets.")
|
|
129
|
+
ground.add_argument("--max-matches", type=int, default=None, metavar="N", help=f"Locations per target, 1 to {MAX_MATCHES} (default 10).")
|
|
130
|
+
ground.add_argument("--previews", action="store_true", help="Link a cropped image of each visual hit.")
|
|
102
131
|
ground.add_argument(
|
|
103
132
|
"--semantic",
|
|
104
133
|
action="store_true",
|
|
@@ -107,7 +136,18 @@ def add_parsers(sub: argparse._SubParsersAction, common: argparse.ArgumentParser
|
|
|
107
136
|
ground.set_defaults(run=run_ground)
|
|
108
137
|
|
|
109
138
|
|
|
110
|
-
def
|
|
139
|
+
def add_upload_workspace(parser: argparse.ArgumentParser, what: str) -> None:
|
|
140
|
+
# Only the flag: an upload without it goes to the server, whatever
|
|
141
|
+
# $NACE_WORKSPACE_ID or `nace workspace-use` say, as uploads did before workspaces.
|
|
142
|
+
parser.add_argument(
|
|
143
|
+
"--workspace",
|
|
144
|
+
default=None,
|
|
145
|
+
metavar="ID",
|
|
146
|
+
help=f"{what}. Without it, the file is uploaded to the server; $NACE_WORKSPACE_ID and `nace workspace-use` don't apply.",
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def add_on_conflict(parser: argparse.ArgumentParser) -> None:
|
|
111
151
|
# Unset means the document service's reject, for a local SOURCE too: a job reads its workspace file's
|
|
112
152
|
# current version when it starts, not when it is created, so new_version changes the input of jobs
|
|
113
153
|
# still queued on that file.
|
|
@@ -129,7 +169,7 @@ def _operation(sub: argparse._SubParsersAction, common: argparse.ArgumentParser,
|
|
|
129
169
|
parser.add_argument("source", help="https:// URL, local file, ws:<workspace>/<file>, or job:<parse job id>.")
|
|
130
170
|
parser.add_argument("--file-name", default=None, metavar="NAME", help="For an https:// SOURCE: the file's name; its extension picks the parser.")
|
|
131
171
|
parser.add_argument("--upload-path", default=None, metavar="PATH", help="For a local SOURCE: its path in the workspace (default: its name).")
|
|
132
|
-
|
|
172
|
+
add_on_conflict(parser)
|
|
133
173
|
if pages:
|
|
134
174
|
# Ground reads whole documents: it has no page selection.
|
|
135
175
|
parser.add_argument("-p", "--pages", default=None, metavar="SPEC", help="1-based pages, e.g. 3 or 1-5,8 (not with a job: SOURCE).")
|
|
@@ -205,10 +245,8 @@ def resolve_source(client: NaceClient, args: argparse.Namespace) -> Source:
|
|
|
205
245
|
if spec.startswith("job:"):
|
|
206
246
|
return ParseResultSource(job_id=_uuid(spec[4:], spec))
|
|
207
247
|
if spec.startswith("ws:"):
|
|
208
|
-
workspace_id,
|
|
209
|
-
|
|
210
|
-
raise ValueError(f"expected ws:<workspace_id>/<file_id>, got {spec!r}")
|
|
211
|
-
return WorkspaceFileSource(workspace_id=_uuid(workspace_id, spec), file_id=_uuid(file_id, spec))
|
|
248
|
+
workspace_id, file_id = parse_ws_source(spec)
|
|
249
|
+
return WorkspaceFileSource(workspace_id=workspace_id, file_id=file_id)
|
|
212
250
|
path = Path(spec)
|
|
213
251
|
if not path.is_file():
|
|
214
252
|
raise ValueError(f"no such file, and not a URL, ws: or job: source: {spec}")
|
|
@@ -218,9 +256,17 @@ def resolve_source(client: NaceClient, args: argparse.Namespace) -> Source:
|
|
|
218
256
|
return uploaded.as_source()
|
|
219
257
|
|
|
220
258
|
|
|
221
|
-
def
|
|
259
|
+
def parse_ws_source(spec: str) -> tuple[str, str]:
|
|
260
|
+
"""``ws:<workspace_id>/<file_id>`` as its two ids, refused as a usage error unless both are ids."""
|
|
261
|
+
workspace_id, sep, file_id = spec.removeprefix("ws:").partition("/")
|
|
262
|
+
if not spec.startswith("ws:") or not sep:
|
|
263
|
+
raise ValueError(f"expected ws:<workspace_id>/<file_id>, got {spec!r}")
|
|
264
|
+
return _uuid(workspace_id, spec), _uuid(file_id, spec)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def upload_file(client: NaceClient, file: Path, *, path: str | None, on_conflict: OnConflict | None, workspace_id: str | None = None) -> UploadedFile:
|
|
222
268
|
try:
|
|
223
|
-
return client.documents.upload(file, path=path, on_conflict=on_conflict)
|
|
269
|
+
return client.documents.upload(file, path=path, on_conflict=on_conflict, workspace_id=workspace_id)
|
|
224
270
|
except NaceAPIError:
|
|
225
271
|
raise
|
|
226
272
|
except NaceError as exc:
|
|
@@ -256,6 +302,17 @@ def _body(args: argparse.Namespace) -> dict[str, Any]:
|
|
|
256
302
|
return body
|
|
257
303
|
|
|
258
304
|
|
|
305
|
+
def _set(body: dict[str, Any], key: str, value: Any) -> None:
|
|
306
|
+
"""Set a dotted request field from its typed flag, over whatever ``--options``/``--option`` gave, keeping the block's other fields."""
|
|
307
|
+
*parents, leaf = key.split(".")
|
|
308
|
+
target = body
|
|
309
|
+
for part in parents:
|
|
310
|
+
child = target.get(part)
|
|
311
|
+
target[part] = child = dict(child) if isinstance(child, dict) else {}
|
|
312
|
+
target = child
|
|
313
|
+
target[leaf] = value
|
|
314
|
+
|
|
315
|
+
|
|
259
316
|
def _take(body: dict[str, Any], key: str, flag: Any, kind: type) -> Any:
|
|
260
317
|
"""A value the SDK takes as an argument: the flag's, else the body field's (which is removed either way)."""
|
|
261
318
|
value = body.pop(key, None)
|
|
@@ -308,21 +365,30 @@ def _submit(client: NaceClient, args: argparse.Namespace, body: dict[str, Any],
|
|
|
308
365
|
return submit_and_report(client, job, args)
|
|
309
366
|
|
|
310
367
|
|
|
311
|
-
def
|
|
312
|
-
|
|
313
|
-
if not
|
|
314
|
-
raise ValueError(f"
|
|
315
|
-
|
|
368
|
+
def run_upload_delete(client: NaceClient, args: argparse.Namespace) -> int:
|
|
369
|
+
spec = args.workspace_file
|
|
370
|
+
if not spec.startswith("ws:"):
|
|
371
|
+
raise ValueError(f"expected ws:<workspace_id>/<file_id>, got {spec!r}")
|
|
372
|
+
workspace_id, sep, file_id = spec[3:].partition("/")
|
|
373
|
+
if not sep:
|
|
374
|
+
raise ValueError(f"expected ws:<workspace_id>/<file_id>, got {spec!r}")
|
|
375
|
+
deletion = client.documents.delete_file(_uuid(workspace_id, spec), _uuid(file_id, spec))
|
|
316
376
|
if args.json:
|
|
317
|
-
print(
|
|
377
|
+
print(deletion.model_dump_json(indent=2))
|
|
378
|
+
if deletion.status == "deleted":
|
|
379
|
+
print(f"deleted {spec}", file=sys.stderr)
|
|
318
380
|
else:
|
|
319
|
-
print(
|
|
381
|
+
print(f"still deleting {spec}: run the command again to check", file=sys.stderr)
|
|
320
382
|
return 0
|
|
321
383
|
|
|
322
384
|
|
|
323
385
|
def run_upload_grant(client: NaceClient, args: argparse.Namespace) -> int:
|
|
324
386
|
grant = client.documents.create_upload_grant(
|
|
325
|
-
path=args.path,
|
|
387
|
+
path=args.path,
|
|
388
|
+
max_bytes=args.max_bytes,
|
|
389
|
+
total_size_bytes=args.total_size_bytes,
|
|
390
|
+
ttl_seconds=args.ttl_seconds,
|
|
391
|
+
workspace_id=args.workspace,
|
|
326
392
|
)
|
|
327
393
|
print(grant.model_dump_json(indent=2))
|
|
328
394
|
print("the token uploads one file into your workspace: hand it only to whoever uploads that file", file=sys.stderr)
|
|
@@ -335,6 +401,7 @@ def run_upload_session(client: NaceClient, args: argparse.Namespace) -> int:
|
|
|
335
401
|
total_size_bytes=args.total_size_bytes,
|
|
336
402
|
ttl_seconds=args.ttl_seconds,
|
|
337
403
|
on_conflict=args.on_conflict,
|
|
404
|
+
workspace_id=args.workspace,
|
|
338
405
|
idempotency_key=args.idempotency_key,
|
|
339
406
|
)
|
|
340
407
|
print(to_json(session.model_dump(mode="json")))
|
|
@@ -348,6 +415,14 @@ def run_parse(client: NaceClient, args: argparse.Namespace) -> int:
|
|
|
348
415
|
output = body.get("output")
|
|
349
416
|
formats = [item.strip() for item in args.formats.split(",") if item.strip()]
|
|
350
417
|
body["output"] = {**(output if isinstance(output, dict) else {}), "formats": formats}
|
|
418
|
+
for key, value in (("output.table_format", args.table_format), ("chunking.strategy", args.chunk), ("figures.mode", args.figures)):
|
|
419
|
+
if value is not None:
|
|
420
|
+
_set(body, key, value)
|
|
421
|
+
if args.sheets is not None:
|
|
422
|
+
sheets = [name.strip() for name in args.sheets.split(",") if name.strip()]
|
|
423
|
+
if not sheets:
|
|
424
|
+
raise ValueError("--sheets takes a comma list of sheet names")
|
|
425
|
+
_set(body, "spreadsheet.sheets", sheets)
|
|
351
426
|
if args.password is not None:
|
|
352
427
|
body["password"] = read_secret(args.password, prompt="PDF password: ")
|
|
353
428
|
return _submit(client, args, body)
|
|
@@ -355,6 +430,10 @@ def run_parse(client: NaceClient, args: argparse.Namespace) -> int:
|
|
|
355
430
|
|
|
356
431
|
def run_split(client: NaceClient, args: argparse.Namespace) -> int:
|
|
357
432
|
body = _body(args)
|
|
433
|
+
if args.unknown is not None:
|
|
434
|
+
body["unknown_policy"] = args.unknown
|
|
435
|
+
if args.include_content:
|
|
436
|
+
_set(body, "output.include_content", True)
|
|
358
437
|
return _submit(client, args, body, classes=_classes(args, body))
|
|
359
438
|
|
|
360
439
|
|
|
@@ -362,6 +441,8 @@ def run_classify(client: NaceClient, args: argparse.Namespace) -> int:
|
|
|
362
441
|
body = _body(args)
|
|
363
442
|
if args.granularity:
|
|
364
443
|
body["granularity"] = args.granularity
|
|
444
|
+
if args.unknown is not None:
|
|
445
|
+
body["unknown_policy"] = args.unknown
|
|
365
446
|
return _submit(client, args, body, classes=_classes(args, body))
|
|
366
447
|
|
|
367
448
|
|
|
@@ -387,11 +468,19 @@ def run_extract(client: NaceClient, args: argparse.Namespace) -> int:
|
|
|
387
468
|
if fields["schema_version"] is not None and fields["schema_id"] is None:
|
|
388
469
|
raise ValueError("schema_version pins a version of a saved schema, so it needs schema_id")
|
|
389
470
|
instructions = _take(body, "instructions", args.instructions, str)
|
|
471
|
+
if args.no_citations:
|
|
472
|
+
_set(body, "citations.enabled", False)
|
|
390
473
|
return _submit(client, args, body, instructions=instructions, **fields)
|
|
391
474
|
|
|
392
475
|
|
|
393
476
|
def run_ground(client: NaceClient, args: argparse.Namespace) -> int:
|
|
477
|
+
if args.max_matches is not None and not 1 <= args.max_matches <= MAX_MATCHES:
|
|
478
|
+
raise ValueError(f"--max-matches must be between 1 and {MAX_MATCHES}")
|
|
394
479
|
body = _body(args)
|
|
480
|
+
if args.max_matches is not None:
|
|
481
|
+
_set(body, "options.max_matches", args.max_matches)
|
|
482
|
+
if args.previews:
|
|
483
|
+
_set(body, "options.include_previews", True)
|
|
395
484
|
targets: list[Any] = []
|
|
396
485
|
if args.targets_file:
|
|
397
486
|
loaded = read_json(args.targets_file, what="--targets")
|