entitybind 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- entitybind/__init__.py +8 -0
- entitybind/__main__.py +3 -0
- entitybind/cli.py +510 -0
- entitybind/commands_api.py +900 -0
- entitybind/commands_auth.py +393 -0
- entitybind/commands_mcp.py +236 -0
- entitybind/context.py +100 -0
- entitybind/credentials.py +349 -0
- entitybind/device.py +433 -0
- entitybind/errors.py +184 -0
- entitybind/files.py +78 -0
- entitybind/http.py +319 -0
- entitybind/mcpconfig.py +778 -0
- entitybind/products.py +126 -0
- entitybind/py.typed +0 -0
- entitybind/records.py +234 -0
- entitybind/runtime.py +163 -0
- entitybind/submissions.py +83 -0
- entitybind-0.1.0.dist-info/METADATA +251 -0
- entitybind-0.1.0.dist-info/RECORD +23 -0
- entitybind-0.1.0.dist-info/WHEEL +4 -0
- entitybind-0.1.0.dist-info/entry_points.txt +2 -0
- entitybind-0.1.0.dist-info/licenses/LICENSE +21 -0
entitybind/products.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""The two EntityBind products this client talks to, and where they live."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from urllib.parse import urlsplit
|
|
8
|
+
|
|
9
|
+
from entitybind import __version__
|
|
10
|
+
from entitybind.errors import UsageError
|
|
11
|
+
|
|
12
|
+
USER_AGENT = f"entitybind-cli/{__version__}"
|
|
13
|
+
CLIENT_HEADER = "cli"
|
|
14
|
+
CLIENT_ID = "entitybind-cli"
|
|
15
|
+
|
|
16
|
+
_LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"})
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class Product:
|
|
21
|
+
name: str
|
|
22
|
+
label: str
|
|
23
|
+
default_base_url: str
|
|
24
|
+
base_url_env: str
|
|
25
|
+
key_env: str
|
|
26
|
+
api_prefix: str
|
|
27
|
+
mcp_path: str
|
|
28
|
+
server_name: str
|
|
29
|
+
supports_mcp_oauth: bool
|
|
30
|
+
|
|
31
|
+
def api_path(self, suffix: str) -> str:
|
|
32
|
+
return f"{self.api_prefix}{suffix}"
|
|
33
|
+
|
|
34
|
+
def mcp_url(self, base_url: str) -> str:
|
|
35
|
+
return f"{base_url}{self.mcp_path}"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
CORE = Product(
|
|
39
|
+
name="core",
|
|
40
|
+
label="Entity Bind - Core",
|
|
41
|
+
default_base_url="https://app.entitybind.com",
|
|
42
|
+
base_url_env="ENTITYBIND_BASE_URL",
|
|
43
|
+
key_env="ENTITYBIND_API_KEY",
|
|
44
|
+
api_prefix="/v1",
|
|
45
|
+
mcp_path="/mcp",
|
|
46
|
+
server_name="entitybind",
|
|
47
|
+
supports_mcp_oauth=True,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
AML = Product(
|
|
51
|
+
name="aml",
|
|
52
|
+
label="EntityBind AML Onboard",
|
|
53
|
+
default_base_url="https://onboard.entitybind.com",
|
|
54
|
+
base_url_env="ENTITYBIND_AML_BASE_URL",
|
|
55
|
+
key_env="ENTITYBIND_AML_KEY",
|
|
56
|
+
api_prefix="/aml/v1",
|
|
57
|
+
mcp_path="/aml/mcp",
|
|
58
|
+
server_name="entitybind-aml",
|
|
59
|
+
supports_mcp_oauth=False,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
PRODUCTS: dict[str, Product] = {CORE.name: CORE, AML.name: AML}
|
|
63
|
+
|
|
64
|
+
# Admin-scoped key used by `keys` commands when set, so a person can keep a
|
|
65
|
+
# resolve key for everyday work and an admin key only for key management.
|
|
66
|
+
ADMIN_KEY_ENV = "ENTITYBIND_ADMIN_KEY"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def product_named(name: str) -> Product:
|
|
70
|
+
try:
|
|
71
|
+
return PRODUCTS[name]
|
|
72
|
+
except KeyError:
|
|
73
|
+
raise UsageError(f"unknown product {name!r}; choose core or aml") from None
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def normalize_base_url(raw: str) -> str:
|
|
77
|
+
"""Validate and canonicalize a base URL.
|
|
78
|
+
|
|
79
|
+
Keys travel in the Authorization header, so plain http is refused except
|
|
80
|
+
on a loopback host used for local development.
|
|
81
|
+
"""
|
|
82
|
+
value = raw.strip().rstrip("/")
|
|
83
|
+
parts = urlsplit(value)
|
|
84
|
+
if parts.scheme not in ("https", "http") or not parts.hostname:
|
|
85
|
+
raise UsageError(f"base URL must be an absolute https URL, got {raw!r}")
|
|
86
|
+
if parts.scheme == "http" and parts.hostname not in _LOOPBACK_HOSTS:
|
|
87
|
+
raise UsageError(
|
|
88
|
+
f"refusing plain http for {parts.hostname}: credentials would travel unencrypted; "
|
|
89
|
+
"use https (http is allowed only for localhost)"
|
|
90
|
+
)
|
|
91
|
+
if parts.query or parts.fragment or parts.username or parts.password:
|
|
92
|
+
raise UsageError("base URL must not carry credentials, a query or a fragment")
|
|
93
|
+
try:
|
|
94
|
+
port = parts.port
|
|
95
|
+
except ValueError:
|
|
96
|
+
raise UsageError(f"base URL has an invalid port: {raw!r}") from None
|
|
97
|
+
host = parts.hostname.lower()
|
|
98
|
+
netloc = f"[{host}]" if ":" in host else host
|
|
99
|
+
# Default ports are dropped, as WHATWG URL parsing (the npm client) does, so
|
|
100
|
+
# both clients store a credential under the same base URL.
|
|
101
|
+
if port is not None and port != {"https": 443, "http": 80}[parts.scheme]:
|
|
102
|
+
netloc = f"{netloc}:{port}"
|
|
103
|
+
return f"{parts.scheme}://{netloc}{parts.path.rstrip('/')}"
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def is_safe_browser_url(url: str) -> bool:
|
|
107
|
+
"""Only https links (or http on a loopback host) are opened in a browser.
|
|
108
|
+
|
|
109
|
+
The verification link comes from the server; a file: or custom-scheme link
|
|
110
|
+
must never reach the operating system's URL handler.
|
|
111
|
+
"""
|
|
112
|
+
try:
|
|
113
|
+
parts = urlsplit(url)
|
|
114
|
+
except ValueError:
|
|
115
|
+
return False
|
|
116
|
+
if parts.scheme == "https" and parts.hostname:
|
|
117
|
+
return True
|
|
118
|
+
return parts.scheme == "http" and parts.hostname in _LOOPBACK_HOSTS
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def resolve_base_url(product: Product, flag: str | None, env: Mapping[str, str]) -> str:
|
|
122
|
+
"""The --base-url flag wins, then the product's environment variable."""
|
|
123
|
+
for candidate in (flag, env.get(product.base_url_env)):
|
|
124
|
+
if candidate and candidate.strip():
|
|
125
|
+
return normalize_base_url(candidate)
|
|
126
|
+
return product.default_base_url
|
entitybind/py.typed
ADDED
|
File without changes
|
entitybind/records.py
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""Company records from a CSV or JSON file, in the shape `/v1/batch-jobs` takes.
|
|
2
|
+
|
|
3
|
+
The column mapping mirrors the in-repository `resolvd import-csv`: `--map
|
|
4
|
+
field=Column` pairs (repeatable, comma-separable), and columns already named
|
|
5
|
+
after a record field are mapped automatically. Address parts are mapped one
|
|
6
|
+
at a time as `address.<part>`.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import csv
|
|
12
|
+
import io
|
|
13
|
+
import json
|
|
14
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any, TextIO
|
|
18
|
+
|
|
19
|
+
from entitybind.errors import UsageError
|
|
20
|
+
|
|
21
|
+
RECORD_FIELDS: tuple[str, ...] = (
|
|
22
|
+
"name",
|
|
23
|
+
"website",
|
|
24
|
+
"domain",
|
|
25
|
+
"email",
|
|
26
|
+
"phone",
|
|
27
|
+
"registration_number",
|
|
28
|
+
"lei",
|
|
29
|
+
"vat_number",
|
|
30
|
+
"duns",
|
|
31
|
+
"isin",
|
|
32
|
+
"ticker",
|
|
33
|
+
"jurisdiction",
|
|
34
|
+
"country",
|
|
35
|
+
"exchange",
|
|
36
|
+
"context",
|
|
37
|
+
"legal_form",
|
|
38
|
+
"parent_name",
|
|
39
|
+
)
|
|
40
|
+
ADDRESS_PREFIX = "address."
|
|
41
|
+
ADDRESS_FIELDS: tuple[str, ...] = ("line1", "line2", "city", "region", "postal_code", "country")
|
|
42
|
+
MAPPABLE_FIELDS: tuple[str, ...] = (
|
|
43
|
+
*RECORD_FIELDS,
|
|
44
|
+
*(f"{ADDRESS_PREFIX}{part}" for part in ADDRESS_FIELDS),
|
|
45
|
+
)
|
|
46
|
+
DEPTHS = ("probabilistic", "adjudication", "research")
|
|
47
|
+
# Core's default max_batch_size; a job above a deployment's own limit is refused
|
|
48
|
+
# with 422 batch_job_size. 5,000 is the most any deployment may configure.
|
|
49
|
+
DEFAULT_CHUNK_SIZE = 500
|
|
50
|
+
MAX_RECORDS_PER_JOB = 5_000
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass
|
|
54
|
+
class LoadedRecords:
|
|
55
|
+
records: list[dict[str, Any]]
|
|
56
|
+
skipped: list[dict[str, Any]] = field(default_factory=list)
|
|
57
|
+
mapping: dict[str, str] = field(default_factory=dict)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def parse_mapping(specs: Sequence[str]) -> dict[str, str]:
|
|
61
|
+
mapping: dict[str, str] = {}
|
|
62
|
+
for spec in specs:
|
|
63
|
+
for pair in spec.split(","):
|
|
64
|
+
if not pair.strip():
|
|
65
|
+
continue
|
|
66
|
+
field_name, _, column = pair.partition("=")
|
|
67
|
+
field_name = field_name.strip()
|
|
68
|
+
if not column.strip():
|
|
69
|
+
raise UsageError(f"--map expects field=column, got {pair.strip()!r}")
|
|
70
|
+
if field_name not in MAPPABLE_FIELDS:
|
|
71
|
+
raise UsageError(
|
|
72
|
+
f"unknown record field {field_name!r}; choose from {', '.join(MAPPABLE_FIELDS)}"
|
|
73
|
+
)
|
|
74
|
+
mapping[field_name] = column.strip()
|
|
75
|
+
return mapping
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _normalize_header(column: str) -> str:
|
|
79
|
+
return "".join(ch if ch.isalnum() else "_" for ch in column.strip().lower())
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def auto_mapping(header: Iterable[str]) -> dict[str, str]:
|
|
83
|
+
normalized = {_normalize_header(column): column for column in header}
|
|
84
|
+
return {
|
|
85
|
+
name: normalized[_normalize_header(name)]
|
|
86
|
+
for name in MAPPABLE_FIELDS
|
|
87
|
+
if _normalize_header(name) in normalized
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def row_to_record(row: Mapping[str, Any], mapping: dict[str, str]) -> dict[str, Any]:
|
|
92
|
+
record: dict[str, Any] = {}
|
|
93
|
+
address: dict[str, str] = {}
|
|
94
|
+
for field_name, column in mapping.items():
|
|
95
|
+
value = (row.get(column) or "").strip()
|
|
96
|
+
if not value:
|
|
97
|
+
continue
|
|
98
|
+
if field_name.startswith(ADDRESS_PREFIX):
|
|
99
|
+
address[field_name[len(ADDRESS_PREFIX) :]] = value
|
|
100
|
+
else:
|
|
101
|
+
record[field_name] = value
|
|
102
|
+
if address:
|
|
103
|
+
record["address"] = address
|
|
104
|
+
return record
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _refusal(record: dict[str, Any]) -> str | None:
|
|
108
|
+
if not str(record.get("name") or "").strip():
|
|
109
|
+
return "missing name"
|
|
110
|
+
if not str(record.get("country") or "").strip():
|
|
111
|
+
return "missing country (use --default-country)"
|
|
112
|
+
return None
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def load_csv(
|
|
116
|
+
stream: TextIO,
|
|
117
|
+
*,
|
|
118
|
+
maps: Sequence[str],
|
|
119
|
+
default_country: str | None,
|
|
120
|
+
key_column: str | None,
|
|
121
|
+
depth: str | None,
|
|
122
|
+
) -> LoadedRecords:
|
|
123
|
+
reader = csv.DictReader(stream)
|
|
124
|
+
header = list(reader.fieldnames or [])
|
|
125
|
+
if not header:
|
|
126
|
+
raise UsageError("the CSV file has no header row")
|
|
127
|
+
explicit = parse_mapping(maps)
|
|
128
|
+
# A column the caller mapped explicitly is not also guessed into another field.
|
|
129
|
+
automatic = {
|
|
130
|
+
name: column
|
|
131
|
+
for name, column in auto_mapping(header).items()
|
|
132
|
+
if column not in explicit.values()
|
|
133
|
+
}
|
|
134
|
+
mapping = {**automatic, **explicit}
|
|
135
|
+
missing = sorted(column for column in mapping.values() if column not in header)
|
|
136
|
+
if missing:
|
|
137
|
+
raise UsageError(f"--map names columns that are not in the file: {', '.join(missing)}")
|
|
138
|
+
if not mapping:
|
|
139
|
+
raise UsageError(
|
|
140
|
+
"no columns map to a record field; use --map, e.g. --map name='Company Name'"
|
|
141
|
+
)
|
|
142
|
+
if key_column is not None and key_column not in header:
|
|
143
|
+
raise UsageError(f"--key-column {key_column!r} is not a column in the file")
|
|
144
|
+
loaded = LoadedRecords(records=[], mapping=mapping)
|
|
145
|
+
for line_number, row in enumerate(reader, start=2):
|
|
146
|
+
record = row_to_record(row, mapping)
|
|
147
|
+
if default_country and not record.get("country"):
|
|
148
|
+
record["country"] = default_country
|
|
149
|
+
if key_column is not None:
|
|
150
|
+
ref = (row.get(key_column) or "").strip()
|
|
151
|
+
if ref:
|
|
152
|
+
record["client_ref"] = ref
|
|
153
|
+
if depth:
|
|
154
|
+
record["depth"] = depth
|
|
155
|
+
reason = _refusal(record)
|
|
156
|
+
if reason is not None:
|
|
157
|
+
loaded.skipped.append({"line": line_number, "reason": reason})
|
|
158
|
+
continue
|
|
159
|
+
loaded.records.append(record)
|
|
160
|
+
return loaded
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _json_records(value: Any, source: str) -> list[dict[str, Any]]:
|
|
164
|
+
if isinstance(value, dict) and isinstance(value.get("records"), list):
|
|
165
|
+
value = value["records"]
|
|
166
|
+
elif isinstance(value, dict) and "name" in value:
|
|
167
|
+
value = [value]
|
|
168
|
+
if not isinstance(value, list) or not all(isinstance(item, dict) for item in value):
|
|
169
|
+
raise UsageError(f"{source} must hold a JSON list of records or an object with 'records'")
|
|
170
|
+
return [dict(item) for item in value]
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def load_json_records(text: str, source: str, *, depth: str | None) -> LoadedRecords:
|
|
174
|
+
stripped = text.strip()
|
|
175
|
+
if not stripped:
|
|
176
|
+
raise UsageError(f"{source} is empty")
|
|
177
|
+
if stripped.startswith("[") or stripped.startswith("{"):
|
|
178
|
+
try:
|
|
179
|
+
records = _json_records(json.loads(stripped), source)
|
|
180
|
+
except ValueError:
|
|
181
|
+
records = _json_lines(stripped, source)
|
|
182
|
+
else:
|
|
183
|
+
raise UsageError(f"{source} is not JSON")
|
|
184
|
+
if depth:
|
|
185
|
+
for record in records:
|
|
186
|
+
record.setdefault("depth", depth)
|
|
187
|
+
return LoadedRecords(records=records)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _json_lines(text: str, source: str) -> list[dict[str, Any]]:
|
|
191
|
+
records: list[dict[str, Any]] = []
|
|
192
|
+
for number, line in enumerate(io.StringIO(text), start=1):
|
|
193
|
+
if not line.strip():
|
|
194
|
+
continue
|
|
195
|
+
try:
|
|
196
|
+
item = json.loads(line)
|
|
197
|
+
except ValueError:
|
|
198
|
+
raise UsageError(f"{source} line {number} is not valid JSON") from None
|
|
199
|
+
if not isinstance(item, dict):
|
|
200
|
+
raise UsageError(f"{source} line {number} is not a JSON object")
|
|
201
|
+
records.append(item)
|
|
202
|
+
return records
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def not_utf8(source: str) -> UsageError:
|
|
206
|
+
return UsageError(
|
|
207
|
+
f"{source} is not UTF-8 text",
|
|
208
|
+
code="input_not_utf8",
|
|
209
|
+
hint="save it as UTF-8 (in Excel: CSV UTF-8) and try again",
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def unreadable(source: str, error: OSError) -> UsageError:
|
|
214
|
+
return UsageError(f"cannot read {source}: {error.strerror or error}", code="input_unreadable")
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def read_text(path: str, stdin: TextIO) -> str:
|
|
218
|
+
try:
|
|
219
|
+
if path == "-":
|
|
220
|
+
return stdin.read()
|
|
221
|
+
file_path = Path(path)
|
|
222
|
+
if not file_path.is_file():
|
|
223
|
+
raise UsageError(f"file not found: {path}")
|
|
224
|
+
return file_path.read_text(encoding="utf-8-sig")
|
|
225
|
+
except UnicodeDecodeError:
|
|
226
|
+
raise not_utf8("standard input" if path == "-" else path) from None
|
|
227
|
+
except OSError as error:
|
|
228
|
+
raise unreadable(path, error) from None
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def chunked(records: list[dict[str, Any]], size: int) -> list[list[dict[str, Any]]]:
|
|
232
|
+
if size < 1 or size > MAX_RECORDS_PER_JOB:
|
|
233
|
+
raise UsageError(f"--chunk-size must be between 1 and {MAX_RECORDS_PER_JOB}")
|
|
234
|
+
return [records[start : start + size] for start in range(0, len(records), size)]
|
entitybind/runtime.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""Process-level dependencies and output, injectable for tests."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import shutil
|
|
8
|
+
import subprocess
|
|
9
|
+
import sys
|
|
10
|
+
import time
|
|
11
|
+
import webbrowser
|
|
12
|
+
from collections.abc import Callable, Mapping
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any, TextIO
|
|
16
|
+
|
|
17
|
+
import httpx
|
|
18
|
+
|
|
19
|
+
from entitybind.credentials import KeyringLike, load_system_keyring
|
|
20
|
+
from entitybind.errors import ApiError, CliError
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class CommandResult:
|
|
25
|
+
returncode: int
|
|
26
|
+
stdout: str = ""
|
|
27
|
+
stderr: str = ""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
SUBPROCESS_TIMEOUT = 120
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def run_subprocess(argv: list[str]) -> CommandResult:
|
|
34
|
+
try:
|
|
35
|
+
completed = subprocess.run( # noqa: S603 - fixed argv list, no shell
|
|
36
|
+
argv, capture_output=True, text=True, check=False, timeout=SUBPROCESS_TIMEOUT
|
|
37
|
+
)
|
|
38
|
+
except subprocess.TimeoutExpired:
|
|
39
|
+
return CommandResult(124, "", f"{argv[0]} did not finish within {SUBPROCESS_TIMEOUT}s")
|
|
40
|
+
except OSError as exc:
|
|
41
|
+
return CommandResult(127, "", f"could not run {argv[0]}: {exc.strerror or exc}")
|
|
42
|
+
return CommandResult(completed.returncode, completed.stdout, completed.stderr)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _open_browser(url: str) -> bool:
|
|
46
|
+
try:
|
|
47
|
+
return webbrowser.open(url, new=2)
|
|
48
|
+
except webbrowser.Error:
|
|
49
|
+
return False
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass
|
|
53
|
+
class Runtime:
|
|
54
|
+
env: Mapping[str, str]
|
|
55
|
+
stdout: TextIO
|
|
56
|
+
stderr: TextIO
|
|
57
|
+
stdin: TextIO
|
|
58
|
+
cwd: Path
|
|
59
|
+
platform: str = sys.platform
|
|
60
|
+
transport: httpx.BaseTransport | None = None
|
|
61
|
+
sleep: Callable[[float], None] = time.sleep
|
|
62
|
+
now: Callable[[], float] = time.time
|
|
63
|
+
open_browser: Callable[[str], bool] = _open_browser
|
|
64
|
+
keyring_loader: Callable[[Mapping[str, str]], KeyringLike | None] = load_system_keyring
|
|
65
|
+
run_command: Callable[[list[str]], CommandResult] = run_subprocess
|
|
66
|
+
which: Callable[[str], str | None] = shutil.which
|
|
67
|
+
_keyring_loaded: bool = field(default=False, repr=False)
|
|
68
|
+
_keyring: KeyringLike | None = field(default=None, repr=False)
|
|
69
|
+
|
|
70
|
+
@classmethod
|
|
71
|
+
def system(cls) -> Runtime:
|
|
72
|
+
return cls(
|
|
73
|
+
env=os.environ,
|
|
74
|
+
stdout=sys.stdout,
|
|
75
|
+
stderr=sys.stderr,
|
|
76
|
+
stdin=sys.stdin,
|
|
77
|
+
cwd=Path.cwd(),
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
def keyring(self) -> KeyringLike | None:
|
|
81
|
+
if not self._keyring_loaded:
|
|
82
|
+
self._keyring = self.keyring_loader(self.env)
|
|
83
|
+
self._keyring_loaded = True
|
|
84
|
+
return self._keyring
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _remediation_lines(remediation: Any) -> list[str]:
|
|
88
|
+
if remediation is None:
|
|
89
|
+
return []
|
|
90
|
+
if isinstance(remediation, str):
|
|
91
|
+
return [remediation]
|
|
92
|
+
if isinstance(remediation, list):
|
|
93
|
+
lines: list[str] = []
|
|
94
|
+
for item in remediation:
|
|
95
|
+
lines.extend(_remediation_lines(item))
|
|
96
|
+
return lines
|
|
97
|
+
if isinstance(remediation, dict):
|
|
98
|
+
words = [
|
|
99
|
+
str(remediation[key])
|
|
100
|
+
for key in ("message", "hint", "description", "action")
|
|
101
|
+
if isinstance(remediation.get(key), str)
|
|
102
|
+
]
|
|
103
|
+
links = [
|
|
104
|
+
str(remediation[key])
|
|
105
|
+
for key in ("url", "checkout_url", "docs_url", "href")
|
|
106
|
+
if isinstance(remediation.get(key), str)
|
|
107
|
+
]
|
|
108
|
+
text = " ".join(words + links).strip()
|
|
109
|
+
return [text or json.dumps(remediation, sort_keys=True)]
|
|
110
|
+
return [str(remediation)]
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class Output:
|
|
114
|
+
"""Machine output on stdout in --json mode; people-facing text otherwise.
|
|
115
|
+
|
|
116
|
+
Progress and prompts always go to stderr so stdout stays parseable.
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
def __init__(self, runtime: Runtime, *, json_mode: bool) -> None:
|
|
120
|
+
self.runtime = runtime
|
|
121
|
+
self.json_mode = json_mode
|
|
122
|
+
|
|
123
|
+
def data(self, value: Any) -> None:
|
|
124
|
+
self.runtime.stdout.write(json.dumps(value, indent=2, ensure_ascii=False) + "\n")
|
|
125
|
+
self.runtime.stdout.flush()
|
|
126
|
+
|
|
127
|
+
def line(self, text: str = "") -> None:
|
|
128
|
+
self.runtime.stdout.write(text + "\n")
|
|
129
|
+
self.runtime.stdout.flush()
|
|
130
|
+
|
|
131
|
+
def info(self, text: str) -> None:
|
|
132
|
+
self.runtime.stderr.write(text + "\n")
|
|
133
|
+
self.runtime.stderr.flush()
|
|
134
|
+
|
|
135
|
+
def event(self, value: dict[str, Any]) -> None:
|
|
136
|
+
"""A JSON progress event on stderr (--json mode) for relaying to a person."""
|
|
137
|
+
self.runtime.stderr.write(json.dumps(value, ensure_ascii=False) + "\n")
|
|
138
|
+
self.runtime.stderr.flush()
|
|
139
|
+
|
|
140
|
+
def fields(self, values: Mapping[str, Any], keys: list[str] | None = None) -> None:
|
|
141
|
+
for key in keys if keys is not None else list(values):
|
|
142
|
+
value = values.get(key)
|
|
143
|
+
if value is None or isinstance(value, (dict, list)):
|
|
144
|
+
continue
|
|
145
|
+
self.line(f"{key}: {value}")
|
|
146
|
+
|
|
147
|
+
def error(self, error: CliError) -> None:
|
|
148
|
+
if self.json_mode:
|
|
149
|
+
self.data(error.to_json())
|
|
150
|
+
return
|
|
151
|
+
lines = [f"error: {error.message}"]
|
|
152
|
+
if isinstance(error, ApiError):
|
|
153
|
+
detail = f"HTTP {error.status}, {error.code}"
|
|
154
|
+
if error.correlation_id:
|
|
155
|
+
detail += f", request {error.correlation_id}"
|
|
156
|
+
lines.append(f" ({detail})")
|
|
157
|
+
for text in _remediation_lines(error.remediation):
|
|
158
|
+
lines.append(f"next step: {text}")
|
|
159
|
+
if error.retry_after is not None:
|
|
160
|
+
lines.append(f"retry after: {error.retry_after:g}s")
|
|
161
|
+
if error.hint:
|
|
162
|
+
lines.append(f"hint: {error.hint}")
|
|
163
|
+
self.info("\n".join(lines))
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Unfinished `jobs submit` runs, so a plain rerun resumes instead of resubmitting.
|
|
2
|
+
|
|
3
|
+
Before the first chunk is sent, the base Idempotency-Key is saved under a
|
|
4
|
+
digest of what is being submitted (the origin, the chunk size and the records)
|
|
5
|
+
in `pending-submissions.json` (mode 0600). Rerunning the same submission
|
|
6
|
+
within 24 hours reuses that key, so the server recovers the jobs it already
|
|
7
|
+
accepted instead of queueing them again. The entry is removed once every chunk
|
|
8
|
+
is accepted and, with --wait, every job has finished. `--idempotency-key`
|
|
9
|
+
bypasses this file.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import contextlib
|
|
15
|
+
import hashlib
|
|
16
|
+
import json
|
|
17
|
+
from collections.abc import Callable, Mapping
|
|
18
|
+
from typing import Any, TypeGuard
|
|
19
|
+
|
|
20
|
+
from entitybind.files import atomic_write, config_dir, ensure_private_dir
|
|
21
|
+
|
|
22
|
+
PENDING_SUBMISSIONS_FILE = "pending-submissions.json"
|
|
23
|
+
RESUME_WINDOW_SECONDS = 24 * 3600.0
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def submission_digest(base_url: str, chunk_size: int, records: list[dict[str, Any]]) -> str:
|
|
27
|
+
canonical = json.dumps(
|
|
28
|
+
{"base_url": base_url, "chunk_size": chunk_size, "records": records},
|
|
29
|
+
sort_keys=True,
|
|
30
|
+
separators=(",", ":"),
|
|
31
|
+
ensure_ascii=False,
|
|
32
|
+
)
|
|
33
|
+
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class SubmissionStore:
|
|
37
|
+
def __init__(self, env: Mapping[str, str], now: Callable[[], float]) -> None:
|
|
38
|
+
self.directory = config_dir(env)
|
|
39
|
+
self.path = self.directory / PENDING_SUBMISSIONS_FILE
|
|
40
|
+
self._now = now
|
|
41
|
+
|
|
42
|
+
def _fresh(self, record: Any) -> TypeGuard[dict[str, Any]]:
|
|
43
|
+
if not isinstance(record, dict) or not isinstance(record.get("idempotency_key"), str):
|
|
44
|
+
return False
|
|
45
|
+
started = record.get("started_at")
|
|
46
|
+
return isinstance(started, (int, float)) and 0 <= self._now() - started < (
|
|
47
|
+
RESUME_WINDOW_SECONDS
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
def _read(self) -> dict[str, Any]:
|
|
51
|
+
try:
|
|
52
|
+
data = json.loads(self.path.read_text(encoding="utf-8"))
|
|
53
|
+
except (OSError, ValueError):
|
|
54
|
+
return {} # a missing or damaged resume file loses the resume, never the submission
|
|
55
|
+
pending = data.get("pending") if isinstance(data, dict) else None
|
|
56
|
+
return pending if isinstance(pending, dict) else {}
|
|
57
|
+
|
|
58
|
+
def _write(self, pending: dict[str, Any]) -> None:
|
|
59
|
+
"""Best effort: an unwritable directory loses the resume, never the submission."""
|
|
60
|
+
live = {digest: record for digest, record in pending.items() if self._fresh(record)}
|
|
61
|
+
with contextlib.suppress(OSError):
|
|
62
|
+
if not live:
|
|
63
|
+
self.path.unlink(missing_ok=True)
|
|
64
|
+
return
|
|
65
|
+
ensure_private_dir(self.directory)
|
|
66
|
+
document = {"version": 1, "pending": live}
|
|
67
|
+
text = json.dumps(document, indent=2, sort_keys=True) + "\n"
|
|
68
|
+
atomic_write(self.path, text, mode=0o600)
|
|
69
|
+
|
|
70
|
+
def get(self, digest: str) -> str | None:
|
|
71
|
+
record = self._read().get(digest)
|
|
72
|
+
return str(record["idempotency_key"]) if self._fresh(record) else None
|
|
73
|
+
|
|
74
|
+
def put(self, digest: str, idempotency_key: str) -> None:
|
|
75
|
+
pending = self._read()
|
|
76
|
+
pending[digest] = {"idempotency_key": idempotency_key, "started_at": self._now()}
|
|
77
|
+
self._write(pending)
|
|
78
|
+
|
|
79
|
+
def delete(self, digest: str) -> None:
|
|
80
|
+
pending = self._read()
|
|
81
|
+
if digest in pending:
|
|
82
|
+
del pending[digest]
|
|
83
|
+
self._write(pending)
|