lumera 0.4.6__py3-none-any.whl → 0.9.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lumera/__init__.py +99 -4
- lumera/_utils.py +782 -0
- lumera/automations.py +904 -0
- lumera/exceptions.py +72 -0
- lumera/files.py +97 -0
- lumera/google.py +47 -270
- lumera/integrations/__init__.py +34 -0
- lumera/integrations/google.py +338 -0
- lumera/llm.py +481 -0
- lumera/locks.py +216 -0
- lumera/pb.py +679 -0
- lumera/sdk.py +927 -380
- lumera/storage.py +270 -0
- lumera/webhooks.py +304 -0
- lumera-0.9.6.dist-info/METADATA +37 -0
- lumera-0.9.6.dist-info/RECORD +18 -0
- {lumera-0.4.6.dist-info → lumera-0.9.6.dist-info}/WHEEL +1 -1
- lumera-0.4.6.dist-info/METADATA +0 -11
- lumera-0.4.6.dist-info/RECORD +0 -7
- {lumera-0.4.6.dist-info → lumera-0.9.6.dist-info}/top_level.txt +0 -0
lumera/_utils.py
ADDED
|
@@ -0,0 +1,782 @@
|
|
|
1
|
+
"""Internal helpers shared across the Lumera SDK."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import datetime as _dt
|
|
6
|
+
import json
|
|
7
|
+
import logging as _logging
|
|
8
|
+
import mimetypes
|
|
9
|
+
import os
|
|
10
|
+
import pathlib
|
|
11
|
+
import time as _time
|
|
12
|
+
from functools import wraps as _wraps
|
|
13
|
+
from typing import IO, Any, Callable, Iterable, Mapping, MutableMapping, Sequence, TypeVar
|
|
14
|
+
|
|
15
|
+
import requests
|
|
16
|
+
import requests.adapters
|
|
17
|
+
from dotenv import load_dotenv
|
|
18
|
+
|
|
19
|
+
TOKEN_ENV = "LUMERA_TOKEN"
|
|
20
|
+
BASE_URL_ENV = "LUMERA_BASE_URL"
|
|
21
|
+
ENV_PATH = "/root/.env"
|
|
22
|
+
|
|
23
|
+
load_dotenv(override=False)
|
|
24
|
+
load_dotenv(ENV_PATH, override=False)
|
|
25
|
+
|
|
26
|
+
_default_api_base = "https://app.lumerahq.com/api"
|
|
27
|
+
API_BASE = os.getenv(BASE_URL_ENV, _default_api_base).rstrip("/")
|
|
28
|
+
MOUNT_ROOT_ENV = "LUMERA_MOUNT_ROOT"
|
|
29
|
+
DEFAULT_MOUNT_ROOT = "/tmp/lumera-files"
|
|
30
|
+
LEGACY_MOUNT_ROOT = "/lumera-files"
|
|
31
|
+
_mount_env = os.getenv(MOUNT_ROOT_ENV, DEFAULT_MOUNT_ROOT).rstrip("/")
|
|
32
|
+
if not _mount_env:
|
|
33
|
+
_mount_env = DEFAULT_MOUNT_ROOT
|
|
34
|
+
MOUNT_ROOT = _mount_env
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
_token_cache: dict[str, tuple[str, float]] = {}
|
|
38
|
+
|
|
39
|
+
# Connection pooling for better performance
|
|
40
|
+
_http_session: requests.Session | None = None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _get_session() -> requests.Session:
|
|
44
|
+
"""Get or create a shared requests Session with connection pooling."""
|
|
45
|
+
global _http_session
|
|
46
|
+
if _http_session is None:
|
|
47
|
+
_http_session = requests.Session()
|
|
48
|
+
# Configure connection pooling
|
|
49
|
+
adapter = requests.adapters.HTTPAdapter(
|
|
50
|
+
pool_connections=10,
|
|
51
|
+
pool_maxsize=20,
|
|
52
|
+
max_retries=0, # Don't retry automatically, let caller handle
|
|
53
|
+
)
|
|
54
|
+
_http_session.mount("https://", adapter)
|
|
55
|
+
_http_session.mount("http://", adapter)
|
|
56
|
+
return _http_session
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def get_lumera_token() -> str:
|
|
60
|
+
token = os.getenv(TOKEN_ENV)
|
|
61
|
+
if token:
|
|
62
|
+
return token
|
|
63
|
+
raise RuntimeError(
|
|
64
|
+
f"{TOKEN_ENV} environment variable not set (checked environment and {ENV_PATH})"
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _parse_expiry(expires_at: int | float | str | None) -> float:
|
|
69
|
+
if not expires_at:
|
|
70
|
+
return float("inf")
|
|
71
|
+
if isinstance(expires_at, (int, float)):
|
|
72
|
+
return float(expires_at)
|
|
73
|
+
if isinstance(expires_at, str):
|
|
74
|
+
if expires_at.endswith("Z"):
|
|
75
|
+
expires_at = expires_at[:-1] + "+00:00"
|
|
76
|
+
return _dt.datetime.fromisoformat(expires_at).timestamp()
|
|
77
|
+
raise TypeError(f"Unsupported expires_at format: {type(expires_at)!r}")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _fetch_access_token(provider: str) -> tuple[str, float]:
|
|
81
|
+
provider = provider.lower().strip()
|
|
82
|
+
if not provider:
|
|
83
|
+
raise ValueError("provider is required")
|
|
84
|
+
|
|
85
|
+
token = get_lumera_token()
|
|
86
|
+
|
|
87
|
+
url = f"{API_BASE}/connections/{provider}/access-token"
|
|
88
|
+
headers = {"Authorization": f"token {token}"}
|
|
89
|
+
|
|
90
|
+
resp = requests.get(url, headers=headers, timeout=30)
|
|
91
|
+
resp.raise_for_status()
|
|
92
|
+
|
|
93
|
+
data = resp.json()
|
|
94
|
+
access_token = data.get("access_token")
|
|
95
|
+
expires_at = data.get("expires_at")
|
|
96
|
+
|
|
97
|
+
if not access_token:
|
|
98
|
+
raise RuntimeError(f"Malformed response from Lumera when fetching {provider} access token")
|
|
99
|
+
|
|
100
|
+
expiry_ts = _parse_expiry(expires_at)
|
|
101
|
+
return access_token, expiry_ts
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def get_access_token(provider: str, min_valid_seconds: int = 900) -> str:
|
|
105
|
+
global _token_cache
|
|
106
|
+
|
|
107
|
+
provider = provider.lower().strip()
|
|
108
|
+
if not provider:
|
|
109
|
+
raise ValueError("provider is required")
|
|
110
|
+
|
|
111
|
+
now = _time.time()
|
|
112
|
+
|
|
113
|
+
cached = _token_cache.get(provider)
|
|
114
|
+
if cached is not None:
|
|
115
|
+
access_token, expiry_ts = cached
|
|
116
|
+
if (expiry_ts - now) >= min_valid_seconds:
|
|
117
|
+
return access_token
|
|
118
|
+
|
|
119
|
+
access_token, expiry_ts = _fetch_access_token(provider)
|
|
120
|
+
_token_cache[provider] = (access_token, expiry_ts)
|
|
121
|
+
return access_token
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def get_google_access_token(min_valid_seconds: int = 900) -> str:
|
|
125
|
+
return get_access_token("google", min_valid_seconds=min_valid_seconds)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
_logger = _logging.getLogger("lumera.sdk")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _utcnow_iso() -> str:
|
|
132
|
+
return (
|
|
133
|
+
_dt.datetime.now(tz=_dt.timezone.utc)
|
|
134
|
+
.replace(microsecond=0)
|
|
135
|
+
.isoformat()
|
|
136
|
+
.replace("+00:00", "Z")
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _default_provenance(automation_id: str, run_id: str | None) -> dict[str, Any]:
|
|
141
|
+
recorded_at = _utcnow_iso()
|
|
142
|
+
env_automation_id = os.getenv("LUMERA_AUTOMATION_ID", "").strip()
|
|
143
|
+
automation_id = (automation_id or "").strip() or env_automation_id
|
|
144
|
+
|
|
145
|
+
run_id = (run_id or "").strip() or os.getenv("LUMERA_RUN_ID", "").strip()
|
|
146
|
+
|
|
147
|
+
company_id = os.getenv("COMPANY_ID", "").strip()
|
|
148
|
+
company_api = os.getenv("COMPANY_API_NAME", "").strip()
|
|
149
|
+
|
|
150
|
+
payload: dict[str, Any] = {
|
|
151
|
+
"type": "user",
|
|
152
|
+
"recorded_at": recorded_at,
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
if automation_id:
|
|
156
|
+
payload["agent"] = {"id": automation_id} # Backend still expects "agent" key
|
|
157
|
+
|
|
158
|
+
if run_id:
|
|
159
|
+
payload["agent_run"] = {"id": run_id}
|
|
160
|
+
|
|
161
|
+
if company_id or company_api:
|
|
162
|
+
company: dict[str, Any] = {}
|
|
163
|
+
if company_id:
|
|
164
|
+
company["id"] = company_id
|
|
165
|
+
if company_api:
|
|
166
|
+
company["api_name"] = company_api
|
|
167
|
+
payload["company"] = company
|
|
168
|
+
|
|
169
|
+
return payload
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
R = TypeVar("R")
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def log_timed(fn: Callable[..., R]) -> Callable[..., R]:
|
|
176
|
+
@_wraps(fn)
|
|
177
|
+
def wrapper(*args: object, **kwargs: object) -> R:
|
|
178
|
+
_logger.info(f"Entering {fn.__name__}()")
|
|
179
|
+
t0 = _time.perf_counter()
|
|
180
|
+
try:
|
|
181
|
+
return fn(*args, **kwargs)
|
|
182
|
+
finally:
|
|
183
|
+
dt = _time.perf_counter() - t0
|
|
184
|
+
_logger.info(f"Exiting {fn.__name__}() - took {dt:.3f}s")
|
|
185
|
+
|
|
186
|
+
return wrapper
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _api_url(path: str) -> str:
|
|
190
|
+
path = path.lstrip("/")
|
|
191
|
+
if path.startswith("pb/"):
|
|
192
|
+
return f"{API_BASE}/{path}"
|
|
193
|
+
if path == "collections" or path.startswith("collections/"):
|
|
194
|
+
return f"{API_BASE}/pb/{path}"
|
|
195
|
+
return f"{API_BASE}/{path}"
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
class LumeraAPIError(RuntimeError):
|
|
199
|
+
"""Raised when requests to the Lumera API fail."""
|
|
200
|
+
|
|
201
|
+
def __init__(
|
|
202
|
+
self, status_code: int, message: str, *, url: str, payload: object | None = None
|
|
203
|
+
) -> None:
|
|
204
|
+
super().__init__(message)
|
|
205
|
+
self.status_code = status_code
|
|
206
|
+
self.payload = payload
|
|
207
|
+
self.url = url
|
|
208
|
+
|
|
209
|
+
def __str__(self) -> str: # pragma: no cover - trivial string formatting
|
|
210
|
+
base = super().__str__()
|
|
211
|
+
return (
|
|
212
|
+
f"{self.status_code} {self.url}: {base}" if base else f"{self.status_code} {self.url}"
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
class RecordNotUniqueError(LumeraAPIError):
|
|
217
|
+
"""Raised when attempting to insert a record that violates a uniqueness constraint."""
|
|
218
|
+
|
|
219
|
+
def __init__(self, url: str, payload: MutableMapping[str, object]) -> None:
|
|
220
|
+
message = next(
|
|
221
|
+
(value for value in payload.values() if isinstance(value, str) and value.strip()),
|
|
222
|
+
"record violates uniqueness constraint",
|
|
223
|
+
)
|
|
224
|
+
super().__init__(400, message, url=url, payload=payload)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _raise_api_error(resp: requests.Response) -> None:
|
|
228
|
+
message = resp.text.strip()
|
|
229
|
+
payload: object | None = None
|
|
230
|
+
content_type = resp.headers.get("Content-Type", "").lower()
|
|
231
|
+
if "application/json" in content_type:
|
|
232
|
+
try:
|
|
233
|
+
payload = resp.json()
|
|
234
|
+
except ValueError:
|
|
235
|
+
payload = None
|
|
236
|
+
else:
|
|
237
|
+
if isinstance(payload, MutableMapping):
|
|
238
|
+
for key in ("error", "message", "detail"):
|
|
239
|
+
value = payload.get(key)
|
|
240
|
+
if isinstance(value, str) and value.strip():
|
|
241
|
+
message = value
|
|
242
|
+
break
|
|
243
|
+
else:
|
|
244
|
+
message = json.dumps(payload)
|
|
245
|
+
else:
|
|
246
|
+
message = json.dumps(payload)
|
|
247
|
+
if resp.status_code == 400 and payload and isinstance(payload, MutableMapping):
|
|
248
|
+
if any(
|
|
249
|
+
isinstance(value, str) and ("unique" in value.lower() or "already" in value.lower())
|
|
250
|
+
for value in payload.values()
|
|
251
|
+
):
|
|
252
|
+
raise RecordNotUniqueError(resp.url, payload) from None
|
|
253
|
+
raise LumeraAPIError(resp.status_code, message, url=resp.url, payload=payload)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _api_request(
|
|
257
|
+
method: str,
|
|
258
|
+
path: str,
|
|
259
|
+
*,
|
|
260
|
+
params: Mapping[str, Any] | None = None,
|
|
261
|
+
json_body: Mapping[str, Any] | None = None,
|
|
262
|
+
data: Mapping[str, Any] | None = None,
|
|
263
|
+
files: Mapping[str, Any] | None = None,
|
|
264
|
+
timeout: int = 30,
|
|
265
|
+
) -> object | None:
|
|
266
|
+
token = get_lumera_token()
|
|
267
|
+
url = _api_url(path)
|
|
268
|
+
|
|
269
|
+
headers = {
|
|
270
|
+
"Authorization": f"token {token}",
|
|
271
|
+
"Accept": "application/json",
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
session = _get_session()
|
|
275
|
+
resp = session.request(
|
|
276
|
+
method,
|
|
277
|
+
url,
|
|
278
|
+
params=params,
|
|
279
|
+
json=json_body,
|
|
280
|
+
data=data,
|
|
281
|
+
files=files,
|
|
282
|
+
headers=headers,
|
|
283
|
+
timeout=timeout,
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
if not resp.ok:
|
|
287
|
+
_raise_api_error(resp)
|
|
288
|
+
|
|
289
|
+
if resp.status_code == 204 or method.upper() == "DELETE":
|
|
290
|
+
return None
|
|
291
|
+
|
|
292
|
+
content_type = resp.headers.get("Content-Type", "").lower()
|
|
293
|
+
if "application/json" in content_type:
|
|
294
|
+
if not resp.text.strip():
|
|
295
|
+
return {}
|
|
296
|
+
return resp.json()
|
|
297
|
+
return resp.text if resp.text else None
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _ensure_mapping(payload: Mapping[str, Any] | None, *, name: str) -> dict[str, Any]:
|
|
301
|
+
if payload is None:
|
|
302
|
+
return {}
|
|
303
|
+
if not isinstance(payload, Mapping):
|
|
304
|
+
raise TypeError(f"{name} must be a mapping, got {type(payload)!r}")
|
|
305
|
+
return dict(payload)
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _prepare_automation_inputs(
|
|
309
|
+
inputs: Mapping[str, Any] | str | None,
|
|
310
|
+
) -> dict[str, Any] | None:
|
|
311
|
+
if inputs is None:
|
|
312
|
+
return None
|
|
313
|
+
if isinstance(inputs, str):
|
|
314
|
+
inputs = inputs.strip()
|
|
315
|
+
if not inputs:
|
|
316
|
+
return {}
|
|
317
|
+
try:
|
|
318
|
+
parsed = json.loads(inputs)
|
|
319
|
+
except json.JSONDecodeError as exc:
|
|
320
|
+
raise ValueError("inputs must be JSON-serialisable") from exc
|
|
321
|
+
if not isinstance(parsed, dict):
|
|
322
|
+
raise TypeError("inputs JSON must deserialize to an object")
|
|
323
|
+
return parsed
|
|
324
|
+
|
|
325
|
+
try:
|
|
326
|
+
serialised = json.dumps(inputs)
|
|
327
|
+
parsed = json.loads(serialised)
|
|
328
|
+
except (TypeError, ValueError) as exc:
|
|
329
|
+
raise ValueError("inputs must be JSON-serialisable") from exc
|
|
330
|
+
if not isinstance(parsed, dict):
|
|
331
|
+
raise TypeError("inputs mapping must serialize to a JSON object")
|
|
332
|
+
return parsed
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _ensure_json_string(value: Mapping[str, Any] | str, *, name: str) -> str:
|
|
336
|
+
if isinstance(value, str):
|
|
337
|
+
return value
|
|
338
|
+
try:
|
|
339
|
+
return json.dumps(value)
|
|
340
|
+
except (TypeError, ValueError) as exc:
|
|
341
|
+
raise ValueError(f"{name} must be JSON-serialisable") from exc
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def _is_sequence(value: object) -> bool:
|
|
345
|
+
if isinstance(value, (str, os.PathLike)):
|
|
346
|
+
return False
|
|
347
|
+
return isinstance(value, Sequence)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def _ensure_sequence(
|
|
351
|
+
value: str | os.PathLike[str] | Sequence[str | os.PathLike[str]],
|
|
352
|
+
) -> list[str | os.PathLike[str]]:
|
|
353
|
+
if isinstance(value, (str, os.PathLike)):
|
|
354
|
+
return [value]
|
|
355
|
+
seq = list(value)
|
|
356
|
+
if not seq:
|
|
357
|
+
raise ValueError("file input sequence must not be empty")
|
|
358
|
+
return seq
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def _upload_file_to_presigned(upload_url: str, path: pathlib.Path, content_type: str) -> None:
|
|
362
|
+
with open(path, "rb") as fh:
|
|
363
|
+
resp = requests.put(
|
|
364
|
+
upload_url, data=fh, headers={"Content-Type": content_type}, timeout=300
|
|
365
|
+
)
|
|
366
|
+
resp.raise_for_status()
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _upload_automation_files(
|
|
370
|
+
run_id: str | None,
|
|
371
|
+
files: Mapping[str, str | os.PathLike[str] | Sequence[str | os.PathLike[str]]],
|
|
372
|
+
*,
|
|
373
|
+
api_request: Callable[..., object] | None = None,
|
|
374
|
+
) -> tuple[str | None, dict[str, list[dict[str, Any]]]]:
|
|
375
|
+
api_fn = api_request or _api_request
|
|
376
|
+
|
|
377
|
+
if not files:
|
|
378
|
+
return run_id, {}
|
|
379
|
+
|
|
380
|
+
results: dict[str, list[dict[str, Any]]] = {}
|
|
381
|
+
for key, value in files.items():
|
|
382
|
+
paths = _ensure_sequence(value)
|
|
383
|
+
descriptors: list[dict[str, Any]] = []
|
|
384
|
+
for path in paths:
|
|
385
|
+
file_path = pathlib.Path(os.fspath(path)).expanduser().resolve()
|
|
386
|
+
if not file_path.is_file():
|
|
387
|
+
raise FileNotFoundError(file_path)
|
|
388
|
+
|
|
389
|
+
filename = file_path.name
|
|
390
|
+
content_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
|
|
391
|
+
size = file_path.stat().st_size
|
|
392
|
+
|
|
393
|
+
body: dict[str, Any] = {
|
|
394
|
+
"scope": "automation_run",
|
|
395
|
+
"filename": filename,
|
|
396
|
+
"content_type": content_type,
|
|
397
|
+
"size": size,
|
|
398
|
+
}
|
|
399
|
+
if run_id:
|
|
400
|
+
body["resource_id"] = run_id
|
|
401
|
+
|
|
402
|
+
presign = api_fn(
|
|
403
|
+
"POST",
|
|
404
|
+
"uploads/presign",
|
|
405
|
+
json_body=body,
|
|
406
|
+
)
|
|
407
|
+
if not isinstance(presign, dict):
|
|
408
|
+
raise RuntimeError("unexpected presign response")
|
|
409
|
+
|
|
410
|
+
upload_url = presign.get("upload_url")
|
|
411
|
+
if not isinstance(upload_url, str) or not upload_url:
|
|
412
|
+
raise RuntimeError("missing upload_url in presign response")
|
|
413
|
+
|
|
414
|
+
resp_run_id = presign.get("run_id")
|
|
415
|
+
if isinstance(resp_run_id, str) and resp_run_id:
|
|
416
|
+
if run_id is None:
|
|
417
|
+
run_id = resp_run_id
|
|
418
|
+
elif run_id != resp_run_id:
|
|
419
|
+
raise RuntimeError("presign returned inconsistent run_id")
|
|
420
|
+
elif run_id is None:
|
|
421
|
+
raise RuntimeError("presign response missing run_id")
|
|
422
|
+
|
|
423
|
+
_upload_file_to_presigned(upload_url, file_path, content_type)
|
|
424
|
+
|
|
425
|
+
descriptor: dict[str, Any] = {"name": filename}
|
|
426
|
+
if presign.get("run_path"):
|
|
427
|
+
descriptor["run_path"] = presign["run_path"]
|
|
428
|
+
if presign.get("object_key"):
|
|
429
|
+
descriptor["object_key"] = presign["object_key"]
|
|
430
|
+
descriptors.append(descriptor)
|
|
431
|
+
|
|
432
|
+
results[key] = descriptors
|
|
433
|
+
return run_id, results
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def _record_mutation(
|
|
437
|
+
method: str,
|
|
438
|
+
collection_id_or_name: str,
|
|
439
|
+
payload: Mapping[str, Any] | None,
|
|
440
|
+
*,
|
|
441
|
+
record_id: str | None = None,
|
|
442
|
+
api_request: Callable[..., object] | None = None,
|
|
443
|
+
) -> dict[str, Any]:
|
|
444
|
+
api_fn = api_request or _api_request
|
|
445
|
+
|
|
446
|
+
if not collection_id_or_name:
|
|
447
|
+
raise ValueError("collection_id_or_name is required")
|
|
448
|
+
|
|
449
|
+
data = _ensure_mapping(payload, name="payload")
|
|
450
|
+
path = f"collections/{collection_id_or_name}/records"
|
|
451
|
+
if record_id:
|
|
452
|
+
if not record_id.strip():
|
|
453
|
+
raise ValueError("record_id is required")
|
|
454
|
+
path = f"{path}/{record_id}".rstrip("/")
|
|
455
|
+
|
|
456
|
+
response = api_fn(method, path, json_body=data)
|
|
457
|
+
|
|
458
|
+
if not isinstance(response, dict):
|
|
459
|
+
raise RuntimeError("unexpected response payload")
|
|
460
|
+
return response
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def _normalize_mount_path(path: str) -> str:
|
|
464
|
+
if not isinstance(path, str):
|
|
465
|
+
return path
|
|
466
|
+
root = MOUNT_ROOT.rstrip("/")
|
|
467
|
+
legacy = LEGACY_MOUNT_ROOT.rstrip("/")
|
|
468
|
+
if legacy and path.startswith(legacy):
|
|
469
|
+
suffix = path[len(legacy) :].lstrip("/")
|
|
470
|
+
return f"{root}/{suffix}" if suffix else root
|
|
471
|
+
return path
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def resolve_path(file_or_path: str | Mapping[str, Any]) -> str:
|
|
475
|
+
if isinstance(file_or_path, str):
|
|
476
|
+
return _normalize_mount_path(file_or_path)
|
|
477
|
+
if isinstance(file_or_path, Mapping):
|
|
478
|
+
path_value = file_or_path.get("path")
|
|
479
|
+
if isinstance(path_value, str):
|
|
480
|
+
return _normalize_mount_path(path_value)
|
|
481
|
+
run_path = file_or_path.get("run_path")
|
|
482
|
+
if isinstance(run_path, str):
|
|
483
|
+
return _normalize_mount_path(run_path)
|
|
484
|
+
raise TypeError("Unsupported file_or_path; expected str or dict with 'path'/'run_path'")
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def open_file(
|
|
488
|
+
file_or_path: str | Mapping[str, Any],
|
|
489
|
+
mode: str = "r",
|
|
490
|
+
**kwargs: object,
|
|
491
|
+
) -> IO[str] | IO[bytes]:
|
|
492
|
+
p = resolve_path(file_or_path)
|
|
493
|
+
return open(p, mode, **kwargs)
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def to_filerefs(
|
|
497
|
+
values: Iterable[str | Mapping[str, Any]],
|
|
498
|
+
scope: str,
|
|
499
|
+
id: str,
|
|
500
|
+
) -> list[dict[str, Any]]:
|
|
501
|
+
out: list[dict[str, Any]] = []
|
|
502
|
+
for v in values:
|
|
503
|
+
if isinstance(v, str):
|
|
504
|
+
normalized = _normalize_mount_path(v)
|
|
505
|
+
name = os.path.basename(normalized)
|
|
506
|
+
object_name = f"{scope}/{id}/{name}"
|
|
507
|
+
out.append(
|
|
508
|
+
{
|
|
509
|
+
"scope": scope,
|
|
510
|
+
"id": id,
|
|
511
|
+
"name": name,
|
|
512
|
+
"path": normalized,
|
|
513
|
+
"object_name": object_name,
|
|
514
|
+
}
|
|
515
|
+
)
|
|
516
|
+
elif isinstance(v, Mapping):
|
|
517
|
+
resolved_path = resolve_path(v)
|
|
518
|
+
normalized = _normalize_mount_path(resolved_path)
|
|
519
|
+
name = v.get("name") or os.path.basename(normalized)
|
|
520
|
+
object_name = v.get("object_name") or f"{scope}/{id}/{name}"
|
|
521
|
+
ref: dict[str, Any] = {
|
|
522
|
+
"scope": v.get("scope", scope),
|
|
523
|
+
"id": v.get("id", id),
|
|
524
|
+
"name": name,
|
|
525
|
+
"path": normalized,
|
|
526
|
+
"object_name": object_name,
|
|
527
|
+
}
|
|
528
|
+
if "mime" in v:
|
|
529
|
+
ref["mime"] = v["mime"]
|
|
530
|
+
if "size" in v:
|
|
531
|
+
ref["size"] = v["size"]
|
|
532
|
+
out.append(ref)
|
|
533
|
+
else:
|
|
534
|
+
raise TypeError("values must contain str or dict entries")
|
|
535
|
+
return out
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
def _upload_lumera_file(
|
|
539
|
+
collection_id_or_name: str,
|
|
540
|
+
field_name: str,
|
|
541
|
+
file_path: str | os.PathLike[str],
|
|
542
|
+
*,
|
|
543
|
+
record_id: str | None = None,
|
|
544
|
+
api_request: Callable[..., object] | None = None,
|
|
545
|
+
) -> dict[str, Any]:
|
|
546
|
+
api_fn = api_request or _api_request
|
|
547
|
+
|
|
548
|
+
collection = str(collection_id_or_name or "").strip()
|
|
549
|
+
if not collection:
|
|
550
|
+
raise ValueError("collection_id_or_name is required")
|
|
551
|
+
field = str(field_name or "").strip()
|
|
552
|
+
if not field:
|
|
553
|
+
raise ValueError("field_name is required")
|
|
554
|
+
|
|
555
|
+
file_obj = pathlib.Path(os.fspath(file_path)).expanduser().resolve()
|
|
556
|
+
if not file_obj.is_file():
|
|
557
|
+
raise FileNotFoundError(file_obj)
|
|
558
|
+
|
|
559
|
+
filename = file_obj.name
|
|
560
|
+
size = file_obj.stat().st_size
|
|
561
|
+
content_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
|
|
562
|
+
|
|
563
|
+
payload: dict[str, Any] = {
|
|
564
|
+
"collection_id": collection,
|
|
565
|
+
"field_name": field,
|
|
566
|
+
"filename": filename,
|
|
567
|
+
"content_type": content_type,
|
|
568
|
+
"size": size,
|
|
569
|
+
}
|
|
570
|
+
if record_id and str(record_id).strip():
|
|
571
|
+
payload["record_id"] = str(record_id).strip()
|
|
572
|
+
|
|
573
|
+
presign = api_fn("POST", "pb/uploads/presign", json_body=payload)
|
|
574
|
+
if not isinstance(presign, Mapping):
|
|
575
|
+
raise RuntimeError("unexpected presign response")
|
|
576
|
+
|
|
577
|
+
upload_url = str(presign.get("upload_url") or "").strip()
|
|
578
|
+
object_key = str(presign.get("object_key") or "").strip()
|
|
579
|
+
if not upload_url or not object_key:
|
|
580
|
+
raise RuntimeError("presign response missing upload metadata")
|
|
581
|
+
|
|
582
|
+
fields = presign.get("fields") if isinstance(presign, Mapping) else None
|
|
583
|
+
if isinstance(fields, Mapping) and fields:
|
|
584
|
+
with open(file_obj, "rb") as fh:
|
|
585
|
+
files = {"file": (filename, fh, content_type)}
|
|
586
|
+
resp = requests.post(upload_url, data=dict(fields), files=files, timeout=300)
|
|
587
|
+
resp.raise_for_status()
|
|
588
|
+
else:
|
|
589
|
+
with open(file_obj, "rb") as fh:
|
|
590
|
+
resp = requests.put(
|
|
591
|
+
upload_url,
|
|
592
|
+
data=fh,
|
|
593
|
+
headers={"Content-Type": content_type},
|
|
594
|
+
timeout=300,
|
|
595
|
+
)
|
|
596
|
+
resp.raise_for_status()
|
|
597
|
+
|
|
598
|
+
descriptor = {
|
|
599
|
+
"object_key": object_key,
|
|
600
|
+
"original_name": filename,
|
|
601
|
+
"size": size,
|
|
602
|
+
"content_type": content_type,
|
|
603
|
+
"uploaded_at": _utcnow_iso(),
|
|
604
|
+
}
|
|
605
|
+
return descriptor
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def _pretty_size(size: int) -> str:
|
|
609
|
+
for unit in ("B", "KB", "MB", "GB"):
|
|
610
|
+
if size < 1024:
|
|
611
|
+
return f"{size:.1f} {unit}" if unit != "B" else f"{size} {unit}"
|
|
612
|
+
size /= 1024
|
|
613
|
+
return f"{size:.1f} TB"
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
def _upload_session_file(file_path: str, session_id: str) -> dict:
|
|
617
|
+
token = get_lumera_token()
|
|
618
|
+
path = pathlib.Path(file_path).expanduser().resolve()
|
|
619
|
+
if not path.is_file():
|
|
620
|
+
raise FileNotFoundError(path)
|
|
621
|
+
|
|
622
|
+
filename = path.name
|
|
623
|
+
size = path.stat().st_size
|
|
624
|
+
mimetype = mimetypes.guess_type(filename)[0] or "application/octet-stream"
|
|
625
|
+
|
|
626
|
+
headers = {"Authorization": f"token {token}", "Content-Type": "application/json"}
|
|
627
|
+
|
|
628
|
+
resp = requests.post(
|
|
629
|
+
f"{API_BASE}/sessions/{session_id}/files/upload-url",
|
|
630
|
+
json={"filename": filename, "content_type": mimetype, "size": size},
|
|
631
|
+
headers=headers,
|
|
632
|
+
timeout=30,
|
|
633
|
+
)
|
|
634
|
+
resp.raise_for_status()
|
|
635
|
+
data = resp.json()
|
|
636
|
+
upload_url: str = data["upload_url"]
|
|
637
|
+
notebook_path: str = data.get("notebook_path", "")
|
|
638
|
+
|
|
639
|
+
with open(path, "rb") as fp:
|
|
640
|
+
put = requests.put(upload_url, data=fp, headers={"Content-Type": mimetype}, timeout=300)
|
|
641
|
+
put.raise_for_status()
|
|
642
|
+
|
|
643
|
+
try:
|
|
644
|
+
requests.post(
|
|
645
|
+
f"{API_BASE}/sessions/{session_id}/enable-docs",
|
|
646
|
+
headers=headers,
|
|
647
|
+
timeout=15,
|
|
648
|
+
)
|
|
649
|
+
except Exception:
|
|
650
|
+
pass
|
|
651
|
+
|
|
652
|
+
return {"name": filename, "notebook_path": notebook_path}
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
def _upload_automation_run_file(file_path: str, run_id: str) -> dict:
|
|
656
|
+
token = get_lumera_token()
|
|
657
|
+
path = pathlib.Path(file_path).expanduser().resolve()
|
|
658
|
+
if not path.is_file():
|
|
659
|
+
raise FileNotFoundError(path)
|
|
660
|
+
|
|
661
|
+
filename = path.name
|
|
662
|
+
size = path.stat().st_size
|
|
663
|
+
mimetype = mimetypes.guess_type(filename)[0] or "application/octet-stream"
|
|
664
|
+
|
|
665
|
+
headers = {"Authorization": f"token {token}", "Content-Type": "application/json"}
|
|
666
|
+
|
|
667
|
+
resp = requests.post(
|
|
668
|
+
f"{API_BASE}/automation-runs/{run_id}/files/upload-url",
|
|
669
|
+
json={"filename": filename, "content_type": mimetype, "size": size},
|
|
670
|
+
headers=headers,
|
|
671
|
+
timeout=30,
|
|
672
|
+
)
|
|
673
|
+
resp.raise_for_status()
|
|
674
|
+
data = resp.json()
|
|
675
|
+
upload_url = data["upload_url"]
|
|
676
|
+
file_ref = data.get("file") if isinstance(data, dict) else None
|
|
677
|
+
|
|
678
|
+
with open(path, "rb") as fp:
|
|
679
|
+
put = requests.put(upload_url, data=fp, headers={"Content-Type": mimetype}, timeout=300)
|
|
680
|
+
put.raise_for_status()
|
|
681
|
+
|
|
682
|
+
if isinstance(file_ref, dict):
|
|
683
|
+
return file_ref
|
|
684
|
+
run_path = (
|
|
685
|
+
data.get("run_path") or data.get("path") or f"/lumera-files/agent_runs/{run_id}/{filename}"
|
|
686
|
+
)
|
|
687
|
+
return {
|
|
688
|
+
"name": filename,
|
|
689
|
+
"run_path": run_path,
|
|
690
|
+
"object_name": data.get("object_name"),
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
def _upload_document(file_path: str) -> dict:
|
|
695
|
+
token = get_lumera_token()
|
|
696
|
+
path = pathlib.Path(file_path).expanduser().resolve()
|
|
697
|
+
if not path.is_file():
|
|
698
|
+
raise FileNotFoundError(path)
|
|
699
|
+
|
|
700
|
+
filename = path.name
|
|
701
|
+
size = path.stat().st_size
|
|
702
|
+
mimetype = mimetypes.guess_type(filename)[0] or "application/octet-stream"
|
|
703
|
+
pretty = _pretty_size(size)
|
|
704
|
+
|
|
705
|
+
headers = {"Authorization": f"token {token}", "Content-Type": "application/json"}
|
|
706
|
+
documents_base = f"{API_BASE}/documents"
|
|
707
|
+
|
|
708
|
+
resp = requests.post(
|
|
709
|
+
documents_base,
|
|
710
|
+
json={
|
|
711
|
+
"title": filename,
|
|
712
|
+
"content": f"File to be uploaded: {filename} ({pretty})",
|
|
713
|
+
"type": mimetype.split("/")[-1],
|
|
714
|
+
"status": "uploading",
|
|
715
|
+
},
|
|
716
|
+
headers=headers,
|
|
717
|
+
timeout=30,
|
|
718
|
+
)
|
|
719
|
+
resp.raise_for_status()
|
|
720
|
+
doc = resp.json()
|
|
721
|
+
doc_id = doc["id"]
|
|
722
|
+
|
|
723
|
+
resp = requests.post(
|
|
724
|
+
f"{documents_base}/{doc_id}/upload-url",
|
|
725
|
+
json={"filename": filename, "content_type": mimetype, "size": size},
|
|
726
|
+
headers=headers,
|
|
727
|
+
timeout=30,
|
|
728
|
+
)
|
|
729
|
+
resp.raise_for_status()
|
|
730
|
+
upload_url: str = resp.json()["upload_url"]
|
|
731
|
+
|
|
732
|
+
with open(path, "rb") as fp:
|
|
733
|
+
put = requests.put(upload_url, data=fp, headers={"Content-Type": mimetype}, timeout=300)
|
|
734
|
+
put.raise_for_status()
|
|
735
|
+
|
|
736
|
+
resp = requests.put(
|
|
737
|
+
f"{documents_base}/{doc_id}",
|
|
738
|
+
json={
|
|
739
|
+
"status": "uploaded",
|
|
740
|
+
"content": f"Uploaded file: {filename} ({pretty})",
|
|
741
|
+
},
|
|
742
|
+
headers=headers,
|
|
743
|
+
timeout=30,
|
|
744
|
+
)
|
|
745
|
+
resp.raise_for_status()
|
|
746
|
+
return resp.json()
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
__all__ = [
|
|
750
|
+
"API_BASE",
|
|
751
|
+
"BASE_URL_ENV",
|
|
752
|
+
"DEFAULT_MOUNT_ROOT",
|
|
753
|
+
"ENV_PATH",
|
|
754
|
+
"LEGACY_MOUNT_ROOT",
|
|
755
|
+
"MOUNT_ROOT",
|
|
756
|
+
"TOKEN_ENV",
|
|
757
|
+
"_api_request",
|
|
758
|
+
"_api_url",
|
|
759
|
+
"_default_provenance",
|
|
760
|
+
"_ensure_json_string",
|
|
761
|
+
"_ensure_mapping",
|
|
762
|
+
"_ensure_sequence",
|
|
763
|
+
"_is_sequence",
|
|
764
|
+
"_normalize_mount_path",
|
|
765
|
+
"_pretty_size",
|
|
766
|
+
"_prepare_automation_inputs",
|
|
767
|
+
"_record_mutation",
|
|
768
|
+
"_upload_automation_files",
|
|
769
|
+
"_upload_automation_run_file",
|
|
770
|
+
"_upload_file_to_presigned",
|
|
771
|
+
"_upload_document",
|
|
772
|
+
"_upload_session_file",
|
|
773
|
+
"_upload_lumera_file",
|
|
774
|
+
"open_file",
|
|
775
|
+
"resolve_path",
|
|
776
|
+
"to_filerefs",
|
|
777
|
+
"LumeraAPIError",
|
|
778
|
+
"RecordNotUniqueError",
|
|
779
|
+
"get_access_token",
|
|
780
|
+
"get_google_access_token",
|
|
781
|
+
"log_timed",
|
|
782
|
+
]
|