lumera 0.39.0.dev1__tar.gz → 0.41.0.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/PKG-INFO +1 -1
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/_utils.py +38 -18
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/integrations/__init__.py +3 -1
- lumera-0.41.0.dev1/lumera/integrations/kafka.py +242 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera.egg-info/PKG-INFO +1 -1
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera.egg-info/SOURCES.txt +2 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/pyproject.toml +1 -1
- lumera-0.41.0.dev1/tests/test_integrations_kafka.py +126 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_sdk.py +100 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/__init__.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/_live.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/agents.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/automations.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/email.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/exceptions.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/files.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/flags.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/function_resources.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/__init__.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/__main__.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_artifacts.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_config.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_context.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_contract.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_dependencies.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_errors.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_execution.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_jsonschema.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_registry.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_remote.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_runner.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_runtime.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/testing.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/google.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/integrations/bill.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/integrations/finch.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/integrations/google.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/integrations/netsuite.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/integrations/salesforce.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/integrations/slack.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/integrations/snowflake.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/llm.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/pb.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/__init__.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/__main__.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_api_child.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_archive.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_builder.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_config.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_contracts.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_errors.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_function_child.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_projection.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_runtime.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_runtime_child.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_schemas.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_target.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_worker_child.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/sdk.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/storage.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/webhooks.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera.egg-info/dependency_links.txt +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera.egg-info/requires.txt +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera.egg-info/top_level.txt +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/setup.cfg +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_agents_live.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_automations.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_flags.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_function_resources.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_integrations_bill.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_integrations_finch.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_integrations_google.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_integrations_netsuite.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_integrations_salesforce.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_integrations_snowflake.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_lazy_imports.py +0 -0
- {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_webhooks.py +0 -0
|
@@ -205,7 +205,7 @@ def _parse_expiry(expires_at: int | float | str | None) -> float:
|
|
|
205
205
|
raise TypeError(f"Unsupported expires_at format: {type(expires_at)!r}")
|
|
206
206
|
|
|
207
207
|
|
|
208
|
-
def _fetch_access_token(provider: str) -> tuple[str, float]:
|
|
208
|
+
def _fetch_access_token(provider: str) -> tuple[str, float, str]:
|
|
209
209
|
provider = provider.lower().strip()
|
|
210
210
|
if not provider:
|
|
211
211
|
raise ValueError("provider is required")
|
|
@@ -218,15 +218,16 @@ def _fetch_access_token(provider: str) -> tuple[str, float]:
|
|
|
218
218
|
data = resp.json()
|
|
219
219
|
access_token = data.get("access_token")
|
|
220
220
|
expires_at = data.get("expires_at")
|
|
221
|
+
credential_scope = str(data.get("credential_scope") or "")
|
|
221
222
|
|
|
222
223
|
if not access_token:
|
|
223
224
|
raise RuntimeError(f"Malformed response from Lumera when fetching {provider} access token")
|
|
224
225
|
|
|
225
226
|
expiry_ts = _parse_expiry(expires_at)
|
|
226
|
-
return access_token, expiry_ts
|
|
227
|
+
return access_token, expiry_ts, credential_scope
|
|
227
228
|
|
|
228
229
|
|
|
229
|
-
def get_access_token(provider: str, min_valid_seconds: int = 900) -> str:
|
|
230
|
+
def get_access_token(provider: str, min_valid_seconds: int = 900, *, cache: bool = True) -> str:
|
|
230
231
|
"""Get the raw access token string for *provider*.
|
|
231
232
|
|
|
232
233
|
.. deprecated::
|
|
@@ -239,8 +240,8 @@ def get_access_token(provider: str, min_valid_seconds: int = 900) -> str:
|
|
|
239
240
|
providers this is the token itself; for basic-auth providers it is the
|
|
240
241
|
base64-encoded ``username:password``.
|
|
241
242
|
|
|
242
|
-
|
|
243
|
-
|
|
243
|
+
Shared results are cached in-process; delegated results are never cached.
|
|
244
|
+
Pass ``cache=False`` to bypass and avoid populating shared caches.
|
|
244
245
|
"""
|
|
245
246
|
global _token_cache
|
|
246
247
|
|
|
@@ -250,19 +251,22 @@ def get_access_token(provider: str, min_valid_seconds: int = 900) -> str:
|
|
|
250
251
|
|
|
251
252
|
now = _time.time()
|
|
252
253
|
|
|
253
|
-
cached = _token_cache.get(provider)
|
|
254
|
+
cached = _token_cache.get(provider) if cache else None
|
|
254
255
|
if cached is not None:
|
|
255
256
|
access_token, expiry_ts = cached
|
|
256
257
|
if (expiry_ts - now) >= min_valid_seconds:
|
|
257
258
|
return access_token
|
|
258
259
|
|
|
259
|
-
access_token, expiry_ts = _fetch_access_token(provider)
|
|
260
|
-
|
|
260
|
+
access_token, expiry_ts, credential_scope = _fetch_access_token(provider)
|
|
261
|
+
if credential_scope == "delegated":
|
|
262
|
+
_evict_credential_caches(provider)
|
|
263
|
+
elif cache:
|
|
264
|
+
_token_cache[provider] = (access_token, expiry_ts)
|
|
261
265
|
return access_token
|
|
262
266
|
|
|
263
267
|
|
|
264
|
-
def get_google_access_token(min_valid_seconds: int = 900) -> str:
|
|
265
|
-
return get_access_token("google", min_valid_seconds=min_valid_seconds)
|
|
268
|
+
def get_google_access_token(min_valid_seconds: int = 900, *, cache: bool = True) -> str:
|
|
269
|
+
return get_access_token("google", min_valid_seconds=min_valid_seconds, cache=cache)
|
|
266
270
|
|
|
267
271
|
|
|
268
272
|
@dataclass(frozen=True)
|
|
@@ -305,6 +309,9 @@ class Credentials:
|
|
|
305
309
|
extra_headers: dict[str, str] | None = None
|
|
306
310
|
"""Additional provider-required HTTP headers."""
|
|
307
311
|
|
|
312
|
+
credential_scope: str = ""
|
|
313
|
+
"""Credential isolation mode. ``"delegated"`` values are never cached."""
|
|
314
|
+
|
|
308
315
|
def as_header_dict(self) -> dict[str, str]:
|
|
309
316
|
"""Return headers suitable for ``requests.get(headers=...)``."""
|
|
310
317
|
headers = {self.header_name: self.header_value}
|
|
@@ -316,6 +323,13 @@ class Credentials:
|
|
|
316
323
|
_credentials_cache: dict[tuple[str, str], tuple[Credentials, float]] = {}
|
|
317
324
|
|
|
318
325
|
|
|
326
|
+
def _evict_credential_caches(api_name: str) -> None:
|
|
327
|
+
_token_cache.pop(api_name, None)
|
|
328
|
+
for key in tuple(_credentials_cache):
|
|
329
|
+
if key[0] == api_name:
|
|
330
|
+
_credentials_cache.pop(key, None)
|
|
331
|
+
|
|
332
|
+
|
|
319
333
|
def _fetch_credentials(api_name: str, provider: str | None = None) -> Credentials:
|
|
320
334
|
api_name = api_name.lower().strip()
|
|
321
335
|
if not api_name:
|
|
@@ -340,6 +354,7 @@ def _fetch_credentials(api_name: str, provider: str | None = None) -> Credential
|
|
|
340
354
|
# Collect provider-specific extra fields (anything beyond the standard set).
|
|
341
355
|
_standard_keys = {
|
|
342
356
|
"access_token",
|
|
357
|
+
"credential_scope",
|
|
343
358
|
"auth_type",
|
|
344
359
|
"header_name",
|
|
345
360
|
"header_value",
|
|
@@ -362,6 +377,7 @@ def _fetch_credentials(api_name: str, provider: str | None = None) -> Credential
|
|
|
362
377
|
token=access_token,
|
|
363
378
|
provider=str(data.get("provider") or provider_hint or api_name),
|
|
364
379
|
header_name=data.get("header_name", "Authorization"),
|
|
380
|
+
credential_scope=str(data.get("credential_scope") or ""),
|
|
365
381
|
header_value=data.get("header_value", f"Bearer {access_token}"),
|
|
366
382
|
username=data.get("username"),
|
|
367
383
|
expires_at=expiry_ts if expiry_ts != float("inf") else None,
|
|
@@ -374,14 +390,16 @@ def get_credentials(
|
|
|
374
390
|
api_name: str | None = None,
|
|
375
391
|
provider: str | None = None,
|
|
376
392
|
min_valid_seconds: int = 900,
|
|
393
|
+
*,
|
|
394
|
+
cache: bool = True,
|
|
377
395
|
) -> Credentials:
|
|
378
396
|
"""Get a :class:`Credentials` object for *api_name*.
|
|
379
397
|
|
|
380
398
|
This is the recommended way to obtain integration credentials. The
|
|
381
399
|
returned object includes the auth type, ready-to-use HTTP headers, and
|
|
382
|
-
(where applicable) the username.
|
|
383
|
-
|
|
384
|
-
|
|
400
|
+
(where applicable) the username. Shared results are cached in-process;
|
|
401
|
+
delegated results are never cached. Pass ``cache=False`` to bypass and
|
|
402
|
+
avoid populating shared caches.
|
|
385
403
|
|
|
386
404
|
Works with any auth type configured in the Lumera integrations page
|
|
387
405
|
(bearer token, basic auth, etc.).
|
|
@@ -432,7 +450,7 @@ def get_credentials(
|
|
|
432
450
|
|
|
433
451
|
now = _time.time()
|
|
434
452
|
cache_key = (api_name, provider_hint)
|
|
435
|
-
cached = _credentials_cache.get(cache_key)
|
|
453
|
+
cached = _credentials_cache.get(cache_key) if cache else None
|
|
436
454
|
if cached is not None:
|
|
437
455
|
creds, expiry_ts = cached
|
|
438
456
|
if (expiry_ts - now) >= min_valid_seconds:
|
|
@@ -440,10 +458,12 @@ def get_credentials(
|
|
|
440
458
|
|
|
441
459
|
creds = _fetch_credentials(api_name, provider=provider_hint or None)
|
|
442
460
|
expiry_ts = creds.expires_at if creds.expires_at is not None else float("inf")
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
461
|
+
if creds.credential_scope == "delegated":
|
|
462
|
+
_evict_credential_caches(api_name)
|
|
463
|
+
elif cache:
|
|
464
|
+
_credentials_cache[cache_key] = (creds, expiry_ts)
|
|
465
|
+
# Keep the legacy _token_cache in sync so get_access_token stays fast.
|
|
466
|
+
_token_cache[api_name] = (creds.token, expiry_ts)
|
|
447
467
|
|
|
448
468
|
return creds
|
|
449
469
|
|
|
@@ -45,6 +45,7 @@ Available integrations:
|
|
|
45
45
|
- ``bill`` - BILL AP/AR and BILL Spend & Expense APIs
|
|
46
46
|
- ``finch`` - Finch organization and payroll API with batch handling
|
|
47
47
|
- ``google`` - Google APIs (Docs, Slides, Sheets, Drive)
|
|
48
|
+
- ``kafka`` - Bounded, on-demand Kafka produce and consumer-group polls
|
|
48
49
|
- ``netsuite`` - NetSuite REST API and SuiteQL (M2M or OAuth)
|
|
49
50
|
- ``salesforce`` - Salesforce REST API via customer-owned Client Credentials
|
|
50
51
|
- ``slack`` - Slack (messages, files, reactions)
|
|
@@ -69,7 +70,7 @@ from .._utils import (
|
|
|
69
70
|
list_integration_connections,
|
|
70
71
|
list_integrations,
|
|
71
72
|
)
|
|
72
|
-
from . import bill, finch, google, netsuite, salesforce, slack, snowflake
|
|
73
|
+
from . import bill, finch, google, kafka, netsuite, salesforce, slack, snowflake
|
|
73
74
|
|
|
74
75
|
__all__ = [
|
|
75
76
|
"Credentials",
|
|
@@ -80,6 +81,7 @@ __all__ = [
|
|
|
80
81
|
"bill",
|
|
81
82
|
"finch",
|
|
82
83
|
"google",
|
|
84
|
+
"kafka",
|
|
83
85
|
"netsuite",
|
|
84
86
|
"salesforce",
|
|
85
87
|
"slack",
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
"""Bounded, server-mediated Kafka produce and consume operations.
|
|
2
|
+
|
|
3
|
+
Kafka SASL credentials never enter the Python runtime. Each call asks Lumera's
|
|
4
|
+
backend to create a temporary Kafka client, perform one acknowledged produce
|
|
5
|
+
or bounded consumer-group poll, and close the client again.
|
|
6
|
+
|
|
7
|
+
Example::
|
|
8
|
+
|
|
9
|
+
from lumera.integrations.kafka import get_client
|
|
10
|
+
|
|
11
|
+
kafka = get_client("finance_kafka")
|
|
12
|
+
kafka.push("finance.events", {"invoice_id": "inv_123"})
|
|
13
|
+
batch = kafka.pull("finance.events", group_id="lumera-ap-import")
|
|
14
|
+
for record in batch.records:
|
|
15
|
+
print(record.json())
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import base64
|
|
21
|
+
import json
|
|
22
|
+
from collections.abc import Iterable, Mapping
|
|
23
|
+
from dataclasses import dataclass
|
|
24
|
+
from datetime import datetime
|
|
25
|
+
from typing import Any, Literal
|
|
26
|
+
from urllib.parse import quote as _url_quote
|
|
27
|
+
|
|
28
|
+
from lumera._utils import (
|
|
29
|
+
API_BASE,
|
|
30
|
+
_api_headers,
|
|
31
|
+
_get_session,
|
|
32
|
+
_raise_if_project_scope_denied,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
_MAX_TIMEOUT_SECONDS = 30
|
|
36
|
+
_MAX_PULL_RECORDS = 500
|
|
37
|
+
_TRANSPORT_GRACE_SECONDS = 5
|
|
38
|
+
|
|
39
|
+
KafkaValue = bytes | bytearray | memoryview | str | dict[str, Any] | list[Any]
|
|
40
|
+
KafkaHeaderValue = bytes | bytearray | memoryview | str
|
|
41
|
+
KafkaHeaders = Mapping[str, KafkaHeaderValue] | Iterable[tuple[str, KafkaHeaderValue]]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class KafkaAPIError(Exception):
|
|
45
|
+
"""Raised when Lumera cannot complete a Kafka operation."""
|
|
46
|
+
|
|
47
|
+
def __init__(self, status_code: int, body: str) -> None:
|
|
48
|
+
self.status_code = status_code
|
|
49
|
+
self.body = body
|
|
50
|
+
super().__init__(f"Kafka integration error ({status_code}): {body}")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class KafkaHeader:
|
|
55
|
+
key: str
|
|
56
|
+
value: bytes
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True)
|
|
60
|
+
class KafkaRecord:
|
|
61
|
+
topic: str
|
|
62
|
+
partition: int
|
|
63
|
+
offset: int
|
|
64
|
+
timestamp: datetime
|
|
65
|
+
key: bytes | None
|
|
66
|
+
value: bytes
|
|
67
|
+
headers: tuple[KafkaHeader, ...]
|
|
68
|
+
|
|
69
|
+
def text(self, encoding: str = "utf-8") -> str:
|
|
70
|
+
"""Decode the record value as text."""
|
|
71
|
+
return self.value.decode(encoding)
|
|
72
|
+
|
|
73
|
+
def json(self) -> object:
|
|
74
|
+
"""Decode the record value as UTF-8 JSON."""
|
|
75
|
+
return json.loads(self.value)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass(frozen=True)
|
|
79
|
+
class KafkaPushResult:
|
|
80
|
+
topic: str
|
|
81
|
+
partition: int
|
|
82
|
+
offset: int
|
|
83
|
+
timestamp: datetime
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass(frozen=True)
|
|
87
|
+
class KafkaPullResult:
|
|
88
|
+
records: tuple[KafkaRecord, ...]
|
|
89
|
+
committed: bool
|
|
90
|
+
timed_out: bool
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class KafkaClient:
|
|
94
|
+
"""On-demand Kafka client backed by a project-scoped Lumera connection."""
|
|
95
|
+
|
|
96
|
+
def __init__(self, api_name: str = "kafka", timeout: int = 10) -> None:
|
|
97
|
+
if not api_name.strip():
|
|
98
|
+
raise ValueError("api_name is required")
|
|
99
|
+
self.api_name = api_name.strip()
|
|
100
|
+
self.timeout = _validate_timeout(timeout)
|
|
101
|
+
|
|
102
|
+
def push(
|
|
103
|
+
self,
|
|
104
|
+
topic: str,
|
|
105
|
+
value: KafkaValue,
|
|
106
|
+
*,
|
|
107
|
+
key: KafkaHeaderValue | None = None,
|
|
108
|
+
headers: KafkaHeaders | None = None,
|
|
109
|
+
timeout: int | None = None,
|
|
110
|
+
) -> KafkaPushResult:
|
|
111
|
+
"""Produce one record and wait for Kafka's acknowledged offset.
|
|
112
|
+
|
|
113
|
+
``dict`` and ``list`` values are serialized as compact UTF-8 JSON.
|
|
114
|
+
Strings are UTF-8 encoded; bytes remain unchanged.
|
|
115
|
+
"""
|
|
116
|
+
operation_timeout = self.timeout if timeout is None else _validate_timeout(timeout)
|
|
117
|
+
payload: dict[str, Any] = {
|
|
118
|
+
"topic": topic,
|
|
119
|
+
"value_base64": _encode_base64(value, allow_json=True),
|
|
120
|
+
"timeout_seconds": operation_timeout,
|
|
121
|
+
}
|
|
122
|
+
if key is not None:
|
|
123
|
+
payload["key_base64"] = _encode_base64(key, allow_json=False)
|
|
124
|
+
if headers is not None:
|
|
125
|
+
items = headers.items() if isinstance(headers, Mapping) else headers
|
|
126
|
+
payload["headers"] = [
|
|
127
|
+
{"key": name, "value_base64": _encode_base64(item, allow_json=False)}
|
|
128
|
+
for name, item in items
|
|
129
|
+
]
|
|
130
|
+
|
|
131
|
+
data = self._request("push", payload, operation_timeout)
|
|
132
|
+
return KafkaPushResult(
|
|
133
|
+
topic=str(data["topic"]),
|
|
134
|
+
partition=int(data["partition"]),
|
|
135
|
+
offset=int(data["offset"]),
|
|
136
|
+
timestamp=_parse_timestamp(data["timestamp"]),
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
def pull(
|
|
140
|
+
self,
|
|
141
|
+
topic: str,
|
|
142
|
+
*,
|
|
143
|
+
group_id: str,
|
|
144
|
+
max_records: int = 100,
|
|
145
|
+
timeout: int | None = None,
|
|
146
|
+
offset_reset: Literal["earliest", "latest"] = "earliest",
|
|
147
|
+
commit: bool = True,
|
|
148
|
+
) -> KafkaPullResult:
|
|
149
|
+
"""Poll once with a temporary consumer and optionally commit offsets.
|
|
150
|
+
|
|
151
|
+
Use a stable ``group_id`` across invocations. ``offset_reset`` matters
|
|
152
|
+
only when the group has no previously committed topic offset.
|
|
153
|
+
"""
|
|
154
|
+
if not group_id.strip():
|
|
155
|
+
raise ValueError("group_id is required")
|
|
156
|
+
if not 1 <= max_records <= _MAX_PULL_RECORDS:
|
|
157
|
+
raise ValueError(f"max_records must be between 1 and {_MAX_PULL_RECORDS}")
|
|
158
|
+
if offset_reset not in {"earliest", "latest"}:
|
|
159
|
+
raise ValueError("offset_reset must be earliest or latest")
|
|
160
|
+
operation_timeout = self.timeout if timeout is None else _validate_timeout(timeout)
|
|
161
|
+
data = self._request(
|
|
162
|
+
"pull",
|
|
163
|
+
{
|
|
164
|
+
"topic": topic,
|
|
165
|
+
"group_id": group_id,
|
|
166
|
+
"max_records": max_records,
|
|
167
|
+
"timeout_seconds": operation_timeout,
|
|
168
|
+
"offset_reset": offset_reset,
|
|
169
|
+
"commit": commit,
|
|
170
|
+
},
|
|
171
|
+
operation_timeout,
|
|
172
|
+
)
|
|
173
|
+
records = tuple(_parse_record(item) for item in data.get("records", []))
|
|
174
|
+
return KafkaPullResult(
|
|
175
|
+
records=records,
|
|
176
|
+
committed=bool(data.get("committed", False)),
|
|
177
|
+
timed_out=bool(data.get("timed_out", False)),
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
def _request(self, operation: str, payload: dict[str, Any], timeout: int) -> dict[str, Any]:
|
|
181
|
+
url = f"{API_BASE}/connections/{_url_quote(self.api_name, safe='')}/kafka/{operation}"
|
|
182
|
+
response = _get_session().post(
|
|
183
|
+
url,
|
|
184
|
+
json=payload,
|
|
185
|
+
headers=_api_headers(content_type="application/json"),
|
|
186
|
+
timeout=timeout + _TRANSPORT_GRACE_SECONDS,
|
|
187
|
+
)
|
|
188
|
+
_raise_if_project_scope_denied(response, self.api_name, url)
|
|
189
|
+
if not response.ok:
|
|
190
|
+
raise KafkaAPIError(response.status_code, response.text[:500])
|
|
191
|
+
data = response.json()
|
|
192
|
+
if not isinstance(data, dict):
|
|
193
|
+
raise KafkaAPIError(response.status_code, "Invalid Kafka integration response")
|
|
194
|
+
return data
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def get_client(api_name: str = "kafka", timeout: int = 10) -> KafkaClient:
|
|
198
|
+
"""Create a lightweight, on-demand client for one Kafka connection."""
|
|
199
|
+
return KafkaClient(api_name=api_name, timeout=timeout)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _validate_timeout(timeout: int) -> int:
|
|
203
|
+
if not isinstance(timeout, int) or not 1 <= timeout <= _MAX_TIMEOUT_SECONDS:
|
|
204
|
+
raise ValueError(f"timeout must be between 1 and {_MAX_TIMEOUT_SECONDS} seconds")
|
|
205
|
+
return timeout
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _encode_base64(value: KafkaValue | KafkaHeaderValue, *, allow_json: bool) -> str:
|
|
209
|
+
if isinstance(value, str):
|
|
210
|
+
raw = value.encode("utf-8")
|
|
211
|
+
elif isinstance(value, (bytes, bytearray, memoryview)):
|
|
212
|
+
raw = bytes(value)
|
|
213
|
+
elif allow_json and isinstance(value, (dict, list)):
|
|
214
|
+
raw = json.dumps(value, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
|
215
|
+
else:
|
|
216
|
+
raise TypeError("Kafka values must be bytes, str, dict, or list")
|
|
217
|
+
return base64.b64encode(raw).decode("ascii")
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _parse_timestamp(value: object) -> datetime:
|
|
221
|
+
if not isinstance(value, str):
|
|
222
|
+
raise ValueError("Kafka timestamp must be an RFC 3339 string")
|
|
223
|
+
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _parse_record(data: dict[str, Any]) -> KafkaRecord:
|
|
227
|
+
key = data.get("key_base64")
|
|
228
|
+
return KafkaRecord(
|
|
229
|
+
topic=str(data["topic"]),
|
|
230
|
+
partition=int(data["partition"]),
|
|
231
|
+
offset=int(data["offset"]),
|
|
232
|
+
timestamp=_parse_timestamp(data["timestamp"]),
|
|
233
|
+
key=base64.b64decode(key, validate=True) if isinstance(key, str) else None,
|
|
234
|
+
value=base64.b64decode(str(data["value_base64"]), validate=True),
|
|
235
|
+
headers=tuple(
|
|
236
|
+
KafkaHeader(
|
|
237
|
+
key=str(header["key"]),
|
|
238
|
+
value=base64.b64decode(str(header["value_base64"]), validate=True),
|
|
239
|
+
)
|
|
240
|
+
for header in data.get("headers", [])
|
|
241
|
+
),
|
|
242
|
+
)
|
|
@@ -39,6 +39,7 @@ lumera/integrations/__init__.py
|
|
|
39
39
|
lumera/integrations/bill.py
|
|
40
40
|
lumera/integrations/finch.py
|
|
41
41
|
lumera/integrations/google.py
|
|
42
|
+
lumera/integrations/kafka.py
|
|
42
43
|
lumera/integrations/netsuite.py
|
|
43
44
|
lumera/integrations/salesforce.py
|
|
44
45
|
lumera/integrations/slack.py
|
|
@@ -65,6 +66,7 @@ tests/test_function_resources.py
|
|
|
65
66
|
tests/test_integrations_bill.py
|
|
66
67
|
tests/test_integrations_finch.py
|
|
67
68
|
tests/test_integrations_google.py
|
|
69
|
+
tests/test_integrations_kafka.py
|
|
68
70
|
tests/test_integrations_netsuite.py
|
|
69
71
|
tests/test_integrations_salesforce.py
|
|
70
72
|
tests/test_integrations_snowflake.py
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
from lumera.integrations import kafka
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class FakeResponse:
|
|
12
|
+
def __init__(self, data: dict[str, Any], status_code: int = 200) -> None:
|
|
13
|
+
self._data = data
|
|
14
|
+
self.status_code = status_code
|
|
15
|
+
self.ok = status_code < 400
|
|
16
|
+
self.text = str(data)
|
|
17
|
+
|
|
18
|
+
def json(self) -> dict[str, Any]:
|
|
19
|
+
return self._data
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class FakeSession:
|
|
23
|
+
def __init__(self, response: FakeResponse) -> None:
|
|
24
|
+
self.response = response
|
|
25
|
+
self.url = ""
|
|
26
|
+
self.kwargs: dict[str, Any] = {}
|
|
27
|
+
|
|
28
|
+
def post(self, url: str, **kwargs: object) -> FakeResponse:
|
|
29
|
+
self.url, self.kwargs = url, kwargs
|
|
30
|
+
return self.response
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def install_session(monkeypatch: pytest.MonkeyPatch, response: FakeResponse) -> FakeSession:
|
|
34
|
+
session = FakeSession(response)
|
|
35
|
+
monkeypatch.setattr(kafka, "_get_session", lambda: session)
|
|
36
|
+
monkeypatch.setattr(kafka, "_api_headers", lambda **_: {"X-Lumera-Project": "finance"})
|
|
37
|
+
return session
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_push_json_is_binary_safe_and_uses_project_scoped_backend(
|
|
41
|
+
monkeypatch: pytest.MonkeyPatch,
|
|
42
|
+
) -> None:
|
|
43
|
+
session = install_session(
|
|
44
|
+
monkeypatch,
|
|
45
|
+
FakeResponse(
|
|
46
|
+
{
|
|
47
|
+
"topic": "finance.events",
|
|
48
|
+
"partition": 2,
|
|
49
|
+
"offset": 41,
|
|
50
|
+
"timestamp": "2026-08-25T10:30:00Z",
|
|
51
|
+
}
|
|
52
|
+
),
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
result = kafka.get_client("finance_kafka").push(
|
|
56
|
+
"finance.events",
|
|
57
|
+
{"invoice_id": "inv_123"},
|
|
58
|
+
key=b"invoice-key",
|
|
59
|
+
headers={"trace": "abc"},
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
assert session.url.endswith("/connections/finance_kafka/kafka/push")
|
|
63
|
+
payload = session.kwargs["json"]
|
|
64
|
+
assert base64.b64decode(payload["value_base64"]) == b'{"invoice_id":"inv_123"}'
|
|
65
|
+
assert base64.b64decode(payload["key_base64"]) == b"invoice-key"
|
|
66
|
+
assert base64.b64decode(payload["headers"][0]["value_base64"]) == b"abc"
|
|
67
|
+
assert session.kwargs["headers"]["X-Lumera-Project"] == "finance"
|
|
68
|
+
assert result.partition == 2
|
|
69
|
+
assert result.offset == 41
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_pull_decodes_records_and_preserves_explicit_commit_choice(
|
|
73
|
+
monkeypatch: pytest.MonkeyPatch,
|
|
74
|
+
) -> None:
|
|
75
|
+
session = install_session(
|
|
76
|
+
monkeypatch,
|
|
77
|
+
FakeResponse(
|
|
78
|
+
{
|
|
79
|
+
"records": [
|
|
80
|
+
{
|
|
81
|
+
"topic": "finance.events",
|
|
82
|
+
"partition": 1,
|
|
83
|
+
"offset": 42,
|
|
84
|
+
"timestamp": "2026-08-25T10:31:00Z",
|
|
85
|
+
"key_base64": None,
|
|
86
|
+
"value_base64": base64.b64encode(b'{"amount":25}').decode(),
|
|
87
|
+
"headers": [
|
|
88
|
+
{
|
|
89
|
+
"key": "trace",
|
|
90
|
+
"value_base64": base64.b64encode(b"xyz").decode(),
|
|
91
|
+
}
|
|
92
|
+
],
|
|
93
|
+
}
|
|
94
|
+
],
|
|
95
|
+
"committed": False,
|
|
96
|
+
"timed_out": False,
|
|
97
|
+
}
|
|
98
|
+
),
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
batch = kafka.get_client("finance_kafka").pull(
|
|
102
|
+
"finance.events", group_id="lumera-ap-import", max_records=25, commit=False
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
assert session.url.endswith("/connections/finance_kafka/kafka/pull")
|
|
106
|
+
assert session.kwargs["json"]["group_id"] == "lumera-ap-import"
|
|
107
|
+
assert session.kwargs["json"]["commit"] is False
|
|
108
|
+
assert batch.committed is False
|
|
109
|
+
assert batch.records[0].json() == {"amount": 25}
|
|
110
|
+
assert batch.records[0].headers[0].value == b"xyz"
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def test_kafka_client_rejects_unbounded_requests() -> None:
|
|
114
|
+
with pytest.raises(ValueError, match="between 1 and 30"):
|
|
115
|
+
kafka.get_client(timeout=60)
|
|
116
|
+
with pytest.raises(ValueError, match="between 1 and 500"):
|
|
117
|
+
kafka.get_client().pull("finance.events", group_id="finance", max_records=501)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_kafka_provider_errors_are_typed(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
121
|
+
install_session(monkeypatch, FakeResponse({"message": "broker rejected request"}, 502))
|
|
122
|
+
|
|
123
|
+
with pytest.raises(kafka.KafkaAPIError) as exc:
|
|
124
|
+
kafka.get_client().push("finance.events", b"message")
|
|
125
|
+
|
|
126
|
+
assert exc.value.status_code == 502
|
|
@@ -648,6 +648,106 @@ def test_get_credentials_cache_key_includes_provider_hint(
|
|
|
648
648
|
assert calls == [{"provider": "slack"}, {"provider": "teams"}]
|
|
649
649
|
|
|
650
650
|
|
|
651
|
+
def test_delegated_credentials_are_never_cached(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
652
|
+
monkeypatch.setenv(sdk.TOKEN_ENV, "tok")
|
|
653
|
+
_utils._credentials_cache.clear()
|
|
654
|
+
_utils._token_cache.clear()
|
|
655
|
+
calls = 0
|
|
656
|
+
|
|
657
|
+
class MockSession:
|
|
658
|
+
def get(self, url: str, **kwargs: object) -> DummyResponse:
|
|
659
|
+
nonlocal calls
|
|
660
|
+
del url, kwargs
|
|
661
|
+
calls += 1
|
|
662
|
+
return DummyResponse(
|
|
663
|
+
status_code=200,
|
|
664
|
+
json_data={
|
|
665
|
+
"access_token": f"member-token-{calls}",
|
|
666
|
+
"provider": "google",
|
|
667
|
+
"credential_scope": "delegated",
|
|
668
|
+
"expires_at": "2030-01-01T00:00:00Z",
|
|
669
|
+
},
|
|
670
|
+
)
|
|
671
|
+
|
|
672
|
+
monkeypatch.setattr(_utils, "_get_session", lambda: MockSession())
|
|
673
|
+
|
|
674
|
+
tokens = [_utils.get_credentials("google_drive", provider="google").token for _ in range(3)]
|
|
675
|
+
|
|
676
|
+
assert tokens == ["member-token-1", "member-token-2", "member-token-3"]
|
|
677
|
+
assert calls == 3
|
|
678
|
+
assert _utils._credentials_cache == {}
|
|
679
|
+
assert _utils._token_cache == {}
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
def test_cache_false_bootstraps_and_evicts_delegated_credentials(
|
|
683
|
+
monkeypatch: pytest.MonkeyPatch,
|
|
684
|
+
) -> None:
|
|
685
|
+
monkeypatch.setenv(sdk.TOKEN_ENV, "tok")
|
|
686
|
+
_utils._credentials_cache.clear()
|
|
687
|
+
_utils._token_cache.clear()
|
|
688
|
+
responses = ["", "delegated", "delegated"]
|
|
689
|
+
calls = 0
|
|
690
|
+
|
|
691
|
+
class MockSession:
|
|
692
|
+
def get(self, url: str, **kwargs: object) -> DummyResponse:
|
|
693
|
+
nonlocal calls
|
|
694
|
+
del url, kwargs
|
|
695
|
+
scope = responses[calls]
|
|
696
|
+
calls += 1
|
|
697
|
+
return DummyResponse(
|
|
698
|
+
status_code=200,
|
|
699
|
+
json_data={
|
|
700
|
+
"access_token": f"token-{calls}",
|
|
701
|
+
"provider": "google",
|
|
702
|
+
"credential_scope": scope,
|
|
703
|
+
"expires_at": "2030-01-01T00:00:00Z",
|
|
704
|
+
},
|
|
705
|
+
)
|
|
706
|
+
|
|
707
|
+
monkeypatch.setattr(_utils, "_get_session", lambda: MockSession())
|
|
708
|
+
|
|
709
|
+
shared = _utils.get_credentials("google_drive", provider="google")
|
|
710
|
+
delegated = _utils.get_credentials("google_drive", provider="google", cache=False)
|
|
711
|
+
fetched_again = _utils.get_credentials("google_drive", provider="google")
|
|
712
|
+
|
|
713
|
+
assert shared.credential_scope == ""
|
|
714
|
+
assert delegated.credential_scope == "delegated"
|
|
715
|
+
assert fetched_again.token == "token-3"
|
|
716
|
+
assert calls == 3
|
|
717
|
+
assert _utils._credentials_cache == {}
|
|
718
|
+
assert _utils._token_cache == {}
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
def test_delegated_legacy_access_tokens_are_never_cached(
|
|
722
|
+
monkeypatch: pytest.MonkeyPatch,
|
|
723
|
+
) -> None:
|
|
724
|
+
monkeypatch.setenv(sdk.TOKEN_ENV, "tok")
|
|
725
|
+
_utils._credentials_cache.clear()
|
|
726
|
+
_utils._token_cache.clear()
|
|
727
|
+
calls = 0
|
|
728
|
+
|
|
729
|
+
class MockSession:
|
|
730
|
+
def get(self, url: str, **kwargs: object) -> DummyResponse:
|
|
731
|
+
nonlocal calls
|
|
732
|
+
del url, kwargs
|
|
733
|
+
calls += 1
|
|
734
|
+
return DummyResponse(
|
|
735
|
+
status_code=200,
|
|
736
|
+
json_data={
|
|
737
|
+
"access_token": f"legacy-{calls}",
|
|
738
|
+
"credential_scope": "delegated",
|
|
739
|
+
"expires_at": "2030-01-01T00:00:00Z",
|
|
740
|
+
},
|
|
741
|
+
)
|
|
742
|
+
|
|
743
|
+
monkeypatch.setattr(_utils, "_get_session", lambda: MockSession())
|
|
744
|
+
|
|
745
|
+
assert _utils.get_access_token("google") == "legacy-1"
|
|
746
|
+
assert _utils.get_access_token("google") == "legacy-2"
|
|
747
|
+
assert calls == 2
|
|
748
|
+
assert _utils._token_cache == {}
|
|
749
|
+
|
|
750
|
+
|
|
651
751
|
def test_get_credentials_preserves_positional_min_valid_seconds(
|
|
652
752
|
monkeypatch: pytest.MonkeyPatch,
|
|
653
753
|
) -> None:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|