lumera 0.39.0.dev1__tar.gz → 0.41.0.dev1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/PKG-INFO +1 -1
  2. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/_utils.py +38 -18
  3. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/integrations/__init__.py +3 -1
  4. lumera-0.41.0.dev1/lumera/integrations/kafka.py +242 -0
  5. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera.egg-info/PKG-INFO +1 -1
  6. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera.egg-info/SOURCES.txt +2 -0
  7. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/pyproject.toml +1 -1
  8. lumera-0.41.0.dev1/tests/test_integrations_kafka.py +126 -0
  9. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_sdk.py +100 -0
  10. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/__init__.py +0 -0
  11. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/_live.py +0 -0
  12. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/agents.py +0 -0
  13. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/automations.py +0 -0
  14. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/email.py +0 -0
  15. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/exceptions.py +0 -0
  16. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/files.py +0 -0
  17. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/flags.py +0 -0
  18. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/function_resources.py +0 -0
  19. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/__init__.py +0 -0
  20. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/__main__.py +0 -0
  21. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_artifacts.py +0 -0
  22. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_config.py +0 -0
  23. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_context.py +0 -0
  24. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_contract.py +0 -0
  25. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_dependencies.py +0 -0
  26. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_errors.py +0 -0
  27. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_execution.py +0 -0
  28. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_jsonschema.py +0 -0
  29. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_registry.py +0 -0
  30. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_remote.py +0 -0
  31. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_runner.py +0 -0
  32. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/_runtime.py +0 -0
  33. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/functions/testing.py +0 -0
  34. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/google.py +0 -0
  35. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/integrations/bill.py +0 -0
  36. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/integrations/finch.py +0 -0
  37. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/integrations/google.py +0 -0
  38. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/integrations/netsuite.py +0 -0
  39. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/integrations/salesforce.py +0 -0
  40. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/integrations/slack.py +0 -0
  41. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/integrations/snowflake.py +0 -0
  42. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/llm.py +0 -0
  43. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/pb.py +0 -0
  44. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/__init__.py +0 -0
  45. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/__main__.py +0 -0
  46. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_api_child.py +0 -0
  47. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_archive.py +0 -0
  48. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_builder.py +0 -0
  49. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_config.py +0 -0
  50. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_contracts.py +0 -0
  51. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_errors.py +0 -0
  52. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_function_child.py +0 -0
  53. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_projection.py +0 -0
  54. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_runtime.py +0 -0
  55. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_runtime_child.py +0 -0
  56. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_schemas.py +0 -0
  57. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_target.py +0 -0
  58. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/releases/_worker_child.py +0 -0
  59. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/sdk.py +0 -0
  60. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/storage.py +0 -0
  61. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera/webhooks.py +0 -0
  62. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera.egg-info/dependency_links.txt +0 -0
  63. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera.egg-info/requires.txt +0 -0
  64. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/lumera.egg-info/top_level.txt +0 -0
  65. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/setup.cfg +0 -0
  66. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_agents_live.py +0 -0
  67. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_automations.py +0 -0
  68. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_flags.py +0 -0
  69. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_function_resources.py +0 -0
  70. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_integrations_bill.py +0 -0
  71. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_integrations_finch.py +0 -0
  72. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_integrations_google.py +0 -0
  73. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_integrations_netsuite.py +0 -0
  74. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_integrations_salesforce.py +0 -0
  75. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_integrations_snowflake.py +0 -0
  76. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_lazy_imports.py +0 -0
  77. {lumera-0.39.0.dev1 → lumera-0.41.0.dev1}/tests/test_webhooks.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: lumera
3
- Version: 0.39.0.dev1
3
+ Version: 0.41.0.dev1
4
4
  Summary: SDK for building on Lumera platform
5
5
  Requires-Python: >=3.11
6
6
  Requires-Dist: requests
@@ -205,7 +205,7 @@ def _parse_expiry(expires_at: int | float | str | None) -> float:
205
205
  raise TypeError(f"Unsupported expires_at format: {type(expires_at)!r}")
206
206
 
207
207
 
208
- def _fetch_access_token(provider: str) -> tuple[str, float]:
208
+ def _fetch_access_token(provider: str) -> tuple[str, float, str]:
209
209
  provider = provider.lower().strip()
210
210
  if not provider:
211
211
  raise ValueError("provider is required")
@@ -218,15 +218,16 @@ def _fetch_access_token(provider: str) -> tuple[str, float]:
218
218
  data = resp.json()
219
219
  access_token = data.get("access_token")
220
220
  expires_at = data.get("expires_at")
221
+ credential_scope = str(data.get("credential_scope") or "")
221
222
 
222
223
  if not access_token:
223
224
  raise RuntimeError(f"Malformed response from Lumera when fetching {provider} access token")
224
225
 
225
226
  expiry_ts = _parse_expiry(expires_at)
226
- return access_token, expiry_ts
227
+ return access_token, expiry_ts, credential_scope
227
228
 
228
229
 
229
- def get_access_token(provider: str, min_valid_seconds: int = 900) -> str:
230
+ def get_access_token(provider: str, min_valid_seconds: int = 900, *, cache: bool = True) -> str:
230
231
  """Get the raw access token string for *provider*.
231
232
 
232
233
  .. deprecated::
@@ -239,8 +240,8 @@ def get_access_token(provider: str, min_valid_seconds: int = 900) -> str:
239
240
  providers this is the token itself; for basic-auth providers it is the
240
241
  base64-encoded ``username:password``.
241
242
 
242
- Results are cached in-process; a fresh fetch is performed when the cached
243
- token is within *min_valid_seconds* of expiry (default 15 min).
243
+ Shared results are cached in-process; delegated results are never cached.
244
+ Pass ``cache=False`` to bypass and avoid populating shared caches.
244
245
  """
245
246
  global _token_cache
246
247
 
@@ -250,19 +251,22 @@ def get_access_token(provider: str, min_valid_seconds: int = 900) -> str:
250
251
 
251
252
  now = _time.time()
252
253
 
253
- cached = _token_cache.get(provider)
254
+ cached = _token_cache.get(provider) if cache else None
254
255
  if cached is not None:
255
256
  access_token, expiry_ts = cached
256
257
  if (expiry_ts - now) >= min_valid_seconds:
257
258
  return access_token
258
259
 
259
- access_token, expiry_ts = _fetch_access_token(provider)
260
- _token_cache[provider] = (access_token, expiry_ts)
260
+ access_token, expiry_ts, credential_scope = _fetch_access_token(provider)
261
+ if credential_scope == "delegated":
262
+ _evict_credential_caches(provider)
263
+ elif cache:
264
+ _token_cache[provider] = (access_token, expiry_ts)
261
265
  return access_token
262
266
 
263
267
 
264
- def get_google_access_token(min_valid_seconds: int = 900) -> str:
265
- return get_access_token("google", min_valid_seconds=min_valid_seconds)
268
+ def get_google_access_token(min_valid_seconds: int = 900, *, cache: bool = True) -> str:
269
+ return get_access_token("google", min_valid_seconds=min_valid_seconds, cache=cache)
266
270
 
267
271
 
268
272
  @dataclass(frozen=True)
@@ -305,6 +309,9 @@ class Credentials:
305
309
  extra_headers: dict[str, str] | None = None
306
310
  """Additional provider-required HTTP headers."""
307
311
 
312
+ credential_scope: str = ""
313
+ """Credential isolation mode. ``"delegated"`` values are never cached."""
314
+
308
315
  def as_header_dict(self) -> dict[str, str]:
309
316
  """Return headers suitable for ``requests.get(headers=...)``."""
310
317
  headers = {self.header_name: self.header_value}
@@ -316,6 +323,13 @@ class Credentials:
316
323
  _credentials_cache: dict[tuple[str, str], tuple[Credentials, float]] = {}
317
324
 
318
325
 
326
+ def _evict_credential_caches(api_name: str) -> None:
327
+ _token_cache.pop(api_name, None)
328
+ for key in tuple(_credentials_cache):
329
+ if key[0] == api_name:
330
+ _credentials_cache.pop(key, None)
331
+
332
+
319
333
  def _fetch_credentials(api_name: str, provider: str | None = None) -> Credentials:
320
334
  api_name = api_name.lower().strip()
321
335
  if not api_name:
@@ -340,6 +354,7 @@ def _fetch_credentials(api_name: str, provider: str | None = None) -> Credential
340
354
  # Collect provider-specific extra fields (anything beyond the standard set).
341
355
  _standard_keys = {
342
356
  "access_token",
357
+ "credential_scope",
343
358
  "auth_type",
344
359
  "header_name",
345
360
  "header_value",
@@ -362,6 +377,7 @@ def _fetch_credentials(api_name: str, provider: str | None = None) -> Credential
362
377
  token=access_token,
363
378
  provider=str(data.get("provider") or provider_hint or api_name),
364
379
  header_name=data.get("header_name", "Authorization"),
380
+ credential_scope=str(data.get("credential_scope") or ""),
365
381
  header_value=data.get("header_value", f"Bearer {access_token}"),
366
382
  username=data.get("username"),
367
383
  expires_at=expiry_ts if expiry_ts != float("inf") else None,
@@ -374,14 +390,16 @@ def get_credentials(
374
390
  api_name: str | None = None,
375
391
  provider: str | None = None,
376
392
  min_valid_seconds: int = 900,
393
+ *,
394
+ cache: bool = True,
377
395
  ) -> Credentials:
378
396
  """Get a :class:`Credentials` object for *api_name*.
379
397
 
380
398
  This is the recommended way to obtain integration credentials. The
381
399
  returned object includes the auth type, ready-to-use HTTP headers, and
382
- (where applicable) the username. Results are cached in-process; a
383
- fresh fetch is performed when the cached credential is within
384
- *min_valid_seconds* of expiry (default 15 min).
400
+ (where applicable) the username. Shared results are cached in-process;
401
+ delegated results are never cached. Pass ``cache=False`` to bypass and
402
+ avoid populating shared caches.
385
403
 
386
404
  Works with any auth type configured in the Lumera integrations page
387
405
  (bearer token, basic auth, etc.).
@@ -432,7 +450,7 @@ def get_credentials(
432
450
 
433
451
  now = _time.time()
434
452
  cache_key = (api_name, provider_hint)
435
- cached = _credentials_cache.get(cache_key)
453
+ cached = _credentials_cache.get(cache_key) if cache else None
436
454
  if cached is not None:
437
455
  creds, expiry_ts = cached
438
456
  if (expiry_ts - now) >= min_valid_seconds:
@@ -440,10 +458,12 @@ def get_credentials(
440
458
 
441
459
  creds = _fetch_credentials(api_name, provider=provider_hint or None)
442
460
  expiry_ts = creds.expires_at if creds.expires_at is not None else float("inf")
443
- _credentials_cache[cache_key] = (creds, expiry_ts)
444
-
445
- # Keep the legacy _token_cache in sync so get_access_token stays fast.
446
- _token_cache[api_name] = (creds.token, expiry_ts)
461
+ if creds.credential_scope == "delegated":
462
+ _evict_credential_caches(api_name)
463
+ elif cache:
464
+ _credentials_cache[cache_key] = (creds, expiry_ts)
465
+ # Keep the legacy _token_cache in sync so get_access_token stays fast.
466
+ _token_cache[api_name] = (creds.token, expiry_ts)
447
467
 
448
468
  return creds
449
469
 
@@ -45,6 +45,7 @@ Available integrations:
45
45
  - ``bill`` - BILL AP/AR and BILL Spend & Expense APIs
46
46
  - ``finch`` - Finch organization and payroll API with batch handling
47
47
  - ``google`` - Google APIs (Docs, Slides, Sheets, Drive)
48
+ - ``kafka`` - Bounded, on-demand Kafka produce and consumer-group polls
48
49
  - ``netsuite`` - NetSuite REST API and SuiteQL (M2M or OAuth)
49
50
  - ``salesforce`` - Salesforce REST API via customer-owned Client Credentials
50
51
  - ``slack`` - Slack (messages, files, reactions)
@@ -69,7 +70,7 @@ from .._utils import (
69
70
  list_integration_connections,
70
71
  list_integrations,
71
72
  )
72
- from . import bill, finch, google, netsuite, salesforce, slack, snowflake
73
+ from . import bill, finch, google, kafka, netsuite, salesforce, slack, snowflake
73
74
 
74
75
  __all__ = [
75
76
  "Credentials",
@@ -80,6 +81,7 @@ __all__ = [
80
81
  "bill",
81
82
  "finch",
82
83
  "google",
84
+ "kafka",
83
85
  "netsuite",
84
86
  "salesforce",
85
87
  "slack",
@@ -0,0 +1,242 @@
1
+ """Bounded, server-mediated Kafka produce and consume operations.
2
+
3
+ Kafka SASL credentials never enter the Python runtime. Each call asks Lumera's
4
+ backend to create a temporary Kafka client, perform one acknowledged produce
5
+ or bounded consumer-group poll, and close the client again.
6
+
7
+ Example::
8
+
9
+ from lumera.integrations.kafka import get_client
10
+
11
+ kafka = get_client("finance_kafka")
12
+ kafka.push("finance.events", {"invoice_id": "inv_123"})
13
+ batch = kafka.pull("finance.events", group_id="lumera-ap-import")
14
+ for record in batch.records:
15
+ print(record.json())
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import base64
21
+ import json
22
+ from collections.abc import Iterable, Mapping
23
+ from dataclasses import dataclass
24
+ from datetime import datetime
25
+ from typing import Any, Literal
26
+ from urllib.parse import quote as _url_quote
27
+
28
+ from lumera._utils import (
29
+ API_BASE,
30
+ _api_headers,
31
+ _get_session,
32
+ _raise_if_project_scope_denied,
33
+ )
34
+
35
+ _MAX_TIMEOUT_SECONDS = 30
36
+ _MAX_PULL_RECORDS = 500
37
+ _TRANSPORT_GRACE_SECONDS = 5
38
+
39
+ KafkaValue = bytes | bytearray | memoryview | str | dict[str, Any] | list[Any]
40
+ KafkaHeaderValue = bytes | bytearray | memoryview | str
41
+ KafkaHeaders = Mapping[str, KafkaHeaderValue] | Iterable[tuple[str, KafkaHeaderValue]]
42
+
43
+
44
+ class KafkaAPIError(Exception):
45
+ """Raised when Lumera cannot complete a Kafka operation."""
46
+
47
+ def __init__(self, status_code: int, body: str) -> None:
48
+ self.status_code = status_code
49
+ self.body = body
50
+ super().__init__(f"Kafka integration error ({status_code}): {body}")
51
+
52
+
53
+ @dataclass(frozen=True)
54
+ class KafkaHeader:
55
+ key: str
56
+ value: bytes
57
+
58
+
59
+ @dataclass(frozen=True)
60
+ class KafkaRecord:
61
+ topic: str
62
+ partition: int
63
+ offset: int
64
+ timestamp: datetime
65
+ key: bytes | None
66
+ value: bytes
67
+ headers: tuple[KafkaHeader, ...]
68
+
69
+ def text(self, encoding: str = "utf-8") -> str:
70
+ """Decode the record value as text."""
71
+ return self.value.decode(encoding)
72
+
73
+ def json(self) -> object:
74
+ """Decode the record value as UTF-8 JSON."""
75
+ return json.loads(self.value)
76
+
77
+
78
+ @dataclass(frozen=True)
79
+ class KafkaPushResult:
80
+ topic: str
81
+ partition: int
82
+ offset: int
83
+ timestamp: datetime
84
+
85
+
86
+ @dataclass(frozen=True)
87
+ class KafkaPullResult:
88
+ records: tuple[KafkaRecord, ...]
89
+ committed: bool
90
+ timed_out: bool
91
+
92
+
93
+ class KafkaClient:
94
+ """On-demand Kafka client backed by a project-scoped Lumera connection."""
95
+
96
+ def __init__(self, api_name: str = "kafka", timeout: int = 10) -> None:
97
+ if not api_name.strip():
98
+ raise ValueError("api_name is required")
99
+ self.api_name = api_name.strip()
100
+ self.timeout = _validate_timeout(timeout)
101
+
102
+ def push(
103
+ self,
104
+ topic: str,
105
+ value: KafkaValue,
106
+ *,
107
+ key: KafkaHeaderValue | None = None,
108
+ headers: KafkaHeaders | None = None,
109
+ timeout: int | None = None,
110
+ ) -> KafkaPushResult:
111
+ """Produce one record and wait for Kafka's acknowledged offset.
112
+
113
+ ``dict`` and ``list`` values are serialized as compact UTF-8 JSON.
114
+ Strings are UTF-8 encoded; bytes remain unchanged.
115
+ """
116
+ operation_timeout = self.timeout if timeout is None else _validate_timeout(timeout)
117
+ payload: dict[str, Any] = {
118
+ "topic": topic,
119
+ "value_base64": _encode_base64(value, allow_json=True),
120
+ "timeout_seconds": operation_timeout,
121
+ }
122
+ if key is not None:
123
+ payload["key_base64"] = _encode_base64(key, allow_json=False)
124
+ if headers is not None:
125
+ items = headers.items() if isinstance(headers, Mapping) else headers
126
+ payload["headers"] = [
127
+ {"key": name, "value_base64": _encode_base64(item, allow_json=False)}
128
+ for name, item in items
129
+ ]
130
+
131
+ data = self._request("push", payload, operation_timeout)
132
+ return KafkaPushResult(
133
+ topic=str(data["topic"]),
134
+ partition=int(data["partition"]),
135
+ offset=int(data["offset"]),
136
+ timestamp=_parse_timestamp(data["timestamp"]),
137
+ )
138
+
139
+ def pull(
140
+ self,
141
+ topic: str,
142
+ *,
143
+ group_id: str,
144
+ max_records: int = 100,
145
+ timeout: int | None = None,
146
+ offset_reset: Literal["earliest", "latest"] = "earliest",
147
+ commit: bool = True,
148
+ ) -> KafkaPullResult:
149
+ """Poll once with a temporary consumer and optionally commit offsets.
150
+
151
+ Use a stable ``group_id`` across invocations. ``offset_reset`` matters
152
+ only when the group has no previously committed topic offset.
153
+ """
154
+ if not group_id.strip():
155
+ raise ValueError("group_id is required")
156
+ if not 1 <= max_records <= _MAX_PULL_RECORDS:
157
+ raise ValueError(f"max_records must be between 1 and {_MAX_PULL_RECORDS}")
158
+ if offset_reset not in {"earliest", "latest"}:
159
+ raise ValueError("offset_reset must be earliest or latest")
160
+ operation_timeout = self.timeout if timeout is None else _validate_timeout(timeout)
161
+ data = self._request(
162
+ "pull",
163
+ {
164
+ "topic": topic,
165
+ "group_id": group_id,
166
+ "max_records": max_records,
167
+ "timeout_seconds": operation_timeout,
168
+ "offset_reset": offset_reset,
169
+ "commit": commit,
170
+ },
171
+ operation_timeout,
172
+ )
173
+ records = tuple(_parse_record(item) for item in data.get("records", []))
174
+ return KafkaPullResult(
175
+ records=records,
176
+ committed=bool(data.get("committed", False)),
177
+ timed_out=bool(data.get("timed_out", False)),
178
+ )
179
+
180
+ def _request(self, operation: str, payload: dict[str, Any], timeout: int) -> dict[str, Any]:
181
+ url = f"{API_BASE}/connections/{_url_quote(self.api_name, safe='')}/kafka/{operation}"
182
+ response = _get_session().post(
183
+ url,
184
+ json=payload,
185
+ headers=_api_headers(content_type="application/json"),
186
+ timeout=timeout + _TRANSPORT_GRACE_SECONDS,
187
+ )
188
+ _raise_if_project_scope_denied(response, self.api_name, url)
189
+ if not response.ok:
190
+ raise KafkaAPIError(response.status_code, response.text[:500])
191
+ data = response.json()
192
+ if not isinstance(data, dict):
193
+ raise KafkaAPIError(response.status_code, "Invalid Kafka integration response")
194
+ return data
195
+
196
+
197
+ def get_client(api_name: str = "kafka", timeout: int = 10) -> KafkaClient:
198
+ """Create a lightweight, on-demand client for one Kafka connection."""
199
+ return KafkaClient(api_name=api_name, timeout=timeout)
200
+
201
+
202
+ def _validate_timeout(timeout: int) -> int:
203
+ if not isinstance(timeout, int) or not 1 <= timeout <= _MAX_TIMEOUT_SECONDS:
204
+ raise ValueError(f"timeout must be between 1 and {_MAX_TIMEOUT_SECONDS} seconds")
205
+ return timeout
206
+
207
+
208
+ def _encode_base64(value: KafkaValue | KafkaHeaderValue, *, allow_json: bool) -> str:
209
+ if isinstance(value, str):
210
+ raw = value.encode("utf-8")
211
+ elif isinstance(value, (bytes, bytearray, memoryview)):
212
+ raw = bytes(value)
213
+ elif allow_json and isinstance(value, (dict, list)):
214
+ raw = json.dumps(value, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
215
+ else:
216
+ raise TypeError("Kafka values must be bytes, str, dict, or list")
217
+ return base64.b64encode(raw).decode("ascii")
218
+
219
+
220
+ def _parse_timestamp(value: object) -> datetime:
221
+ if not isinstance(value, str):
222
+ raise ValueError("Kafka timestamp must be an RFC 3339 string")
223
+ return datetime.fromisoformat(value.replace("Z", "+00:00"))
224
+
225
+
226
+ def _parse_record(data: dict[str, Any]) -> KafkaRecord:
227
+ key = data.get("key_base64")
228
+ return KafkaRecord(
229
+ topic=str(data["topic"]),
230
+ partition=int(data["partition"]),
231
+ offset=int(data["offset"]),
232
+ timestamp=_parse_timestamp(data["timestamp"]),
233
+ key=base64.b64decode(key, validate=True) if isinstance(key, str) else None,
234
+ value=base64.b64decode(str(data["value_base64"]), validate=True),
235
+ headers=tuple(
236
+ KafkaHeader(
237
+ key=str(header["key"]),
238
+ value=base64.b64decode(str(header["value_base64"]), validate=True),
239
+ )
240
+ for header in data.get("headers", [])
241
+ ),
242
+ )
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: lumera
3
- Version: 0.39.0.dev1
3
+ Version: 0.41.0.dev1
4
4
  Summary: SDK for building on Lumera platform
5
5
  Requires-Python: >=3.11
6
6
  Requires-Dist: requests
@@ -39,6 +39,7 @@ lumera/integrations/__init__.py
39
39
  lumera/integrations/bill.py
40
40
  lumera/integrations/finch.py
41
41
  lumera/integrations/google.py
42
+ lumera/integrations/kafka.py
42
43
  lumera/integrations/netsuite.py
43
44
  lumera/integrations/salesforce.py
44
45
  lumera/integrations/slack.py
@@ -65,6 +66,7 @@ tests/test_function_resources.py
65
66
  tests/test_integrations_bill.py
66
67
  tests/test_integrations_finch.py
67
68
  tests/test_integrations_google.py
69
+ tests/test_integrations_kafka.py
68
70
  tests/test_integrations_netsuite.py
69
71
  tests/test_integrations_salesforce.py
70
72
  tests/test_integrations_snowflake.py
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "lumera"
3
- version = "0.39.0.dev1"
3
+ version = "0.41.0.dev1"
4
4
  description = "SDK for building on Lumera platform"
5
5
  requires-python = ">=3.11"
6
6
  dependencies = [
@@ -0,0 +1,126 @@
1
+ from __future__ import annotations
2
+
3
+ import base64
4
+ from typing import Any
5
+
6
+ import pytest
7
+
8
+ from lumera.integrations import kafka
9
+
10
+
11
+ class FakeResponse:
12
+ def __init__(self, data: dict[str, Any], status_code: int = 200) -> None:
13
+ self._data = data
14
+ self.status_code = status_code
15
+ self.ok = status_code < 400
16
+ self.text = str(data)
17
+
18
+ def json(self) -> dict[str, Any]:
19
+ return self._data
20
+
21
+
22
+ class FakeSession:
23
+ def __init__(self, response: FakeResponse) -> None:
24
+ self.response = response
25
+ self.url = ""
26
+ self.kwargs: dict[str, Any] = {}
27
+
28
+ def post(self, url: str, **kwargs: object) -> FakeResponse:
29
+ self.url, self.kwargs = url, kwargs
30
+ return self.response
31
+
32
+
33
+ def install_session(monkeypatch: pytest.MonkeyPatch, response: FakeResponse) -> FakeSession:
34
+ session = FakeSession(response)
35
+ monkeypatch.setattr(kafka, "_get_session", lambda: session)
36
+ monkeypatch.setattr(kafka, "_api_headers", lambda **_: {"X-Lumera-Project": "finance"})
37
+ return session
38
+
39
+
40
+ def test_push_json_is_binary_safe_and_uses_project_scoped_backend(
41
+ monkeypatch: pytest.MonkeyPatch,
42
+ ) -> None:
43
+ session = install_session(
44
+ monkeypatch,
45
+ FakeResponse(
46
+ {
47
+ "topic": "finance.events",
48
+ "partition": 2,
49
+ "offset": 41,
50
+ "timestamp": "2026-08-25T10:30:00Z",
51
+ }
52
+ ),
53
+ )
54
+
55
+ result = kafka.get_client("finance_kafka").push(
56
+ "finance.events",
57
+ {"invoice_id": "inv_123"},
58
+ key=b"invoice-key",
59
+ headers={"trace": "abc"},
60
+ )
61
+
62
+ assert session.url.endswith("/connections/finance_kafka/kafka/push")
63
+ payload = session.kwargs["json"]
64
+ assert base64.b64decode(payload["value_base64"]) == b'{"invoice_id":"inv_123"}'
65
+ assert base64.b64decode(payload["key_base64"]) == b"invoice-key"
66
+ assert base64.b64decode(payload["headers"][0]["value_base64"]) == b"abc"
67
+ assert session.kwargs["headers"]["X-Lumera-Project"] == "finance"
68
+ assert result.partition == 2
69
+ assert result.offset == 41
70
+
71
+
72
+ def test_pull_decodes_records_and_preserves_explicit_commit_choice(
73
+ monkeypatch: pytest.MonkeyPatch,
74
+ ) -> None:
75
+ session = install_session(
76
+ monkeypatch,
77
+ FakeResponse(
78
+ {
79
+ "records": [
80
+ {
81
+ "topic": "finance.events",
82
+ "partition": 1,
83
+ "offset": 42,
84
+ "timestamp": "2026-08-25T10:31:00Z",
85
+ "key_base64": None,
86
+ "value_base64": base64.b64encode(b'{"amount":25}').decode(),
87
+ "headers": [
88
+ {
89
+ "key": "trace",
90
+ "value_base64": base64.b64encode(b"xyz").decode(),
91
+ }
92
+ ],
93
+ }
94
+ ],
95
+ "committed": False,
96
+ "timed_out": False,
97
+ }
98
+ ),
99
+ )
100
+
101
+ batch = kafka.get_client("finance_kafka").pull(
102
+ "finance.events", group_id="lumera-ap-import", max_records=25, commit=False
103
+ )
104
+
105
+ assert session.url.endswith("/connections/finance_kafka/kafka/pull")
106
+ assert session.kwargs["json"]["group_id"] == "lumera-ap-import"
107
+ assert session.kwargs["json"]["commit"] is False
108
+ assert batch.committed is False
109
+ assert batch.records[0].json() == {"amount": 25}
110
+ assert batch.records[0].headers[0].value == b"xyz"
111
+
112
+
113
+ def test_kafka_client_rejects_unbounded_requests() -> None:
114
+ with pytest.raises(ValueError, match="between 1 and 30"):
115
+ kafka.get_client(timeout=60)
116
+ with pytest.raises(ValueError, match="between 1 and 500"):
117
+ kafka.get_client().pull("finance.events", group_id="finance", max_records=501)
118
+
119
+
120
+ def test_kafka_provider_errors_are_typed(monkeypatch: pytest.MonkeyPatch) -> None:
121
+ install_session(monkeypatch, FakeResponse({"message": "broker rejected request"}, 502))
122
+
123
+ with pytest.raises(kafka.KafkaAPIError) as exc:
124
+ kafka.get_client().push("finance.events", b"message")
125
+
126
+ assert exc.value.status_code == 502
@@ -648,6 +648,106 @@ def test_get_credentials_cache_key_includes_provider_hint(
648
648
  assert calls == [{"provider": "slack"}, {"provider": "teams"}]
649
649
 
650
650
 
651
+ def test_delegated_credentials_are_never_cached(monkeypatch: pytest.MonkeyPatch) -> None:
652
+ monkeypatch.setenv(sdk.TOKEN_ENV, "tok")
653
+ _utils._credentials_cache.clear()
654
+ _utils._token_cache.clear()
655
+ calls = 0
656
+
657
+ class MockSession:
658
+ def get(self, url: str, **kwargs: object) -> DummyResponse:
659
+ nonlocal calls
660
+ del url, kwargs
661
+ calls += 1
662
+ return DummyResponse(
663
+ status_code=200,
664
+ json_data={
665
+ "access_token": f"member-token-{calls}",
666
+ "provider": "google",
667
+ "credential_scope": "delegated",
668
+ "expires_at": "2030-01-01T00:00:00Z",
669
+ },
670
+ )
671
+
672
+ monkeypatch.setattr(_utils, "_get_session", lambda: MockSession())
673
+
674
+ tokens = [_utils.get_credentials("google_drive", provider="google").token for _ in range(3)]
675
+
676
+ assert tokens == ["member-token-1", "member-token-2", "member-token-3"]
677
+ assert calls == 3
678
+ assert _utils._credentials_cache == {}
679
+ assert _utils._token_cache == {}
680
+
681
+
682
+ def test_cache_false_bootstraps_and_evicts_delegated_credentials(
683
+ monkeypatch: pytest.MonkeyPatch,
684
+ ) -> None:
685
+ monkeypatch.setenv(sdk.TOKEN_ENV, "tok")
686
+ _utils._credentials_cache.clear()
687
+ _utils._token_cache.clear()
688
+ responses = ["", "delegated", "delegated"]
689
+ calls = 0
690
+
691
+ class MockSession:
692
+ def get(self, url: str, **kwargs: object) -> DummyResponse:
693
+ nonlocal calls
694
+ del url, kwargs
695
+ scope = responses[calls]
696
+ calls += 1
697
+ return DummyResponse(
698
+ status_code=200,
699
+ json_data={
700
+ "access_token": f"token-{calls}",
701
+ "provider": "google",
702
+ "credential_scope": scope,
703
+ "expires_at": "2030-01-01T00:00:00Z",
704
+ },
705
+ )
706
+
707
+ monkeypatch.setattr(_utils, "_get_session", lambda: MockSession())
708
+
709
+ shared = _utils.get_credentials("google_drive", provider="google")
710
+ delegated = _utils.get_credentials("google_drive", provider="google", cache=False)
711
+ fetched_again = _utils.get_credentials("google_drive", provider="google")
712
+
713
+ assert shared.credential_scope == ""
714
+ assert delegated.credential_scope == "delegated"
715
+ assert fetched_again.token == "token-3"
716
+ assert calls == 3
717
+ assert _utils._credentials_cache == {}
718
+ assert _utils._token_cache == {}
719
+
720
+
721
+ def test_delegated_legacy_access_tokens_are_never_cached(
722
+ monkeypatch: pytest.MonkeyPatch,
723
+ ) -> None:
724
+ monkeypatch.setenv(sdk.TOKEN_ENV, "tok")
725
+ _utils._credentials_cache.clear()
726
+ _utils._token_cache.clear()
727
+ calls = 0
728
+
729
+ class MockSession:
730
+ def get(self, url: str, **kwargs: object) -> DummyResponse:
731
+ nonlocal calls
732
+ del url, kwargs
733
+ calls += 1
734
+ return DummyResponse(
735
+ status_code=200,
736
+ json_data={
737
+ "access_token": f"legacy-{calls}",
738
+ "credential_scope": "delegated",
739
+ "expires_at": "2030-01-01T00:00:00Z",
740
+ },
741
+ )
742
+
743
+ monkeypatch.setattr(_utils, "_get_session", lambda: MockSession())
744
+
745
+ assert _utils.get_access_token("google") == "legacy-1"
746
+ assert _utils.get_access_token("google") == "legacy-2"
747
+ assert calls == 2
748
+ assert _utils._token_cache == {}
749
+
750
+
651
751
  def test_get_credentials_preserves_positional_min_valid_seconds(
652
752
  monkeypatch: pytest.MonkeyPatch,
653
753
  ) -> None:
File without changes
File without changes
File without changes
File without changes