airbyte-agent-greenhouse 0.17.48__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_agent_greenhouse/__init__.py +105 -0
- airbyte_agent_greenhouse/_vendored/__init__.py +1 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/__init__.py +82 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/auth_strategies.py +1120 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/auth_template.py +135 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/cloud_utils/__init__.py +5 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/cloud_utils/client.py +213 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/connector_model_loader.py +965 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/constants.py +78 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/exceptions.py +23 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/executor/__init__.py +31 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/executor/hosted_executor.py +196 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/executor/local_executor.py +1724 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/executor/models.py +190 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/extensions.py +693 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/http/__init__.py +37 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/http/adapters/__init__.py +9 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/http/adapters/httpx_adapter.py +251 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/http/config.py +98 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/http/exceptions.py +119 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/http/protocols.py +114 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/http/response.py +104 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/http_client.py +693 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/introspection.py +262 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/logging/__init__.py +11 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/logging/logger.py +273 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/logging/types.py +93 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/observability/__init__.py +11 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/observability/config.py +179 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/observability/models.py +19 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/observability/redactor.py +81 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/observability/session.py +103 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/performance/__init__.py +6 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/performance/instrumentation.py +57 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/performance/metrics.py +93 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/schema/__init__.py +75 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/schema/base.py +164 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/schema/components.py +239 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/schema/connector.py +120 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/schema/extensions.py +230 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/schema/operations.py +146 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/schema/security.py +223 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/secrets.py +182 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/telemetry/__init__.py +10 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/telemetry/config.py +32 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/telemetry/events.py +59 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/telemetry/tracker.py +155 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/types.py +245 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/utils.py +60 -0
- airbyte_agent_greenhouse/_vendored/connector_sdk/validation.py +828 -0
- airbyte_agent_greenhouse/connector.py +1391 -0
- airbyte_agent_greenhouse/connector_model.py +2356 -0
- airbyte_agent_greenhouse/models.py +281 -0
- airbyte_agent_greenhouse/types.py +136 -0
- airbyte_agent_greenhouse-0.17.48.dist-info/METADATA +116 -0
- airbyte_agent_greenhouse-0.17.48.dist-info/RECORD +57 -0
- airbyte_agent_greenhouse-0.17.48.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,1724 @@
|
|
|
1
|
+
"""Local executor for direct HTTP execution of connector operations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import inspect
|
|
7
|
+
import logging
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import time
|
|
11
|
+
from collections.abc import AsyncIterator
|
|
12
|
+
from typing import Any, Protocol
|
|
13
|
+
from urllib.parse import quote
|
|
14
|
+
|
|
15
|
+
from jinja2 import Environment, StrictUndefined
|
|
16
|
+
from jsonpath_ng import parse as parse_jsonpath
|
|
17
|
+
from opentelemetry import trace
|
|
18
|
+
|
|
19
|
+
from ..auth_template import apply_auth_mapping
|
|
20
|
+
from ..connector_model_loader import load_connector_model
|
|
21
|
+
from ..constants import (
|
|
22
|
+
DEFAULT_MAX_CONNECTIONS,
|
|
23
|
+
DEFAULT_MAX_KEEPALIVE_CONNECTIONS,
|
|
24
|
+
)
|
|
25
|
+
from ..http_client import HTTPClient, TokenRefreshCallback
|
|
26
|
+
from ..logging import NullLogger, RequestLogger
|
|
27
|
+
from ..observability import ObservabilitySession
|
|
28
|
+
from ..schema.extensions import RetryConfig
|
|
29
|
+
from ..secrets import SecretStr
|
|
30
|
+
from ..telemetry import SegmentTracker
|
|
31
|
+
from ..types import (
|
|
32
|
+
Action,
|
|
33
|
+
AuthConfig,
|
|
34
|
+
AuthOption,
|
|
35
|
+
ConnectorModel,
|
|
36
|
+
EndpointDefinition,
|
|
37
|
+
EntityDefinition,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
from .models import (
|
|
41
|
+
ActionNotSupportedError,
|
|
42
|
+
EntityNotFoundError,
|
|
43
|
+
ExecutionConfig,
|
|
44
|
+
ExecutionResult,
|
|
45
|
+
ExecutorError,
|
|
46
|
+
InvalidParameterError,
|
|
47
|
+
MissingParameterError,
|
|
48
|
+
StandardExecuteResult,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class _OperationContext:
|
|
53
|
+
"""Shared context for operation handlers."""
|
|
54
|
+
|
|
55
|
+
def __init__(self, executor: LocalExecutor):
|
|
56
|
+
self.executor = executor
|
|
57
|
+
self.http_client = executor.http_client
|
|
58
|
+
self.tracker = executor.tracker
|
|
59
|
+
self.session = executor.session
|
|
60
|
+
self.logger = executor.logger
|
|
61
|
+
self.entity_index = executor._entity_index
|
|
62
|
+
self.operation_index = executor._operation_index
|
|
63
|
+
# Bind helper methods
|
|
64
|
+
self.build_path = executor._build_path
|
|
65
|
+
self.extract_query_params = executor._extract_query_params
|
|
66
|
+
self.extract_body = executor._extract_body
|
|
67
|
+
self.build_request_body = executor._build_request_body
|
|
68
|
+
self.determine_request_format = executor._determine_request_format
|
|
69
|
+
self.validate_required_body_fields = executor._validate_required_body_fields
|
|
70
|
+
self.extract_records = executor._extract_records
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class _OperationHandler(Protocol):
|
|
74
|
+
"""Protocol for operation handlers."""
|
|
75
|
+
|
|
76
|
+
def can_handle(self, action: Action) -> bool:
|
|
77
|
+
"""Check if this handler can handle the given action."""
|
|
78
|
+
...
|
|
79
|
+
|
|
80
|
+
async def execute_operation(
|
|
81
|
+
self,
|
|
82
|
+
entity: str,
|
|
83
|
+
action: Action,
|
|
84
|
+
params: dict[str, Any],
|
|
85
|
+
) -> StandardExecuteResult | AsyncIterator[bytes]:
|
|
86
|
+
"""Execute the operation and return result.
|
|
87
|
+
|
|
88
|
+
Returns:
|
|
89
|
+
StandardExecuteResult for standard operations (GET, LIST, CREATE, etc.)
|
|
90
|
+
AsyncIterator[bytes] for download operations
|
|
91
|
+
"""
|
|
92
|
+
...
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class LocalExecutor:
|
|
96
|
+
"""Async executor for Entity×Action operations with direct HTTP execution.
|
|
97
|
+
|
|
98
|
+
This is the "local mode" executor that makes direct HTTP calls to external APIs.
|
|
99
|
+
It performs local entity/action lookups, validation, and request building.
|
|
100
|
+
|
|
101
|
+
Implements ExecutorProtocol.
|
|
102
|
+
"""
|
|
103
|
+
|
|
104
|
+
def __init__(
|
|
105
|
+
self,
|
|
106
|
+
config_path: str | None = None,
|
|
107
|
+
model: ConnectorModel | None = None,
|
|
108
|
+
secrets: dict[str, SecretStr] | None = None,
|
|
109
|
+
auth_config: dict[str, SecretStr] | None = None,
|
|
110
|
+
auth_scheme: str | None = None,
|
|
111
|
+
enable_logging: bool = False,
|
|
112
|
+
log_file: str | None = None,
|
|
113
|
+
execution_context: str | None = None,
|
|
114
|
+
max_connections: int = DEFAULT_MAX_CONNECTIONS,
|
|
115
|
+
max_keepalive_connections: int = DEFAULT_MAX_KEEPALIVE_CONNECTIONS,
|
|
116
|
+
max_logs: int | None = 10000,
|
|
117
|
+
config_values: dict[str, str] | None = None,
|
|
118
|
+
on_token_refresh: TokenRefreshCallback = None,
|
|
119
|
+
retry_config: RetryConfig | None = None,
|
|
120
|
+
):
|
|
121
|
+
"""Initialize async executor.
|
|
122
|
+
|
|
123
|
+
Args:
|
|
124
|
+
config_path: Path to connector.yaml.
|
|
125
|
+
If neither config_path nor model is provided, an error will be raised.
|
|
126
|
+
model: ConnectorModel object to execute.
|
|
127
|
+
secrets: (Legacy) Auth parameters that bypass x-airbyte-auth-config mapping.
|
|
128
|
+
Directly passed to auth strategies (e.g., {"username": "...", "password": "..."}).
|
|
129
|
+
Cannot be used together with auth_config.
|
|
130
|
+
auth_config: User-facing auth configuration following x-airbyte-auth-config spec.
|
|
131
|
+
Will be transformed via auth_mapping to produce auth parameters.
|
|
132
|
+
Cannot be used together with secrets.
|
|
133
|
+
auth_scheme: (Multi-auth only) Explicit security scheme name to use.
|
|
134
|
+
If None, SDK will auto-select based on provided credentials.
|
|
135
|
+
Example: auth_scheme="githubOAuth"
|
|
136
|
+
enable_logging: Enable request/response logging
|
|
137
|
+
log_file: Path to log file (if enable_logging=True)
|
|
138
|
+
execution_context: Execution context (mcp, direct, blessed, agent)
|
|
139
|
+
max_connections: Maximum number of concurrent connections
|
|
140
|
+
max_keepalive_connections: Maximum number of keepalive connections
|
|
141
|
+
max_logs: Maximum number of logs to keep in memory before rotation.
|
|
142
|
+
Set to None for unlimited (not recommended for production).
|
|
143
|
+
Defaults to 10000.
|
|
144
|
+
config_values: Optional dict of config values for server variable substitution
|
|
145
|
+
(e.g., {"subdomain": "acme"} for URLs like https://{subdomain}.api.example.com).
|
|
146
|
+
on_token_refresh: Optional callback function(new_tokens: dict) called when
|
|
147
|
+
OAuth2 tokens are refreshed. Use this to persist updated tokens.
|
|
148
|
+
Can be sync or async. Example: lambda tokens: save_to_db(tokens)
|
|
149
|
+
retry_config: Optional retry configuration override. If provided, overrides
|
|
150
|
+
the connector.yaml x-airbyte-retry-config. If None, uses connector.yaml
|
|
151
|
+
config or SDK defaults.
|
|
152
|
+
"""
|
|
153
|
+
# Validate mutual exclusivity of secrets and auth_config
|
|
154
|
+
if secrets is not None and auth_config is not None:
|
|
155
|
+
raise ValueError(
|
|
156
|
+
"Cannot provide both 'secrets' and 'auth_config' parameters. "
|
|
157
|
+
"Use 'auth_config' for user-facing credentials (recommended), "
|
|
158
|
+
"or 'secrets' for direct auth parameters (legacy)."
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
# Validate mutual exclusivity of config_path and model
|
|
162
|
+
if config_path is not None and model is not None:
|
|
163
|
+
raise ValueError("Cannot provide both 'config_path' and 'model' parameters.")
|
|
164
|
+
|
|
165
|
+
if config_path is None and model is None:
|
|
166
|
+
raise ValueError("Must provide either 'config_path' or 'model' parameter.")
|
|
167
|
+
|
|
168
|
+
# Load model from path or use provided model
|
|
169
|
+
if config_path is not None:
|
|
170
|
+
self.model: ConnectorModel = load_connector_model(config_path)
|
|
171
|
+
else:
|
|
172
|
+
self.model: ConnectorModel = model
|
|
173
|
+
|
|
174
|
+
self.on_token_refresh = on_token_refresh
|
|
175
|
+
self.config_values = config_values or {}
|
|
176
|
+
|
|
177
|
+
# Handle auth selection for multi-auth or single-auth connectors
|
|
178
|
+
user_credentials = auth_config if auth_config is not None else secrets
|
|
179
|
+
selected_auth_config, self.secrets = self._initialize_auth(user_credentials, auth_scheme)
|
|
180
|
+
|
|
181
|
+
# Create shared observability session
|
|
182
|
+
self.session = ObservabilitySession(
|
|
183
|
+
connector_name=self.model.name,
|
|
184
|
+
connector_version=getattr(self.model, "version", None),
|
|
185
|
+
execution_context=(execution_context or os.getenv("AIRBYTE_EXECUTION_CONTEXT", "direct")),
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
# Initialize telemetry tracker
|
|
189
|
+
self.tracker = SegmentTracker(self.session)
|
|
190
|
+
self.tracker.track_connector_init(connector_version=getattr(self.model, "version", None))
|
|
191
|
+
|
|
192
|
+
# Initialize logger
|
|
193
|
+
if enable_logging:
|
|
194
|
+
self.logger = RequestLogger(
|
|
195
|
+
log_file=log_file,
|
|
196
|
+
connector_name=self.model.name,
|
|
197
|
+
max_logs=max_logs,
|
|
198
|
+
)
|
|
199
|
+
else:
|
|
200
|
+
self.logger = NullLogger()
|
|
201
|
+
|
|
202
|
+
# Initialize async HTTP client with connection pooling
|
|
203
|
+
self.http_client = HTTPClient(
|
|
204
|
+
base_url=self.model.base_url,
|
|
205
|
+
auth_config=selected_auth_config,
|
|
206
|
+
secrets=self.secrets,
|
|
207
|
+
config_values=self.config_values,
|
|
208
|
+
logger=self.logger,
|
|
209
|
+
max_connections=max_connections,
|
|
210
|
+
max_keepalive_connections=max_keepalive_connections,
|
|
211
|
+
on_token_refresh=on_token_refresh,
|
|
212
|
+
retry_config=retry_config or self.model.retry_config,
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
# Build O(1) lookup indexes
|
|
216
|
+
self._entity_index: dict[str, EntityDefinition] = {entity.name: entity for entity in self.model.entities}
|
|
217
|
+
|
|
218
|
+
# Build O(1) operation index: (entity, action) -> endpoint
|
|
219
|
+
self._operation_index: dict[tuple[str, Action], Any] = {}
|
|
220
|
+
for entity in self.model.entities:
|
|
221
|
+
for action in entity.actions:
|
|
222
|
+
endpoint = entity.endpoints.get(action)
|
|
223
|
+
if endpoint:
|
|
224
|
+
self._operation_index[(entity.name, action)] = endpoint
|
|
225
|
+
|
|
226
|
+
# Register operation handlers (order matters for can_handle priority)
|
|
227
|
+
op_context = _OperationContext(self)
|
|
228
|
+
self._operation_handlers: list[_OperationHandler] = [
|
|
229
|
+
_DownloadOperationHandler(op_context),
|
|
230
|
+
_StandardOperationHandler(op_context),
|
|
231
|
+
]
|
|
232
|
+
|
|
233
|
+
def _apply_auth_config_mapping(self, user_secrets: dict[str, SecretStr]) -> dict[str, SecretStr]:
|
|
234
|
+
"""Apply auth_mapping from x-airbyte-auth-config to transform user secrets.
|
|
235
|
+
|
|
236
|
+
This method takes user-provided secrets (e.g., {"api_token": "abc123"}) and
|
|
237
|
+
transforms them into the auth scheme format (e.g., {"username": "abc123", "password": "api_token"})
|
|
238
|
+
using the template mappings defined in x-airbyte-auth-config.
|
|
239
|
+
|
|
240
|
+
Args:
|
|
241
|
+
user_secrets: User-provided secrets from config
|
|
242
|
+
|
|
243
|
+
Returns:
|
|
244
|
+
Transformed secrets matching the auth scheme requirements
|
|
245
|
+
"""
|
|
246
|
+
if not self.model.auth.user_config_spec:
|
|
247
|
+
# No x-airbyte-auth-config defined, use secrets as-is
|
|
248
|
+
return user_secrets
|
|
249
|
+
|
|
250
|
+
user_config_spec = self.model.auth.user_config_spec
|
|
251
|
+
auth_mapping = None
|
|
252
|
+
required_fields: list[str] | None = None
|
|
253
|
+
|
|
254
|
+
# Check for single option (direct auth_mapping)
|
|
255
|
+
if user_config_spec.auth_mapping:
|
|
256
|
+
auth_mapping = user_config_spec.auth_mapping
|
|
257
|
+
required_fields = user_config_spec.required
|
|
258
|
+
# Check for oneOf (multiple auth options)
|
|
259
|
+
elif user_config_spec.one_of:
|
|
260
|
+
# Find the matching option based on which required fields are present
|
|
261
|
+
for option in user_config_spec.one_of:
|
|
262
|
+
option_required = option.required or []
|
|
263
|
+
if all(field in user_secrets for field in option_required):
|
|
264
|
+
auth_mapping = option.auth_mapping
|
|
265
|
+
required_fields = option_required
|
|
266
|
+
break
|
|
267
|
+
|
|
268
|
+
if not auth_mapping:
|
|
269
|
+
# No matching auth_mapping found, use secrets as-is
|
|
270
|
+
return user_secrets
|
|
271
|
+
|
|
272
|
+
# If required fields are missing and user provided no credentials,
|
|
273
|
+
# return as-is (allows empty auth for testing or optional auth)
|
|
274
|
+
if required_fields and not user_secrets:
|
|
275
|
+
return user_secrets
|
|
276
|
+
|
|
277
|
+
# Convert SecretStr values to plain strings for template processing
|
|
278
|
+
user_config_values = {
|
|
279
|
+
key: (value.get_secret_value() if hasattr(value, "get_secret_value") else str(value)) for key, value in user_secrets.items()
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
# Apply the auth_mapping templates, passing required_fields so optional
|
|
283
|
+
# fields that are not provided can be skipped
|
|
284
|
+
mapped_values = apply_auth_mapping(auth_mapping, user_config_values, required_fields=required_fields)
|
|
285
|
+
|
|
286
|
+
# Convert back to SecretStr
|
|
287
|
+
mapped_secrets = {key: SecretStr(value) for key, value in mapped_values.items()}
|
|
288
|
+
|
|
289
|
+
return mapped_secrets
|
|
290
|
+
|
|
291
|
+
def _initialize_auth(
|
|
292
|
+
self,
|
|
293
|
+
user_credentials: dict[str, SecretStr] | None,
|
|
294
|
+
explicit_scheme: str | None,
|
|
295
|
+
) -> tuple[AuthConfig, dict[str, SecretStr] | None]:
|
|
296
|
+
"""Initialize authentication for single or multi-auth connectors.
|
|
297
|
+
|
|
298
|
+
Handles both legacy single-auth and new multi-auth connectors.
|
|
299
|
+
For multi-auth, the auth scheme can be explicitly provided or inferred
|
|
300
|
+
from the provided credentials by matching against each scheme's required fields.
|
|
301
|
+
|
|
302
|
+
Args:
|
|
303
|
+
user_credentials: User-provided credentials (auth_config or secrets)
|
|
304
|
+
explicit_scheme: Explicit scheme name for multi-auth (optional, will be
|
|
305
|
+
inferred from credentials if not provided)
|
|
306
|
+
|
|
307
|
+
Returns:
|
|
308
|
+
Tuple of (selected AuthConfig for HTTPClient, transformed secrets)
|
|
309
|
+
|
|
310
|
+
Raises:
|
|
311
|
+
ValueError: If multi-auth connector can't determine which scheme to use
|
|
312
|
+
"""
|
|
313
|
+
# Multi-auth: explicit scheme selection or inference from credentials
|
|
314
|
+
if self.model.auth.is_multi_auth():
|
|
315
|
+
if not user_credentials:
|
|
316
|
+
available_schemes = [opt.scheme_name for opt in self.model.auth.options]
|
|
317
|
+
raise ValueError(f"Multi-auth connector requires credentials. Available schemes: {available_schemes}")
|
|
318
|
+
|
|
319
|
+
# If explicit scheme provided, use it directly
|
|
320
|
+
if explicit_scheme:
|
|
321
|
+
selected_option, transformed_secrets = self._select_auth_option(user_credentials, explicit_scheme)
|
|
322
|
+
else:
|
|
323
|
+
# Infer auth scheme from provided credentials
|
|
324
|
+
selected_option, transformed_secrets = self._infer_auth_scheme(user_credentials)
|
|
325
|
+
|
|
326
|
+
# Convert AuthOption to single-auth AuthConfig for HTTPClient
|
|
327
|
+
selected_auth_config = AuthConfig(
|
|
328
|
+
type=selected_option.type,
|
|
329
|
+
config=selected_option.config,
|
|
330
|
+
user_config_spec=None, # Not needed by HTTPClient
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
return (selected_auth_config, transformed_secrets)
|
|
334
|
+
|
|
335
|
+
# Single-auth: use existing logic
|
|
336
|
+
if user_credentials is not None:
|
|
337
|
+
# Apply mapping if this is auth_config (not legacy secrets)
|
|
338
|
+
transformed_secrets = self._apply_auth_config_mapping(user_credentials)
|
|
339
|
+
else:
|
|
340
|
+
transformed_secrets = None
|
|
341
|
+
|
|
342
|
+
return (self.model.auth, transformed_secrets)
|
|
343
|
+
|
|
344
|
+
def _infer_auth_scheme(
|
|
345
|
+
self,
|
|
346
|
+
user_credentials: dict[str, SecretStr],
|
|
347
|
+
) -> tuple[AuthOption, dict[str, SecretStr]]:
|
|
348
|
+
"""Infer authentication scheme from provided credentials.
|
|
349
|
+
|
|
350
|
+
Matches user credentials against each auth option's required fields
|
|
351
|
+
to determine which scheme to use.
|
|
352
|
+
|
|
353
|
+
Args:
|
|
354
|
+
user_credentials: User-provided credentials
|
|
355
|
+
|
|
356
|
+
Returns:
|
|
357
|
+
Tuple of (inferred AuthOption, transformed secrets)
|
|
358
|
+
|
|
359
|
+
Raises:
|
|
360
|
+
ValueError: If no scheme matches, or multiple schemes match
|
|
361
|
+
"""
|
|
362
|
+
options = self.model.auth.options
|
|
363
|
+
if not options:
|
|
364
|
+
raise ValueError("No auth options available in multi-auth config")
|
|
365
|
+
|
|
366
|
+
# Get the credential keys provided by the user
|
|
367
|
+
provided_keys = set(user_credentials.keys())
|
|
368
|
+
|
|
369
|
+
# Find all options where all required fields are present
|
|
370
|
+
matching_options: list[AuthOption] = []
|
|
371
|
+
for option in options:
|
|
372
|
+
if option.user_config_spec and option.user_config_spec.required:
|
|
373
|
+
required_fields = set(option.user_config_spec.required)
|
|
374
|
+
if required_fields.issubset(provided_keys):
|
|
375
|
+
matching_options.append(option)
|
|
376
|
+
elif not option.user_config_spec or not option.user_config_spec.required:
|
|
377
|
+
# Option has no required fields - it matches any credentials
|
|
378
|
+
matching_options.append(option)
|
|
379
|
+
|
|
380
|
+
# Handle matching results
|
|
381
|
+
if len(matching_options) == 0:
|
|
382
|
+
# No matches - provide helpful error message
|
|
383
|
+
scheme_requirements = []
|
|
384
|
+
for opt in options:
|
|
385
|
+
required = opt.user_config_spec.required if opt.user_config_spec and opt.user_config_spec.required else []
|
|
386
|
+
scheme_requirements.append(f" - {opt.scheme_name}: requires {required}")
|
|
387
|
+
raise ValueError(
|
|
388
|
+
f"Could not infer auth scheme from provided credentials. "
|
|
389
|
+
f"Provided keys: {list(provided_keys)}. "
|
|
390
|
+
f"Available schemes and their required fields:\n" + "\n".join(scheme_requirements)
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
if len(matching_options) > 1:
|
|
394
|
+
# Multiple matches - need explicit scheme
|
|
395
|
+
matching_names = [opt.scheme_name for opt in matching_options]
|
|
396
|
+
raise ValueError(
|
|
397
|
+
f"Multiple auth schemes match the provided credentials: {matching_names}. Please specify 'auth_scheme' explicitly to disambiguate."
|
|
398
|
+
)
|
|
399
|
+
|
|
400
|
+
# Exactly one match - use it
|
|
401
|
+
selected_option = matching_options[0]
|
|
402
|
+
transformed_secrets = self._apply_auth_mapping_for_option(user_credentials, selected_option)
|
|
403
|
+
return (selected_option, transformed_secrets)
|
|
404
|
+
|
|
405
|
+
def _select_auth_option(
|
|
406
|
+
self,
|
|
407
|
+
user_credentials: dict[str, SecretStr],
|
|
408
|
+
scheme_name: str,
|
|
409
|
+
) -> tuple[AuthOption, dict[str, SecretStr]]:
|
|
410
|
+
"""Select authentication option by explicit scheme name.
|
|
411
|
+
|
|
412
|
+
Args:
|
|
413
|
+
user_credentials: User-provided credentials
|
|
414
|
+
scheme_name: Explicit scheme name (e.g., "githubOAuth")
|
|
415
|
+
|
|
416
|
+
Returns:
|
|
417
|
+
Tuple of (selected AuthOption, transformed secrets)
|
|
418
|
+
|
|
419
|
+
Raises:
|
|
420
|
+
ValueError: If scheme not found
|
|
421
|
+
"""
|
|
422
|
+
options = self.model.auth.options
|
|
423
|
+
if not options:
|
|
424
|
+
raise ValueError("No auth options available in multi-auth config")
|
|
425
|
+
|
|
426
|
+
# Find matching scheme
|
|
427
|
+
for option in options:
|
|
428
|
+
if option.scheme_name == scheme_name:
|
|
429
|
+
transformed_secrets = self._apply_auth_mapping_for_option(user_credentials, option)
|
|
430
|
+
return (option, transformed_secrets)
|
|
431
|
+
|
|
432
|
+
# Scheme not found
|
|
433
|
+
available = [opt.scheme_name for opt in options]
|
|
434
|
+
raise ValueError(f"Auth scheme '{scheme_name}' not found. Available schemes: {available}")
|
|
435
|
+
|
|
436
|
+
def _apply_auth_mapping_for_option(
|
|
437
|
+
self,
|
|
438
|
+
user_credentials: dict[str, SecretStr],
|
|
439
|
+
option: AuthOption,
|
|
440
|
+
) -> dict[str, SecretStr]:
|
|
441
|
+
"""Apply auth mapping for a specific auth option.
|
|
442
|
+
|
|
443
|
+
Transforms user credentials using the option's auth_mapping templates.
|
|
444
|
+
|
|
445
|
+
Args:
|
|
446
|
+
user_credentials: User-provided credentials
|
|
447
|
+
option: AuthOption to apply
|
|
448
|
+
|
|
449
|
+
Returns:
|
|
450
|
+
Transformed secrets after applying auth_mapping
|
|
451
|
+
|
|
452
|
+
Raises:
|
|
453
|
+
ValueError: If required fields are missing or mapping fails
|
|
454
|
+
"""
|
|
455
|
+
if not option.user_config_spec:
|
|
456
|
+
# No mapping defined, use credentials as-is
|
|
457
|
+
return user_credentials
|
|
458
|
+
|
|
459
|
+
# Extract auth_mapping and required fields
|
|
460
|
+
user_config_spec = option.user_config_spec
|
|
461
|
+
auth_mapping = user_config_spec.auth_mapping
|
|
462
|
+
required_fields = user_config_spec.required
|
|
463
|
+
|
|
464
|
+
if not auth_mapping:
|
|
465
|
+
raise ValueError(f"No auth_mapping found for scheme '{option.scheme_name}'")
|
|
466
|
+
|
|
467
|
+
# Convert SecretStr to plain strings for template processing
|
|
468
|
+
user_config_values = {
|
|
469
|
+
key: (value.get_secret_value() if hasattr(value, "get_secret_value") else str(value)) for key, value in user_credentials.items()
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
# Apply the auth_mapping templates
|
|
473
|
+
mapped_values = apply_auth_mapping(auth_mapping, user_config_values, required_fields=required_fields)
|
|
474
|
+
|
|
475
|
+
# Convert back to SecretStr
|
|
476
|
+
return {key: SecretStr(value) for key, value in mapped_values.items()}
|
|
477
|
+
|
|
478
|
+
async def execute(self, config: ExecutionConfig) -> ExecutionResult:
|
|
479
|
+
"""Execute connector operation using handler pattern.
|
|
480
|
+
|
|
481
|
+
Args:
|
|
482
|
+
config: Execution configuration (entity, action, params)
|
|
483
|
+
|
|
484
|
+
Returns:
|
|
485
|
+
ExecutionResult with success/failure status and data
|
|
486
|
+
|
|
487
|
+
Example:
|
|
488
|
+
config = ExecutionConfig(
|
|
489
|
+
entity="customers",
|
|
490
|
+
action="list",
|
|
491
|
+
params={"limit": 10}
|
|
492
|
+
)
|
|
493
|
+
result = await executor.execute(config)
|
|
494
|
+
if result.success:
|
|
495
|
+
print(result.data)
|
|
496
|
+
"""
|
|
497
|
+
try:
|
|
498
|
+
# Check for hosted-only actions before converting to Action enum
|
|
499
|
+
if config.action == "search":
|
|
500
|
+
raise NotImplementedError(
|
|
501
|
+
"search is only available in hosted execution mode. "
|
|
502
|
+
"Initialize the connector with external_user_id, airbyte_client_id, "
|
|
503
|
+
"and airbyte_client_secret to use this feature."
|
|
504
|
+
)
|
|
505
|
+
|
|
506
|
+
# Convert config to internal format
|
|
507
|
+
action = Action(config.action) if isinstance(config.action, str) else config.action
|
|
508
|
+
params = config.params or {}
|
|
509
|
+
|
|
510
|
+
# Dispatch to handler (handlers handle telemetry internally)
|
|
511
|
+
handler = next((h for h in self._operation_handlers if h.can_handle(action)), None)
|
|
512
|
+
if not handler:
|
|
513
|
+
raise ExecutorError(f"No handler registered for action '{action.value}'.")
|
|
514
|
+
|
|
515
|
+
# Execute handler
|
|
516
|
+
result = handler.execute_operation(config.entity, action, params)
|
|
517
|
+
|
|
518
|
+
# Check if it's an async generator (download) or awaitable (standard)
|
|
519
|
+
if inspect.isasyncgen(result):
|
|
520
|
+
# Download operation: return generator directly
|
|
521
|
+
return ExecutionResult(
|
|
522
|
+
success=True,
|
|
523
|
+
data=result,
|
|
524
|
+
error=None,
|
|
525
|
+
meta=None,
|
|
526
|
+
)
|
|
527
|
+
else:
|
|
528
|
+
# Standard operation: await and extract data and metadata
|
|
529
|
+
handler_result = await result
|
|
530
|
+
return ExecutionResult(
|
|
531
|
+
success=True,
|
|
532
|
+
data=handler_result.data,
|
|
533
|
+
error=None,
|
|
534
|
+
meta=handler_result.metadata,
|
|
535
|
+
)
|
|
536
|
+
|
|
537
|
+
except (
|
|
538
|
+
EntityNotFoundError,
|
|
539
|
+
ActionNotSupportedError,
|
|
540
|
+
MissingParameterError,
|
|
541
|
+
InvalidParameterError,
|
|
542
|
+
) as e:
|
|
543
|
+
# These are "expected" execution errors - return them in ExecutionResult
|
|
544
|
+
return ExecutionResult(success=False, data={}, error=str(e))
|
|
545
|
+
|
|
546
|
+
async def _execute_operation(
|
|
547
|
+
self,
|
|
548
|
+
entity: str,
|
|
549
|
+
action: str | Action,
|
|
550
|
+
params: dict[str, Any] | None = None,
|
|
551
|
+
) -> dict[str, Any]:
|
|
552
|
+
"""Internal method: Execute an action on an entity asynchronously.
|
|
553
|
+
|
|
554
|
+
This method now delegates to the appropriate handler and extracts just the data.
|
|
555
|
+
External code should use execute(config) instead for full ExecutionResult with metadata.
|
|
556
|
+
|
|
557
|
+
Args:
|
|
558
|
+
entity: Entity name (e.g., "Customer")
|
|
559
|
+
action: Action to execute (e.g., "get" or Action.GET)
|
|
560
|
+
params: Parameters for the operation
|
|
561
|
+
- For GET: {"id": "cus_123"} for path params
|
|
562
|
+
- For LIST: {"limit": 10} for query params
|
|
563
|
+
- For CREATE/UPDATE: {"email": "...", "name": "..."} for body
|
|
564
|
+
|
|
565
|
+
Returns:
|
|
566
|
+
API response as dictionary
|
|
567
|
+
|
|
568
|
+
Raises:
|
|
569
|
+
ValueError: If entity or action not found
|
|
570
|
+
HTTPClientError: If API request fails
|
|
571
|
+
"""
|
|
572
|
+
params = params or {}
|
|
573
|
+
action = Action(action) if isinstance(action, str) else action
|
|
574
|
+
|
|
575
|
+
# Delegate to the appropriate handler
|
|
576
|
+
handler = next((h for h in self._operation_handlers if h.can_handle(action)), None)
|
|
577
|
+
if not handler:
|
|
578
|
+
raise ExecutorError(f"No handler registered for action '{action.value}'.")
|
|
579
|
+
|
|
580
|
+
# Execute handler and extract just the data for backward compatibility
|
|
581
|
+
result = await handler.execute_operation(entity, action, params)
|
|
582
|
+
if isinstance(result, StandardExecuteResult):
|
|
583
|
+
return result.data
|
|
584
|
+
else:
|
|
585
|
+
# Download operation returns AsyncIterator directly
|
|
586
|
+
return result
|
|
587
|
+
|
|
588
|
+
async def execute_batch(self, operations: list[tuple[str, str | Action, dict[str, Any] | None]]) -> list[dict[str, Any] | AsyncIterator[bytes]]:
|
|
589
|
+
"""Execute multiple operations concurrently (supports all action types including download).
|
|
590
|
+
|
|
591
|
+
Args:
|
|
592
|
+
operations: List of (entity, action, params) tuples
|
|
593
|
+
|
|
594
|
+
Returns:
|
|
595
|
+
List of responses in the same order as operations.
|
|
596
|
+
Standard operations return dict[str, Any].
|
|
597
|
+
Download operations return AsyncIterator[bytes].
|
|
598
|
+
|
|
599
|
+
Raises:
|
|
600
|
+
ValueError: If any entity or action not found
|
|
601
|
+
HTTPClientError: If any API request fails
|
|
602
|
+
|
|
603
|
+
Example:
|
|
604
|
+
results = await executor.execute_batch([
|
|
605
|
+
("Customer", "list", {"limit": 10}),
|
|
606
|
+
("Customer", "get", {"id": "cus_123"}),
|
|
607
|
+
("attachments", "download", {"id": "att_456"}),
|
|
608
|
+
])
|
|
609
|
+
"""
|
|
610
|
+
# Build tasks by dispatching directly to handlers
|
|
611
|
+
tasks = []
|
|
612
|
+
for entity, action, params in operations:
|
|
613
|
+
# Convert action to Action enum if needed
|
|
614
|
+
action = Action(action) if isinstance(action, str) else action
|
|
615
|
+
params = params or {}
|
|
616
|
+
|
|
617
|
+
# Find appropriate handler
|
|
618
|
+
handler = next((h for h in self._operation_handlers if h.can_handle(action)), None)
|
|
619
|
+
if not handler:
|
|
620
|
+
raise ExecutorError(f"No handler registered for action '{action.value}'.")
|
|
621
|
+
|
|
622
|
+
# Call handler directly (exceptions propagate naturally)
|
|
623
|
+
tasks.append(handler.execute_operation(entity, action, params))
|
|
624
|
+
|
|
625
|
+
# Execute all tasks concurrently - exceptions propagate via asyncio.gather
|
|
626
|
+
results = await asyncio.gather(*tasks)
|
|
627
|
+
|
|
628
|
+
# Extract data from results
|
|
629
|
+
extracted_results = []
|
|
630
|
+
for result in results:
|
|
631
|
+
if isinstance(result, StandardExecuteResult):
|
|
632
|
+
# Standard operation: extract data
|
|
633
|
+
extracted_results.append(result.data)
|
|
634
|
+
else:
|
|
635
|
+
# Download operation: return iterator as-is
|
|
636
|
+
extracted_results.append(result)
|
|
637
|
+
|
|
638
|
+
return extracted_results
|
|
639
|
+
|
|
640
|
+
def _build_path(self, path_template: str, params: dict[str, Any]) -> str:
|
|
641
|
+
"""Build path by replacing {param} placeholders with URL-encoded values.
|
|
642
|
+
|
|
643
|
+
Args:
|
|
644
|
+
path_template: Path with placeholders (e.g., /v1/customers/{id})
|
|
645
|
+
params: Parameters containing values for placeholders
|
|
646
|
+
|
|
647
|
+
Returns:
|
|
648
|
+
Completed path with URL-encoded values (e.g., /v1/customers/cus_123)
|
|
649
|
+
|
|
650
|
+
Raises:
|
|
651
|
+
MissingParameterError: If required path parameter is missing
|
|
652
|
+
"""
|
|
653
|
+
placeholders = re.findall(r"\{(\w+)\}", path_template)
|
|
654
|
+
|
|
655
|
+
path = path_template
|
|
656
|
+
for placeholder in placeholders:
|
|
657
|
+
if placeholder not in params:
|
|
658
|
+
raise MissingParameterError(
|
|
659
|
+
f"Missing required path parameter '{placeholder}' for path '{path_template}'. Provided parameters: {list(params.keys())}"
|
|
660
|
+
)
|
|
661
|
+
|
|
662
|
+
# Validate parameter value is not None or empty string
|
|
663
|
+
value = params[placeholder]
|
|
664
|
+
if value is None or (isinstance(value, str) and value.strip() == ""):
|
|
665
|
+
raise InvalidParameterError(f"Path parameter '{placeholder}' cannot be None or empty string")
|
|
666
|
+
|
|
667
|
+
encoded_value = quote(str(value), safe="")
|
|
668
|
+
path = path.replace(f"{{{placeholder}}}", encoded_value)
|
|
669
|
+
|
|
670
|
+
return path
|
|
671
|
+
|
|
672
|
+
def _extract_query_params(self, allowed_params: list[str], params: dict[str, Any]) -> dict[str, Any]:
|
|
673
|
+
"""Extract query parameters from params.
|
|
674
|
+
|
|
675
|
+
Args:
|
|
676
|
+
allowed_params: List of allowed query parameter names
|
|
677
|
+
params: All parameters
|
|
678
|
+
|
|
679
|
+
Returns:
|
|
680
|
+
Dictionary of query parameters
|
|
681
|
+
"""
|
|
682
|
+
return {key: value for key, value in params.items() if key in allowed_params}
|
|
683
|
+
|
|
684
|
+
def _extract_body(self, allowed_fields: list[str], params: dict[str, Any]) -> dict[str, Any]:
|
|
685
|
+
"""Extract body fields from params, filtering out None values.
|
|
686
|
+
|
|
687
|
+
Args:
|
|
688
|
+
allowed_fields: List of allowed body field names
|
|
689
|
+
params: All parameters
|
|
690
|
+
|
|
691
|
+
Returns:
|
|
692
|
+
Dictionary of body fields with None values filtered out
|
|
693
|
+
"""
|
|
694
|
+
return {key: value for key, value in params.items() if key in allowed_fields and value is not None}
|
|
695
|
+
|
|
696
|
+
def _serialize_deep_object_params(self, params: dict[str, Any], deep_object_param_names: list[str]) -> dict[str, Any]:
|
|
697
|
+
"""Serialize deepObject parameters to bracket notation format.
|
|
698
|
+
|
|
699
|
+
Converts nested dict parameters to the deepObject format expected by APIs
|
|
700
|
+
like Stripe. For example:
|
|
701
|
+
- Input: {'created': {'gte': 123, 'lte': 456}}
|
|
702
|
+
- Output: {'created[gte]': 123, 'created[lte]': 456}
|
|
703
|
+
|
|
704
|
+
Args:
|
|
705
|
+
params: Query parameters dict (may contain nested dicts)
|
|
706
|
+
deep_object_param_names: List of parameter names that use deepObject style
|
|
707
|
+
|
|
708
|
+
Returns:
|
|
709
|
+
Dictionary with deepObject params serialized to bracket notation
|
|
710
|
+
"""
|
|
711
|
+
serialized = {}
|
|
712
|
+
|
|
713
|
+
for key, value in params.items():
|
|
714
|
+
if key in deep_object_param_names and isinstance(value, dict):
|
|
715
|
+
# Serialize nested dict to bracket notation
|
|
716
|
+
for subkey, subvalue in value.items():
|
|
717
|
+
if subvalue is not None: # Skip None values
|
|
718
|
+
serialized[f"{key}[{subkey}]"] = subvalue
|
|
719
|
+
else:
|
|
720
|
+
# Keep non-deepObject params as-is (already validated by _extract_query_params)
|
|
721
|
+
serialized[key] = value
|
|
722
|
+
|
|
723
|
+
return serialized
|
|
724
|
+
|
|
725
|
+
@staticmethod
|
|
726
|
+
def _extract_download_url(
|
|
727
|
+
response: dict[str, Any],
|
|
728
|
+
file_field: str,
|
|
729
|
+
entity: str,
|
|
730
|
+
) -> str:
|
|
731
|
+
"""Extract download URL from metadata response using x-airbyte-file-url.
|
|
732
|
+
|
|
733
|
+
Supports both simple dot notation (e.g., "article.content_url") and array
|
|
734
|
+
indexing with bracket notation (e.g., "calls[0].media.audioUrl").
|
|
735
|
+
|
|
736
|
+
Args:
|
|
737
|
+
response: Metadata response containing file reference
|
|
738
|
+
file_field: JSON path to file URL field (from x-airbyte-file-url)
|
|
739
|
+
entity: Entity name (for error messages)
|
|
740
|
+
|
|
741
|
+
Returns:
|
|
742
|
+
Extracted file URL
|
|
743
|
+
|
|
744
|
+
Raises:
|
|
745
|
+
ExecutorError: If file field not found or invalid
|
|
746
|
+
"""
|
|
747
|
+
# Navigate nested path (e.g., "article_attachment.content_url" or "calls[0].media.audioUrl")
|
|
748
|
+
parts = file_field.split(".")
|
|
749
|
+
current = response
|
|
750
|
+
|
|
751
|
+
for i, part in enumerate(parts):
|
|
752
|
+
# Check if part has array indexing (e.g., "calls[0]")
|
|
753
|
+
array_match = re.match(r"^(\w+)\[(\d+)\]$", part)
|
|
754
|
+
|
|
755
|
+
if array_match:
|
|
756
|
+
field_name = array_match.group(1)
|
|
757
|
+
index = int(array_match.group(2))
|
|
758
|
+
|
|
759
|
+
# Navigate to the field
|
|
760
|
+
if not isinstance(current, dict):
|
|
761
|
+
raise ExecutorError(
|
|
762
|
+
f"Cannot extract download URL for {entity}: Expected dict at '{'.'.join(parts[:i])}', got {type(current).__name__}"
|
|
763
|
+
)
|
|
764
|
+
|
|
765
|
+
if field_name not in current:
|
|
766
|
+
raise ExecutorError(
|
|
767
|
+
f"Cannot extract download URL for {entity}: "
|
|
768
|
+
f"Field '{field_name}' not found in response. Available fields: {list(current.keys())}"
|
|
769
|
+
)
|
|
770
|
+
|
|
771
|
+
# Get the array
|
|
772
|
+
array_value = current[field_name]
|
|
773
|
+
if not isinstance(array_value, list):
|
|
774
|
+
raise ExecutorError(
|
|
775
|
+
f"Cannot extract download URL for {entity}: Expected list at '{field_name}', got {type(array_value).__name__}"
|
|
776
|
+
)
|
|
777
|
+
|
|
778
|
+
# Check index bounds
|
|
779
|
+
if index >= len(array_value):
|
|
780
|
+
raise ExecutorError(
|
|
781
|
+
f"Cannot extract download URL for {entity}: Index {index} out of bounds for '{field_name}' (length: {len(array_value)})"
|
|
782
|
+
)
|
|
783
|
+
|
|
784
|
+
current = array_value[index]
|
|
785
|
+
else:
|
|
786
|
+
# Regular dict navigation
|
|
787
|
+
if not isinstance(current, dict):
|
|
788
|
+
raise ExecutorError(
|
|
789
|
+
f"Cannot extract download URL for {entity}: Expected dict at '{'.'.join(parts[:i])}', got {type(current).__name__}"
|
|
790
|
+
)
|
|
791
|
+
|
|
792
|
+
if part not in current:
|
|
793
|
+
raise ExecutorError(
|
|
794
|
+
f"Cannot extract download URL for {entity}: Field '{part}' not found in response. Available fields: {list(current.keys())}"
|
|
795
|
+
)
|
|
796
|
+
|
|
797
|
+
current = current[part]
|
|
798
|
+
|
|
799
|
+
if not isinstance(current, str):
|
|
800
|
+
raise ExecutorError(f"Cannot extract download URL for {entity}: Expected string at '{file_field}', got {type(current).__name__}")
|
|
801
|
+
|
|
802
|
+
return current
|
|
803
|
+
|
|
804
|
+
@staticmethod
|
|
805
|
+
def _substitute_file_field_params(
|
|
806
|
+
file_field: str,
|
|
807
|
+
params: dict[str, Any],
|
|
808
|
+
) -> str:
|
|
809
|
+
"""Substitute template variables in file_field with parameter values.
|
|
810
|
+
|
|
811
|
+
Uses Jinja2 with custom delimiters to support OpenAPI-style syntax like
|
|
812
|
+
"attachments[{index}].url" where {index} is replaced with params["index"].
|
|
813
|
+
|
|
814
|
+
Args:
|
|
815
|
+
file_field: File field path with optional template variables
|
|
816
|
+
params: Parameters from execute() call
|
|
817
|
+
|
|
818
|
+
Returns:
|
|
819
|
+
File field with template variables substituted
|
|
820
|
+
|
|
821
|
+
Example:
|
|
822
|
+
>>> _substitute_file_field_params("attachments[{attachment_index}].url", {"attachment_index": 0})
|
|
823
|
+
"attachments[0].url"
|
|
824
|
+
"""
|
|
825
|
+
|
|
826
|
+
# Use custom delimiters to match OpenAPI path parameter syntax {var}
|
|
827
|
+
# StrictUndefined raises clear error if a template variable is missing
|
|
828
|
+
env = Environment(
|
|
829
|
+
variable_start_string="{",
|
|
830
|
+
variable_end_string="}",
|
|
831
|
+
undefined=StrictUndefined,
|
|
832
|
+
)
|
|
833
|
+
template = env.from_string(file_field)
|
|
834
|
+
return template.render(params)
|
|
835
|
+
|
|
836
|
+
def _build_request_body(self, endpoint: EndpointDefinition, params: dict[str, Any]) -> dict[str, Any] | None:
|
|
837
|
+
"""Build request body (GraphQL or standard).
|
|
838
|
+
|
|
839
|
+
Args:
|
|
840
|
+
endpoint: Endpoint definition
|
|
841
|
+
params: Parameters from execute() call
|
|
842
|
+
|
|
843
|
+
Returns:
|
|
844
|
+
Request body dict or None if no body needed
|
|
845
|
+
"""
|
|
846
|
+
if endpoint.graphql_body:
|
|
847
|
+
# Extract defaults from query_params_schema for GraphQL variable interpolation
|
|
848
|
+
param_defaults = {name: schema.get("default") for name, schema in endpoint.query_params_schema.items() if "default" in schema}
|
|
849
|
+
return self._build_graphql_body(endpoint.graphql_body, params, param_defaults)
|
|
850
|
+
elif endpoint.body_fields:
|
|
851
|
+
return self._extract_body(endpoint.body_fields, params)
|
|
852
|
+
return None
|
|
853
|
+
|
|
854
|
+
def _flatten_form_data(self, data: dict[str, Any], parent_key: str = "") -> dict[str, Any]:
|
|
855
|
+
"""Flatten nested dict/list structures into bracket notation for form encoding.
|
|
856
|
+
|
|
857
|
+
Stripe and similar APIs require nested arrays/objects to be encoded using bracket
|
|
858
|
+
notation when using application/x-www-form-urlencoded content type.
|
|
859
|
+
|
|
860
|
+
Args:
|
|
861
|
+
data: Nested dict with arrays/objects to flatten
|
|
862
|
+
parent_key: Parent key for nested structures (used in recursion)
|
|
863
|
+
|
|
864
|
+
Returns:
|
|
865
|
+
Flattened dict with bracket notation keys
|
|
866
|
+
|
|
867
|
+
Examples:
|
|
868
|
+
>>> _flatten_form_data({"items": [{"price": "p1", "qty": 1}]})
|
|
869
|
+
{"items[0][price]": "p1", "items[0][qty]": 1}
|
|
870
|
+
|
|
871
|
+
>>> _flatten_form_data({"customer": "cus_123", "metadata": {"key": "value"}})
|
|
872
|
+
{"customer": "cus_123", "metadata[key]": "value"}
|
|
873
|
+
"""
|
|
874
|
+
flattened = {}
|
|
875
|
+
|
|
876
|
+
for key, value in data.items():
|
|
877
|
+
new_key = f"{parent_key}[{key}]" if parent_key else key
|
|
878
|
+
|
|
879
|
+
if isinstance(value, dict):
|
|
880
|
+
# Recursively flatten nested dicts
|
|
881
|
+
flattened.update(self._flatten_form_data(value, new_key))
|
|
882
|
+
elif isinstance(value, list):
|
|
883
|
+
# Flatten arrays with indexed bracket notation
|
|
884
|
+
for i, item in enumerate(value):
|
|
885
|
+
indexed_key = f"{new_key}[{i}]"
|
|
886
|
+
if isinstance(item, dict):
|
|
887
|
+
# Nested dict in array - recurse
|
|
888
|
+
flattened.update(self._flatten_form_data(item, indexed_key))
|
|
889
|
+
elif isinstance(item, list):
|
|
890
|
+
# Nested list in array - recurse
|
|
891
|
+
flattened.update(self._flatten_form_data({str(i): item}, new_key))
|
|
892
|
+
else:
|
|
893
|
+
# Primitive value in array
|
|
894
|
+
flattened[indexed_key] = item
|
|
895
|
+
else:
|
|
896
|
+
# Primitive value - add directly
|
|
897
|
+
flattened[new_key] = value
|
|
898
|
+
|
|
899
|
+
return flattened
|
|
900
|
+
|
|
901
|
+
def _determine_request_format(self, endpoint: EndpointDefinition, body: dict[str, Any] | None) -> dict[str, Any]:
|
|
902
|
+
"""Determine json/data parameters for HTTP request.
|
|
903
|
+
|
|
904
|
+
GraphQL always uses JSON, regardless of content_type setting.
|
|
905
|
+
For form-encoded requests, nested structures are flattened into bracket notation.
|
|
906
|
+
|
|
907
|
+
Args:
|
|
908
|
+
endpoint: Endpoint definition
|
|
909
|
+
body: Request body dict or None
|
|
910
|
+
|
|
911
|
+
Returns:
|
|
912
|
+
Dict with 'json' and/or 'data' keys for http_client.request()
|
|
913
|
+
"""
|
|
914
|
+
if not body:
|
|
915
|
+
return {}
|
|
916
|
+
|
|
917
|
+
is_graphql = endpoint.graphql_body is not None
|
|
918
|
+
|
|
919
|
+
if is_graphql or endpoint.content_type.value == "application/json":
|
|
920
|
+
return {"json": body}
|
|
921
|
+
elif endpoint.content_type.value == "application/x-www-form-urlencoded":
|
|
922
|
+
# Flatten nested structures for form encoding
|
|
923
|
+
flattened_body = self._flatten_form_data(body)
|
|
924
|
+
return {"data": flattened_body}
|
|
925
|
+
|
|
926
|
+
return {}
|
|
927
|
+
|
|
928
|
+
def _process_graphql_fields(self, query: str, graphql_config: dict[str, Any], params: dict[str, Any]) -> str:
|
|
929
|
+
"""Process GraphQL query field selection.
|
|
930
|
+
|
|
931
|
+
Handles:
|
|
932
|
+
- Dynamic fields from params['fields']
|
|
933
|
+
- Default fields from config
|
|
934
|
+
- String vs list format for default_fields
|
|
935
|
+
|
|
936
|
+
Args:
|
|
937
|
+
query: GraphQL query string (may contain {{ fields }} placeholder)
|
|
938
|
+
graphql_config: GraphQL configuration dict
|
|
939
|
+
params: Parameters from execute() call
|
|
940
|
+
|
|
941
|
+
Returns:
|
|
942
|
+
Processed query string with fields injected
|
|
943
|
+
"""
|
|
944
|
+
if "{{ fields }}" not in query:
|
|
945
|
+
return query
|
|
946
|
+
|
|
947
|
+
# Check for explicit fields parameter
|
|
948
|
+
if "fields" in params and params["fields"]:
|
|
949
|
+
return self._inject_graphql_fields(query, params["fields"])
|
|
950
|
+
|
|
951
|
+
# Use default fields if available
|
|
952
|
+
if "default_fields" not in graphql_config:
|
|
953
|
+
return query # Placeholder remains (could raise error in the future)
|
|
954
|
+
|
|
955
|
+
default_fields = graphql_config["default_fields"]
|
|
956
|
+
if isinstance(default_fields, str):
|
|
957
|
+
# Already in GraphQL format - direct replacement
|
|
958
|
+
return query.replace("{{ fields }}", default_fields)
|
|
959
|
+
elif isinstance(default_fields, list):
|
|
960
|
+
# List format - convert to GraphQL
|
|
961
|
+
return self._inject_graphql_fields(query, default_fields)
|
|
962
|
+
|
|
963
|
+
return query
|
|
964
|
+
|
|
965
|
+
def _build_graphql_body(
|
|
966
|
+
self,
|
|
967
|
+
graphql_config: dict[str, Any],
|
|
968
|
+
params: dict[str, Any],
|
|
969
|
+
param_defaults: dict[str, Any] | None = None,
|
|
970
|
+
) -> dict[str, Any]:
|
|
971
|
+
"""Build GraphQL request body with variable substitution and field selection.
|
|
972
|
+
|
|
973
|
+
Args:
|
|
974
|
+
graphql_config: GraphQL configuration from x-airbyte-body-type extension
|
|
975
|
+
params: Parameters from execute() call
|
|
976
|
+
param_defaults: Default values for params from query_params_schema
|
|
977
|
+
|
|
978
|
+
Returns:
|
|
979
|
+
GraphQL request body: {"query": "...", "variables": {...}}
|
|
980
|
+
"""
|
|
981
|
+
query = graphql_config["query"]
|
|
982
|
+
|
|
983
|
+
# Process field selection (dynamic fields or default fields)
|
|
984
|
+
query = self._process_graphql_fields(query, graphql_config, params)
|
|
985
|
+
|
|
986
|
+
body = {"query": query}
|
|
987
|
+
|
|
988
|
+
# Substitute variables from params
|
|
989
|
+
if "variables" in graphql_config and graphql_config["variables"]:
|
|
990
|
+
variables = self._interpolate_variables(graphql_config["variables"], params, param_defaults)
|
|
991
|
+
# Filter out None values (optional fields not provided) - matches REST _extract_body() behavior
|
|
992
|
+
body["variables"] = {k: v for k, v in variables.items() if v is not None}
|
|
993
|
+
|
|
994
|
+
# Add operation name if specified
|
|
995
|
+
if "operationName" in graphql_config:
|
|
996
|
+
body["operationName"] = graphql_config["operationName"]
|
|
997
|
+
|
|
998
|
+
return body
|
|
999
|
+
|
|
1000
|
+
def _convert_nested_field_to_graphql(self, field: str) -> str:
|
|
1001
|
+
"""Convert dot-notation field to GraphQL field selection.
|
|
1002
|
+
|
|
1003
|
+
Example: "primaryLanguage.name" -> "primaryLanguage { name }"
|
|
1004
|
+
|
|
1005
|
+
Args:
|
|
1006
|
+
field: Field name in dot notation (e.g., "primaryLanguage.name")
|
|
1007
|
+
|
|
1008
|
+
Returns:
|
|
1009
|
+
GraphQL field selection string
|
|
1010
|
+
"""
|
|
1011
|
+
if "." not in field:
|
|
1012
|
+
return field
|
|
1013
|
+
|
|
1014
|
+
parts = field.split(".")
|
|
1015
|
+
result = parts[0]
|
|
1016
|
+
for part in parts[1:]:
|
|
1017
|
+
result += f" {{ {part}"
|
|
1018
|
+
result += " }" * (len(parts) - 1)
|
|
1019
|
+
return result
|
|
1020
|
+
|
|
1021
|
+
def _inject_graphql_fields(self, query: str, fields: list[str]) -> str:
|
|
1022
|
+
"""Inject field selection into GraphQL query.
|
|
1023
|
+
|
|
1024
|
+
Replaces field selection placeholders ({{ fields }}) with actual field list.
|
|
1025
|
+
Supports nested fields using dot notation (e.g., "primaryLanguage.name").
|
|
1026
|
+
|
|
1027
|
+
Args:
|
|
1028
|
+
query: GraphQL query string (may contain {{ fields }} placeholder)
|
|
1029
|
+
fields: List of fields to select (e.g., ["id", "name", "primaryLanguage.name"])
|
|
1030
|
+
|
|
1031
|
+
Returns:
|
|
1032
|
+
GraphQL query with fields injected
|
|
1033
|
+
|
|
1034
|
+
Example:
|
|
1035
|
+
Input query: "query { repository { {{ fields }} } }"
|
|
1036
|
+
Fields: ["id", "name", "primaryLanguage { name }"]
|
|
1037
|
+
Output: "query { repository { id name primaryLanguage { name } } }"
|
|
1038
|
+
"""
|
|
1039
|
+
# Check if query has field placeholder
|
|
1040
|
+
if "{{ fields }}" not in query:
|
|
1041
|
+
# No placeholder - return query as-is (backward compatible)
|
|
1042
|
+
return query
|
|
1043
|
+
|
|
1044
|
+
# Convert field list to GraphQL field selection
|
|
1045
|
+
graphql_fields = [self._convert_nested_field_to_graphql(field) for field in fields]
|
|
1046
|
+
|
|
1047
|
+
# Replace placeholder with field list
|
|
1048
|
+
fields_str = " ".join(graphql_fields)
|
|
1049
|
+
return query.replace("{{ fields }}", fields_str)
|
|
1050
|
+
|
|
1051
|
+
def _interpolate_variables(
|
|
1052
|
+
self,
|
|
1053
|
+
variables: dict[str, Any],
|
|
1054
|
+
params: dict[str, Any],
|
|
1055
|
+
param_defaults: dict[str, Any] | None = None,
|
|
1056
|
+
) -> dict[str, Any]:
|
|
1057
|
+
"""Recursively interpolate variables using params.
|
|
1058
|
+
|
|
1059
|
+
Preserves types (doesn't stringify everything).
|
|
1060
|
+
|
|
1061
|
+
Supports:
|
|
1062
|
+
- Direct replacement: "{{ owner }}" → params["owner"] (preserves type)
|
|
1063
|
+
- Nested objects: {"input": {"name": "{{ name }}"}}
|
|
1064
|
+
- Arrays: [{"id": "{{ id }}"}]
|
|
1065
|
+
- Default values: "{{ per_page }}" → param_defaults["per_page"] if not in params
|
|
1066
|
+
- Unsubstituted placeholders: "{{ states }}" → None (for optional params without defaults)
|
|
1067
|
+
|
|
1068
|
+
Args:
|
|
1069
|
+
variables: Variables dict with template placeholders
|
|
1070
|
+
params: Parameters to substitute
|
|
1071
|
+
param_defaults: Default values for params from query_params_schema
|
|
1072
|
+
|
|
1073
|
+
Returns:
|
|
1074
|
+
Interpolated variables dict with types preserved
|
|
1075
|
+
"""
|
|
1076
|
+
defaults = param_defaults or {}
|
|
1077
|
+
|
|
1078
|
+
def interpolate_value(value: Any) -> Any:
|
|
1079
|
+
if isinstance(value, str):
|
|
1080
|
+
# Check for exact template match (preserve type)
|
|
1081
|
+
for key, param_value in params.items():
|
|
1082
|
+
placeholder = f"{{{{ {key} }}}}"
|
|
1083
|
+
if value == placeholder:
|
|
1084
|
+
return param_value # Return actual value (int, list, etc.)
|
|
1085
|
+
elif placeholder in value:
|
|
1086
|
+
# Partial match - do string replacement
|
|
1087
|
+
value = value.replace(placeholder, str(param_value))
|
|
1088
|
+
|
|
1089
|
+
# Check if any unsubstituted placeholders remain
|
|
1090
|
+
if re.search(r"\{\{\s*\w+\s*\}\}", value):
|
|
1091
|
+
# Extract placeholder name and check for default value
|
|
1092
|
+
match = re.search(r"\{\{\s*(\w+)\s*\}\}", value)
|
|
1093
|
+
if match:
|
|
1094
|
+
param_name = match.group(1)
|
|
1095
|
+
if param_name in defaults:
|
|
1096
|
+
# Use default value (preserves type)
|
|
1097
|
+
return defaults[param_name]
|
|
1098
|
+
# No default found - return None (for optional params)
|
|
1099
|
+
return None
|
|
1100
|
+
|
|
1101
|
+
return value
|
|
1102
|
+
elif isinstance(value, dict):
|
|
1103
|
+
return {k: interpolate_value(v) for k, v in value.items()}
|
|
1104
|
+
elif isinstance(value, list):
|
|
1105
|
+
return [interpolate_value(item) for item in value]
|
|
1106
|
+
else:
|
|
1107
|
+
return value
|
|
1108
|
+
|
|
1109
|
+
return interpolate_value(variables)
|
|
1110
|
+
|
|
1111
|
+
def _wrap_primitives(self, data: Any) -> dict[str, Any] | list[dict[str, Any]] | None:
|
|
1112
|
+
"""Wrap primitive values in dict format for consistent response structure.
|
|
1113
|
+
|
|
1114
|
+
Transforms primitive API responses into dict format so downstream code
|
|
1115
|
+
can always expect dict-based data structures.
|
|
1116
|
+
|
|
1117
|
+
Args:
|
|
1118
|
+
data: Response data (could be primitive, list, dict, or None)
|
|
1119
|
+
|
|
1120
|
+
Returns:
|
|
1121
|
+
- If data is a primitive (str, int, float, bool): {"value": data}
|
|
1122
|
+
- If data is a list: wraps all non-dict elements as {"value": item}
|
|
1123
|
+
- If data is already a dict or list of dicts: unchanged
|
|
1124
|
+
- If data is None: None
|
|
1125
|
+
|
|
1126
|
+
Examples:
|
|
1127
|
+
>>> executor._wrap_primitives(42)
|
|
1128
|
+
{"value": 42}
|
|
1129
|
+
>>> executor._wrap_primitives([1, 2, 3])
|
|
1130
|
+
[{"value": 1}, {"value": 2}, {"value": 3}]
|
|
1131
|
+
>>> executor._wrap_primitives([1, {"id": 2}, 3])
|
|
1132
|
+
[{"value": 1}, {"id": 2}, {"value": 3}]
|
|
1133
|
+
>>> executor._wrap_primitives([[1, 2], 3])
|
|
1134
|
+
[{"value": [1, 2]}, {"value": 3}]
|
|
1135
|
+
>>> executor._wrap_primitives({"id": 1})
|
|
1136
|
+
{"id": 1} # unchanged
|
|
1137
|
+
"""
|
|
1138
|
+
if data is None:
|
|
1139
|
+
return None
|
|
1140
|
+
|
|
1141
|
+
# Handle primitive scalars
|
|
1142
|
+
if isinstance(data, (bool, str, int, float)):
|
|
1143
|
+
return {"value": data}
|
|
1144
|
+
|
|
1145
|
+
# Handle lists - wrap non-dict elements
|
|
1146
|
+
if isinstance(data, list):
|
|
1147
|
+
if not data:
|
|
1148
|
+
return [] # Empty list unchanged
|
|
1149
|
+
|
|
1150
|
+
wrapped = []
|
|
1151
|
+
for item in data:
|
|
1152
|
+
if isinstance(item, dict):
|
|
1153
|
+
wrapped.append(item)
|
|
1154
|
+
else:
|
|
1155
|
+
wrapped.append({"value": item})
|
|
1156
|
+
return wrapped
|
|
1157
|
+
|
|
1158
|
+
# Dict - return unchanged
|
|
1159
|
+
if isinstance(data, dict):
|
|
1160
|
+
return data
|
|
1161
|
+
|
|
1162
|
+
# Unknown type - wrap for safety
|
|
1163
|
+
return {"value": data}
|
|
1164
|
+
|
|
1165
|
+
def _extract_records(
|
|
1166
|
+
self,
|
|
1167
|
+
response_data: Any,
|
|
1168
|
+
endpoint: EndpointDefinition,
|
|
1169
|
+
) -> dict[str, Any] | list[dict[str, Any]] | None:
|
|
1170
|
+
"""Extract records from response using record extractor.
|
|
1171
|
+
|
|
1172
|
+
Type inference based on action:
|
|
1173
|
+
- list, search: Returns array ([] if not found)
|
|
1174
|
+
- get, create, update, delete: Returns single record (None if not found)
|
|
1175
|
+
|
|
1176
|
+
Automatically wraps primitive values (int, str, float, bool) in {"value": primitive}
|
|
1177
|
+
format to ensure consistent dict-based responses for downstream code.
|
|
1178
|
+
|
|
1179
|
+
Args:
|
|
1180
|
+
response_data: Full API response (can be dict, list, primitive, or None)
|
|
1181
|
+
endpoint: Endpoint with optional record extractor and action
|
|
1182
|
+
|
|
1183
|
+
Returns:
|
|
1184
|
+
- Extracted data if extractor configured and path found
|
|
1185
|
+
- [] or None if path not found (based on action)
|
|
1186
|
+
- Original response if no extractor configured or on error
|
|
1187
|
+
- Primitives are wrapped as {"value": primitive}
|
|
1188
|
+
"""
|
|
1189
|
+
# Check if endpoint has record extractor
|
|
1190
|
+
extractor = endpoint.record_extractor
|
|
1191
|
+
if not extractor:
|
|
1192
|
+
return self._wrap_primitives(response_data)
|
|
1193
|
+
|
|
1194
|
+
# Determine if this action returns array or single record
|
|
1195
|
+
action = endpoint.action
|
|
1196
|
+
if not action:
|
|
1197
|
+
return self._wrap_primitives(response_data)
|
|
1198
|
+
|
|
1199
|
+
is_array_action = action in (Action.LIST, Action.API_SEARCH)
|
|
1200
|
+
|
|
1201
|
+
try:
|
|
1202
|
+
# Parse and apply JSONPath expression
|
|
1203
|
+
jsonpath_expr = parse_jsonpath(extractor)
|
|
1204
|
+
matches = [match.value for match in jsonpath_expr.find(response_data)]
|
|
1205
|
+
|
|
1206
|
+
if not matches:
|
|
1207
|
+
# Path not found - return empty based on action
|
|
1208
|
+
return [] if is_array_action else None
|
|
1209
|
+
|
|
1210
|
+
# Return extracted data with primitive wrapping
|
|
1211
|
+
if is_array_action:
|
|
1212
|
+
# For array actions, return the array (or list of matches)
|
|
1213
|
+
result = matches[0] if len(matches) == 1 else matches
|
|
1214
|
+
else:
|
|
1215
|
+
# For single record actions, return first match
|
|
1216
|
+
result = matches[0]
|
|
1217
|
+
|
|
1218
|
+
return self._wrap_primitives(result)
|
|
1219
|
+
|
|
1220
|
+
except Exception as e:
|
|
1221
|
+
logging.warning(f"Failed to apply record extractor '{extractor}': {e}. Returning original response.")
|
|
1222
|
+
return self._wrap_primitives(response_data)
|
|
1223
|
+
|
|
1224
|
+
def _extract_metadata(
|
|
1225
|
+
self,
|
|
1226
|
+
response_data: dict[str, Any],
|
|
1227
|
+
response_headers: dict[str, str],
|
|
1228
|
+
endpoint: EndpointDefinition,
|
|
1229
|
+
) -> dict[str, Any] | None:
|
|
1230
|
+
"""Extract metadata from response using meta extractor.
|
|
1231
|
+
|
|
1232
|
+
Each field in meta_extractor dict is independently extracted using JSONPath
|
|
1233
|
+
for body extraction, or special prefixes for header extraction:
|
|
1234
|
+
- @link.{rel}: Extract URL from RFC 5988 Link header by rel type
|
|
1235
|
+
- @header.{name}: Extract raw header value by header name
|
|
1236
|
+
- Otherwise: JSONPath expression for body extraction
|
|
1237
|
+
|
|
1238
|
+
Missing or invalid paths result in None for that field (no crash).
|
|
1239
|
+
|
|
1240
|
+
Args:
|
|
1241
|
+
response_data: Full API response (before record extraction)
|
|
1242
|
+
response_headers: HTTP response headers
|
|
1243
|
+
endpoint: Endpoint with optional meta extractor configuration
|
|
1244
|
+
|
|
1245
|
+
Returns:
|
|
1246
|
+
- Dict of extracted metadata if extractor configured
|
|
1247
|
+
- None if no extractor configured
|
|
1248
|
+
- Dict with None values for failed extractions
|
|
1249
|
+
|
|
1250
|
+
Example:
|
|
1251
|
+
meta_extractor = {
|
|
1252
|
+
"pagination": "$.records",
|
|
1253
|
+
"request_id": "$.requestId",
|
|
1254
|
+
"next_page_url": "@link.next",
|
|
1255
|
+
"rate_limit": "@header.X-RateLimit-Remaining"
|
|
1256
|
+
}
|
|
1257
|
+
Returns: {
|
|
1258
|
+
"pagination": {"cursor": "abc", "total": 100},
|
|
1259
|
+
"request_id": "xyz123",
|
|
1260
|
+
"next_page_url": "https://api.example.com/data?cursor=abc",
|
|
1261
|
+
"rate_limit": "99"
|
|
1262
|
+
}
|
|
1263
|
+
"""
|
|
1264
|
+
# Check if endpoint has meta extractor
|
|
1265
|
+
if endpoint.meta_extractor is None:
|
|
1266
|
+
return None
|
|
1267
|
+
|
|
1268
|
+
extracted_meta: dict[str, Any] = {}
|
|
1269
|
+
|
|
1270
|
+
# Extract each field independently
|
|
1271
|
+
for field_name, extractor_expr in endpoint.meta_extractor.items():
|
|
1272
|
+
try:
|
|
1273
|
+
if extractor_expr.startswith("@link."):
|
|
1274
|
+
# RFC 5988 Link header extraction
|
|
1275
|
+
rel = extractor_expr[6:]
|
|
1276
|
+
extracted_meta[field_name] = self._extract_link_url(response_headers, rel)
|
|
1277
|
+
elif extractor_expr.startswith("@header."):
|
|
1278
|
+
# Raw header value extraction (case-insensitive lookup)
|
|
1279
|
+
header_name = extractor_expr[8:]
|
|
1280
|
+
extracted_meta[field_name] = self._get_header_value(response_headers, header_name)
|
|
1281
|
+
else:
|
|
1282
|
+
# JSONPath body extraction
|
|
1283
|
+
jsonpath_expr = parse_jsonpath(extractor_expr)
|
|
1284
|
+
matches = [match.value for match in jsonpath_expr.find(response_data)]
|
|
1285
|
+
|
|
1286
|
+
if matches:
|
|
1287
|
+
# Return first match (most common case)
|
|
1288
|
+
extracted_meta[field_name] = matches[0]
|
|
1289
|
+
else:
|
|
1290
|
+
# Path not found - set to None
|
|
1291
|
+
extracted_meta[field_name] = None
|
|
1292
|
+
|
|
1293
|
+
except Exception as e:
|
|
1294
|
+
# Log error but continue with other fields
|
|
1295
|
+
logging.warning(f"Failed to apply meta extractor for field '{field_name}' with expression '{extractor_expr}': {e}. Setting to None.")
|
|
1296
|
+
extracted_meta[field_name] = None
|
|
1297
|
+
|
|
1298
|
+
return extracted_meta
|
|
1299
|
+
|
|
1300
|
+
@staticmethod
|
|
1301
|
+
def _extract_link_url(headers: dict[str, str], rel: str) -> str | None:
|
|
1302
|
+
"""Extract URL from RFC 5988 Link header by rel type.
|
|
1303
|
+
|
|
1304
|
+
Parses Link header format: <url>; param1="value1"; rel="next"; param2="value2"
|
|
1305
|
+
|
|
1306
|
+
Supports:
|
|
1307
|
+
- Multiple parameters per link in any order
|
|
1308
|
+
- Both quoted and unquoted rel values
|
|
1309
|
+
- Multiple links separated by commas
|
|
1310
|
+
|
|
1311
|
+
Args:
|
|
1312
|
+
headers: Response headers dict
|
|
1313
|
+
rel: The rel type to extract (e.g., "next", "prev", "first", "last")
|
|
1314
|
+
|
|
1315
|
+
Returns:
|
|
1316
|
+
The URL for the specified rel type, or None if not found
|
|
1317
|
+
"""
|
|
1318
|
+
link_header = headers.get("Link") or headers.get("link", "")
|
|
1319
|
+
if not link_header:
|
|
1320
|
+
return None
|
|
1321
|
+
|
|
1322
|
+
for link_segment in re.split(r",(?=\s*<)", link_header):
|
|
1323
|
+
link_segment = link_segment.strip()
|
|
1324
|
+
|
|
1325
|
+
url_match = re.match(r"<([^>]+)>", link_segment)
|
|
1326
|
+
if not url_match:
|
|
1327
|
+
continue
|
|
1328
|
+
|
|
1329
|
+
url = url_match.group(1)
|
|
1330
|
+
params_str = link_segment[url_match.end() :]
|
|
1331
|
+
|
|
1332
|
+
rel_match = re.search(r';\s*rel="?([^";,]+)"?', params_str, re.IGNORECASE)
|
|
1333
|
+
if rel_match and rel_match.group(1).strip() == rel:
|
|
1334
|
+
return url
|
|
1335
|
+
|
|
1336
|
+
return None
|
|
1337
|
+
|
|
1338
|
+
@staticmethod
|
|
1339
|
+
def _get_header_value(headers: dict[str, str], header_name: str) -> str | None:
|
|
1340
|
+
"""Get header value with case-insensitive lookup.
|
|
1341
|
+
|
|
1342
|
+
Args:
|
|
1343
|
+
headers: Response headers dict
|
|
1344
|
+
header_name: Header name to look up
|
|
1345
|
+
|
|
1346
|
+
Returns:
|
|
1347
|
+
Header value or None if not found
|
|
1348
|
+
"""
|
|
1349
|
+
# Try exact match first
|
|
1350
|
+
if header_name in headers:
|
|
1351
|
+
return headers[header_name]
|
|
1352
|
+
|
|
1353
|
+
# Case-insensitive lookup
|
|
1354
|
+
header_name_lower = header_name.lower()
|
|
1355
|
+
for key, value in headers.items():
|
|
1356
|
+
if key.lower() == header_name_lower:
|
|
1357
|
+
return value
|
|
1358
|
+
|
|
1359
|
+
return None
|
|
1360
|
+
|
|
1361
|
+
def _validate_required_body_fields(self, endpoint: Any, params: dict[str, Any], action: Action, entity: str) -> None:
|
|
1362
|
+
"""Validate that required body fields are present for CREATE/UPDATE operations.
|
|
1363
|
+
|
|
1364
|
+
Args:
|
|
1365
|
+
endpoint: Endpoint definition
|
|
1366
|
+
params: Parameters provided
|
|
1367
|
+
action: Operation action
|
|
1368
|
+
entity: Entity name
|
|
1369
|
+
|
|
1370
|
+
Raises:
|
|
1371
|
+
MissingParameterError: If required body fields are missing
|
|
1372
|
+
"""
|
|
1373
|
+
# Only validate for operations that typically have required body fields
|
|
1374
|
+
if action not in (Action.CREATE, Action.UPDATE):
|
|
1375
|
+
return
|
|
1376
|
+
|
|
1377
|
+
# Get the request schema to find truly required fields
|
|
1378
|
+
request_schema = endpoint.request_schema
|
|
1379
|
+
if not request_schema:
|
|
1380
|
+
return
|
|
1381
|
+
|
|
1382
|
+
# Only validate fields explicitly marked as required in the schema
|
|
1383
|
+
required_fields = request_schema.get("required", [])
|
|
1384
|
+
missing_fields = [field for field in required_fields if field not in params]
|
|
1385
|
+
|
|
1386
|
+
if missing_fields:
|
|
1387
|
+
raise MissingParameterError(
|
|
1388
|
+
f"Missing required body fields for {entity}.{action.value}: {missing_fields}. Provided parameters: {list(params.keys())}"
|
|
1389
|
+
)
|
|
1390
|
+
|
|
1391
|
+
async def close(self):
|
|
1392
|
+
"""Close async HTTP client and logger."""
|
|
1393
|
+
self.tracker.track_session_end()
|
|
1394
|
+
await self.http_client.close()
|
|
1395
|
+
self.logger.close()
|
|
1396
|
+
|
|
1397
|
+
async def __aenter__(self):
|
|
1398
|
+
"""Async context manager entry."""
|
|
1399
|
+
return self
|
|
1400
|
+
|
|
1401
|
+
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
|
1402
|
+
"""Async context manager exit."""
|
|
1403
|
+
await self.close()
|
|
1404
|
+
|
|
1405
|
+
|
|
1406
|
+
# =============================================================================
|
|
1407
|
+
# Operation Handlers
|
|
1408
|
+
# =============================================================================
|
|
1409
|
+
|
|
1410
|
+
|
|
1411
|
+
class _StandardOperationHandler:
|
|
1412
|
+
"""Handler for standard REST operations (GET, LIST, CREATE, UPDATE, DELETE, API_SEARCH, AUTHORIZE)."""
|
|
1413
|
+
|
|
1414
|
+
def __init__(self, context: _OperationContext):
|
|
1415
|
+
self.ctx = context
|
|
1416
|
+
|
|
1417
|
+
def can_handle(self, action: Action) -> bool:
|
|
1418
|
+
"""Check if this handler can handle the given action."""
|
|
1419
|
+
return action in {
|
|
1420
|
+
Action.GET,
|
|
1421
|
+
Action.LIST,
|
|
1422
|
+
Action.CREATE,
|
|
1423
|
+
Action.UPDATE,
|
|
1424
|
+
Action.DELETE,
|
|
1425
|
+
Action.API_SEARCH,
|
|
1426
|
+
Action.AUTHORIZE,
|
|
1427
|
+
}
|
|
1428
|
+
|
|
1429
|
+
async def execute_operation(self, entity: str, action: Action, params: dict[str, Any]) -> StandardExecuteResult:
|
|
1430
|
+
"""Execute standard REST operation with full telemetry and error handling."""
|
|
1431
|
+
tracer = trace.get_tracer("airbyte.connector-sdk.executor.local")
|
|
1432
|
+
|
|
1433
|
+
with tracer.start_as_current_span("airbyte.local_executor.execute_operation") as span:
|
|
1434
|
+
# Add span attributes
|
|
1435
|
+
span.set_attribute("connector.name", self.ctx.executor.model.name)
|
|
1436
|
+
span.set_attribute("connector.entity", entity)
|
|
1437
|
+
span.set_attribute("connector.action", action.value)
|
|
1438
|
+
if params:
|
|
1439
|
+
span.set_attribute("connector.param_keys", list(params.keys()))
|
|
1440
|
+
|
|
1441
|
+
# Increment operation counter
|
|
1442
|
+
self.ctx.session.increment_operations()
|
|
1443
|
+
|
|
1444
|
+
# Track operation timing and status
|
|
1445
|
+
start_time = time.time()
|
|
1446
|
+
error_type = None
|
|
1447
|
+
status_code = None
|
|
1448
|
+
|
|
1449
|
+
try:
|
|
1450
|
+
# O(1) entity lookup
|
|
1451
|
+
entity_def = self.ctx.entity_index.get(entity)
|
|
1452
|
+
if not entity_def:
|
|
1453
|
+
available_entities = list(self.ctx.entity_index.keys())
|
|
1454
|
+
raise EntityNotFoundError(f"Entity '{entity}' not found in connector. Available entities: {available_entities}")
|
|
1455
|
+
|
|
1456
|
+
# Check if action is supported
|
|
1457
|
+
if action not in entity_def.actions:
|
|
1458
|
+
supported_actions = [a.value for a in entity_def.actions]
|
|
1459
|
+
raise ActionNotSupportedError(
|
|
1460
|
+
f"Action '{action.value}' not supported for entity '{entity}'. Supported actions: {supported_actions}"
|
|
1461
|
+
)
|
|
1462
|
+
|
|
1463
|
+
# O(1) operation lookup
|
|
1464
|
+
endpoint = self.ctx.operation_index.get((entity, action))
|
|
1465
|
+
if not endpoint:
|
|
1466
|
+
raise ExecutorError(f"No endpoint defined for {entity}.{action.value}. This is a configuration error.")
|
|
1467
|
+
|
|
1468
|
+
# Validate required body fields for CREATE/UPDATE operations
|
|
1469
|
+
self.ctx.validate_required_body_fields(endpoint, params, action, entity)
|
|
1470
|
+
|
|
1471
|
+
# Build request parameters
|
|
1472
|
+
# Use path_override if available, otherwise use the OpenAPI path
|
|
1473
|
+
actual_path = endpoint.path_override.path if endpoint.path_override else endpoint.path
|
|
1474
|
+
path = self.ctx.build_path(actual_path, params)
|
|
1475
|
+
query_params = self.ctx.extract_query_params(endpoint.query_params, params)
|
|
1476
|
+
|
|
1477
|
+
# Serialize deepObject parameters to bracket notation
|
|
1478
|
+
if endpoint.deep_object_params:
|
|
1479
|
+
query_params = self.ctx.executor._serialize_deep_object_params(query_params, endpoint.deep_object_params)
|
|
1480
|
+
|
|
1481
|
+
# Build request body (GraphQL or standard)
|
|
1482
|
+
body = self.ctx.build_request_body(endpoint, params)
|
|
1483
|
+
|
|
1484
|
+
# Determine request format (json/data parameters)
|
|
1485
|
+
request_kwargs = self.ctx.determine_request_format(endpoint, body)
|
|
1486
|
+
|
|
1487
|
+
# Execute async HTTP request
|
|
1488
|
+
response_data, response_headers = await self.ctx.http_client.request(
|
|
1489
|
+
method=endpoint.method,
|
|
1490
|
+
path=path,
|
|
1491
|
+
params=query_params if query_params else None,
|
|
1492
|
+
json=request_kwargs.get("json"),
|
|
1493
|
+
data=request_kwargs.get("data"),
|
|
1494
|
+
)
|
|
1495
|
+
|
|
1496
|
+
# Extract metadata from original response (before record extraction)
|
|
1497
|
+
metadata = self.ctx.executor._extract_metadata(response_data, response_headers, endpoint)
|
|
1498
|
+
|
|
1499
|
+
# Extract records if extractor configured
|
|
1500
|
+
response = self.ctx.extract_records(response_data, endpoint)
|
|
1501
|
+
|
|
1502
|
+
# Assume success with 200 status code if no exception raised
|
|
1503
|
+
status_code = 200
|
|
1504
|
+
|
|
1505
|
+
# Mark span as successful
|
|
1506
|
+
span.set_attribute("connector.success", True)
|
|
1507
|
+
span.set_attribute("http.status_code", status_code)
|
|
1508
|
+
|
|
1509
|
+
# Return StandardExecuteResult with data and metadata
|
|
1510
|
+
return StandardExecuteResult(data=response, metadata=metadata)
|
|
1511
|
+
|
|
1512
|
+
except (EntityNotFoundError, ActionNotSupportedError) as e:
|
|
1513
|
+
# Validation errors - record in span
|
|
1514
|
+
error_type = type(e).__name__
|
|
1515
|
+
span.set_attribute("connector.success", False)
|
|
1516
|
+
span.set_attribute("connector.error_type", error_type)
|
|
1517
|
+
span.record_exception(e)
|
|
1518
|
+
raise
|
|
1519
|
+
|
|
1520
|
+
except Exception as e:
|
|
1521
|
+
# Capture error details
|
|
1522
|
+
error_type = type(e).__name__
|
|
1523
|
+
|
|
1524
|
+
# Try to get status code from HTTP errors
|
|
1525
|
+
if hasattr(e, "response") and hasattr(e.response, "status_code"):
|
|
1526
|
+
status_code = e.response.status_code
|
|
1527
|
+
span.set_attribute("http.status_code", status_code)
|
|
1528
|
+
|
|
1529
|
+
span.set_attribute("connector.success", False)
|
|
1530
|
+
span.set_attribute("connector.error_type", error_type)
|
|
1531
|
+
span.record_exception(e)
|
|
1532
|
+
raise
|
|
1533
|
+
|
|
1534
|
+
finally:
|
|
1535
|
+
# Always track operation (success or failure)
|
|
1536
|
+
timing_ms = (time.time() - start_time) * 1000
|
|
1537
|
+
self.ctx.tracker.track_operation(
|
|
1538
|
+
entity=entity,
|
|
1539
|
+
action=action.value if isinstance(action, Action) else action,
|
|
1540
|
+
status_code=status_code,
|
|
1541
|
+
timing_ms=timing_ms,
|
|
1542
|
+
error_type=error_type,
|
|
1543
|
+
)
|
|
1544
|
+
|
|
1545
|
+
|
|
1546
|
+
class _DownloadOperationHandler:
|
|
1547
|
+
"""Handler for download operations.
|
|
1548
|
+
|
|
1549
|
+
Supports two modes:
|
|
1550
|
+
- Two-step (with x-airbyte-file-url): metadata request → extract URL → stream file
|
|
1551
|
+
- One-step (without x-airbyte-file-url): stream file directly from endpoint
|
|
1552
|
+
"""
|
|
1553
|
+
|
|
1554
|
+
def __init__(self, context: _OperationContext):
|
|
1555
|
+
self.ctx = context
|
|
1556
|
+
|
|
1557
|
+
def can_handle(self, action: Action) -> bool:
|
|
1558
|
+
"""Check if this handler can handle the given action."""
|
|
1559
|
+
return action == Action.DOWNLOAD
|
|
1560
|
+
|
|
1561
|
+
async def execute_operation(self, entity: str, action: Action, params: dict[str, Any]) -> AsyncIterator[bytes]:
|
|
1562
|
+
"""Execute download operation (one-step or two-step) with full telemetry."""
|
|
1563
|
+
tracer = trace.get_tracer("airbyte.connector-sdk.executor.local")
|
|
1564
|
+
|
|
1565
|
+
with tracer.start_as_current_span("airbyte.local_executor.execute_operation") as span:
|
|
1566
|
+
# Add span attributes
|
|
1567
|
+
span.set_attribute("connector.name", self.ctx.executor.model.name)
|
|
1568
|
+
span.set_attribute("connector.entity", entity)
|
|
1569
|
+
span.set_attribute("connector.action", action.value)
|
|
1570
|
+
if params:
|
|
1571
|
+
span.set_attribute("connector.param_keys", list(params.keys()))
|
|
1572
|
+
|
|
1573
|
+
# Increment operation counter
|
|
1574
|
+
self.ctx.session.increment_operations()
|
|
1575
|
+
|
|
1576
|
+
# Track operation timing and status
|
|
1577
|
+
start_time = time.time()
|
|
1578
|
+
error_type = None
|
|
1579
|
+
status_code = None
|
|
1580
|
+
|
|
1581
|
+
try:
|
|
1582
|
+
# Look up entity
|
|
1583
|
+
entity_def = self.ctx.entity_index.get(entity)
|
|
1584
|
+
if not entity_def:
|
|
1585
|
+
raise EntityNotFoundError(f"Entity '{entity}' not found in connector. Available entities: {list(self.ctx.entity_index.keys())}")
|
|
1586
|
+
|
|
1587
|
+
# Look up operation
|
|
1588
|
+
operation = self.ctx.operation_index.get((entity, action))
|
|
1589
|
+
if not operation:
|
|
1590
|
+
raise ActionNotSupportedError(
|
|
1591
|
+
f"Action '{action.value}' not supported for entity '{entity}'. Supported actions: {[a.value for a in entity_def.actions]}"
|
|
1592
|
+
)
|
|
1593
|
+
|
|
1594
|
+
# Common setup for both download modes
|
|
1595
|
+
actual_path = operation.path_override.path if operation.path_override else operation.path
|
|
1596
|
+
path = self.ctx.build_path(actual_path, params)
|
|
1597
|
+
query_params = self.ctx.extract_query_params(operation.query_params, params)
|
|
1598
|
+
|
|
1599
|
+
# Serialize deepObject parameters to bracket notation
|
|
1600
|
+
if operation.deep_object_params:
|
|
1601
|
+
query_params = self.ctx.executor._serialize_deep_object_params(query_params, operation.deep_object_params)
|
|
1602
|
+
|
|
1603
|
+
# Prepare headers (with optional Range support)
|
|
1604
|
+
range_header = params.get("range_header")
|
|
1605
|
+
headers = {"Accept": "*/*"}
|
|
1606
|
+
if range_header is not None:
|
|
1607
|
+
headers["Range"] = range_header
|
|
1608
|
+
|
|
1609
|
+
# Check download mode: two-step (with file_field) or one-step (without)
|
|
1610
|
+
file_field = operation.file_field
|
|
1611
|
+
|
|
1612
|
+
if file_field:
|
|
1613
|
+
# Substitute template variables in file_field (e.g., "attachments[{index}].url")
|
|
1614
|
+
file_field = LocalExecutor._substitute_file_field_params(file_field, params)
|
|
1615
|
+
|
|
1616
|
+
if file_field:
|
|
1617
|
+
# Two-step download: metadata → extract URL → stream file
|
|
1618
|
+
# Step 1: Get metadata (standard request)
|
|
1619
|
+
request_body = self.ctx.build_request_body(
|
|
1620
|
+
endpoint=operation,
|
|
1621
|
+
params=params,
|
|
1622
|
+
)
|
|
1623
|
+
request_format = self.ctx.determine_request_format(operation, request_body)
|
|
1624
|
+
self.ctx.validate_required_body_fields(operation, params, action, entity)
|
|
1625
|
+
|
|
1626
|
+
metadata_response, _ = await self.ctx.http_client.request(
|
|
1627
|
+
method=operation.method,
|
|
1628
|
+
path=path,
|
|
1629
|
+
params=query_params,
|
|
1630
|
+
**request_format,
|
|
1631
|
+
)
|
|
1632
|
+
|
|
1633
|
+
# Step 2: Extract file URL from metadata
|
|
1634
|
+
file_url = LocalExecutor._extract_download_url(
|
|
1635
|
+
response=metadata_response,
|
|
1636
|
+
file_field=file_field,
|
|
1637
|
+
entity=entity,
|
|
1638
|
+
)
|
|
1639
|
+
|
|
1640
|
+
# Step 3: Stream file from extracted URL
|
|
1641
|
+
file_response, _ = await self.ctx.http_client.request(
|
|
1642
|
+
method="GET",
|
|
1643
|
+
path=file_url,
|
|
1644
|
+
headers=headers,
|
|
1645
|
+
stream=True,
|
|
1646
|
+
)
|
|
1647
|
+
else:
|
|
1648
|
+
# One-step direct download: stream file directly from endpoint
|
|
1649
|
+
file_response, _ = await self.ctx.http_client.request(
|
|
1650
|
+
method=operation.method,
|
|
1651
|
+
path=path,
|
|
1652
|
+
params=query_params,
|
|
1653
|
+
headers=headers,
|
|
1654
|
+
stream=True,
|
|
1655
|
+
)
|
|
1656
|
+
|
|
1657
|
+
# Assume success once we start streaming
|
|
1658
|
+
status_code = 200
|
|
1659
|
+
|
|
1660
|
+
# Mark span as successful
|
|
1661
|
+
span.set_attribute("connector.success", True)
|
|
1662
|
+
span.set_attribute("http.status_code", status_code)
|
|
1663
|
+
|
|
1664
|
+
# Stream file chunks
|
|
1665
|
+
default_chunk_size = 8 * 1024 * 1024 # 8 MB
|
|
1666
|
+
async for chunk in file_response.original_response.aiter_bytes(chunk_size=default_chunk_size):
|
|
1667
|
+
# Log each chunk for cassette recording
|
|
1668
|
+
self.ctx.logger.log_chunk_fetch(chunk)
|
|
1669
|
+
yield chunk
|
|
1670
|
+
|
|
1671
|
+
except (EntityNotFoundError, ActionNotSupportedError) as e:
|
|
1672
|
+
# Validation errors - record in span
|
|
1673
|
+
error_type = type(e).__name__
|
|
1674
|
+
span.set_attribute("connector.success", False)
|
|
1675
|
+
span.set_attribute("connector.error_type", error_type)
|
|
1676
|
+
span.record_exception(e)
|
|
1677
|
+
|
|
1678
|
+
# Track the failed operation before re-raising
|
|
1679
|
+
timing_ms = (time.time() - start_time) * 1000
|
|
1680
|
+
self.ctx.tracker.track_operation(
|
|
1681
|
+
entity=entity,
|
|
1682
|
+
action=action.value,
|
|
1683
|
+
status_code=status_code,
|
|
1684
|
+
timing_ms=timing_ms,
|
|
1685
|
+
error_type=error_type,
|
|
1686
|
+
)
|
|
1687
|
+
raise
|
|
1688
|
+
|
|
1689
|
+
except Exception as e:
|
|
1690
|
+
# Capture error details
|
|
1691
|
+
error_type = type(e).__name__
|
|
1692
|
+
|
|
1693
|
+
# Try to get status code from HTTP errors
|
|
1694
|
+
if hasattr(e, "response") and hasattr(e.response, "status_code"):
|
|
1695
|
+
status_code = e.response.status_code
|
|
1696
|
+
span.set_attribute("http.status_code", status_code)
|
|
1697
|
+
|
|
1698
|
+
span.set_attribute("connector.success", False)
|
|
1699
|
+
span.set_attribute("connector.error_type", error_type)
|
|
1700
|
+
span.record_exception(e)
|
|
1701
|
+
|
|
1702
|
+
# Track the failed operation before re-raising
|
|
1703
|
+
timing_ms = (time.time() - start_time) * 1000
|
|
1704
|
+
self.ctx.tracker.track_operation(
|
|
1705
|
+
entity=entity,
|
|
1706
|
+
action=action.value,
|
|
1707
|
+
status_code=status_code,
|
|
1708
|
+
timing_ms=timing_ms,
|
|
1709
|
+
error_type=error_type,
|
|
1710
|
+
)
|
|
1711
|
+
raise
|
|
1712
|
+
|
|
1713
|
+
finally:
|
|
1714
|
+
# Track successful operation (if no exception was raised)
|
|
1715
|
+
# Note: For generators, this runs after all chunks are yielded
|
|
1716
|
+
if error_type is None:
|
|
1717
|
+
timing_ms = (time.time() - start_time) * 1000
|
|
1718
|
+
self.ctx.tracker.track_operation(
|
|
1719
|
+
entity=entity,
|
|
1720
|
+
action=action.value,
|
|
1721
|
+
status_code=status_code,
|
|
1722
|
+
timing_ms=timing_ms,
|
|
1723
|
+
error_type=None,
|
|
1724
|
+
)
|