airbyte-agent-mcp 0.1.33__py3-none-any.whl → 0.1.53__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_agent_mcp/_vendored/connector_sdk/connector_model_loader.py +10 -2
- airbyte_agent_mcp/_vendored/connector_sdk/constants.py +1 -1
- airbyte_agent_mcp/_vendored/connector_sdk/executor/local_executor.py +93 -23
- airbyte_agent_mcp/_vendored/connector_sdk/extensions.py +42 -3
- airbyte_agent_mcp/_vendored/connector_sdk/http_client.py +50 -43
- airbyte_agent_mcp/_vendored/connector_sdk/introspection.py +262 -0
- airbyte_agent_mcp/_vendored/connector_sdk/observability/config.py +179 -0
- airbyte_agent_mcp/_vendored/connector_sdk/observability/session.py +35 -28
- airbyte_agent_mcp/_vendored/connector_sdk/schema/components.py +2 -1
- airbyte_agent_mcp/_vendored/connector_sdk/schema/operations.py +1 -1
- airbyte_agent_mcp/_vendored/connector_sdk/schema/security.py +10 -0
- airbyte_agent_mcp/_vendored/connector_sdk/telemetry/events.py +2 -1
- airbyte_agent_mcp/_vendored/connector_sdk/telemetry/tracker.py +3 -0
- airbyte_agent_mcp/_vendored/connector_sdk/types.py +5 -1
- {airbyte_agent_mcp-0.1.33.dist-info → airbyte_agent_mcp-0.1.53.dist-info}/METADATA +1 -1
- {airbyte_agent_mcp-0.1.33.dist-info → airbyte_agent_mcp-0.1.53.dist-info}/RECORD +17 -15
- {airbyte_agent_mcp-0.1.33.dist-info → airbyte_agent_mcp-0.1.53.dist-info}/WHEEL +0 -0
|
@@ -393,16 +393,24 @@ def convert_openapi_to_connector_model(spec: OpenAPIConnector) -> ConnectorModel
|
|
|
393
393
|
for entity_name, endpoints_dict in entities_map.items():
|
|
394
394
|
actions = list(endpoints_dict.keys())
|
|
395
395
|
|
|
396
|
-
# Get schema from components if available
|
|
396
|
+
# Get schema and stream_name from components if available
|
|
397
397
|
schema = None
|
|
398
|
+
entity_stream_name = None
|
|
398
399
|
if spec.components:
|
|
399
400
|
# Look for a schema matching the entity name
|
|
400
401
|
for schema_name, schema_def in spec.components.schemas.items():
|
|
401
402
|
if schema_def.x_airbyte_entity_name == entity_name or schema_name.lower() == entity_name.lower():
|
|
402
403
|
schema = schema_def.model_dump(by_alias=True)
|
|
404
|
+
entity_stream_name = schema_def.x_airbyte_stream_name
|
|
403
405
|
break
|
|
404
406
|
|
|
405
|
-
entity = EntityDefinition(
|
|
407
|
+
entity = EntityDefinition(
|
|
408
|
+
name=entity_name,
|
|
409
|
+
stream_name=entity_stream_name,
|
|
410
|
+
actions=actions,
|
|
411
|
+
endpoints=endpoints_dict,
|
|
412
|
+
schema=schema,
|
|
413
|
+
)
|
|
406
414
|
entities.append(entity)
|
|
407
415
|
|
|
408
416
|
# Extract retry config from x-airbyte-retry-config extension
|
|
@@ -674,16 +674,16 @@ class LocalExecutor:
|
|
|
674
674
|
return {key: value for key, value in params.items() if key in allowed_params}
|
|
675
675
|
|
|
676
676
|
def _extract_body(self, allowed_fields: list[str], params: dict[str, Any]) -> dict[str, Any]:
|
|
677
|
-
"""Extract body fields from params.
|
|
677
|
+
"""Extract body fields from params, filtering out None values.
|
|
678
678
|
|
|
679
679
|
Args:
|
|
680
680
|
allowed_fields: List of allowed body field names
|
|
681
681
|
params: All parameters
|
|
682
682
|
|
|
683
683
|
Returns:
|
|
684
|
-
Dictionary of body fields
|
|
684
|
+
Dictionary of body fields with None values filtered out
|
|
685
685
|
"""
|
|
686
|
-
return {key: value for key, value in params.items() if key in allowed_fields}
|
|
686
|
+
return {key: value for key, value in params.items() if key in allowed_fields and value is not None}
|
|
687
687
|
|
|
688
688
|
def _serialize_deep_object_params(self, params: dict[str, Any], deep_object_param_names: list[str]) -> dict[str, Any]:
|
|
689
689
|
"""Serialize deepObject parameters to bracket notation format.
|
|
@@ -837,15 +837,65 @@ class LocalExecutor:
|
|
|
837
837
|
Request body dict or None if no body needed
|
|
838
838
|
"""
|
|
839
839
|
if endpoint.graphql_body:
|
|
840
|
-
|
|
840
|
+
# Extract defaults from query_params_schema for GraphQL variable interpolation
|
|
841
|
+
param_defaults = {name: schema.get("default") for name, schema in endpoint.query_params_schema.items() if "default" in schema}
|
|
842
|
+
return self._build_graphql_body(endpoint.graphql_body, params, param_defaults)
|
|
841
843
|
elif endpoint.body_fields:
|
|
842
844
|
return self._extract_body(endpoint.body_fields, params)
|
|
843
845
|
return None
|
|
844
846
|
|
|
847
|
+
def _flatten_form_data(self, data: dict[str, Any], parent_key: str = "") -> dict[str, Any]:
|
|
848
|
+
"""Flatten nested dict/list structures into bracket notation for form encoding.
|
|
849
|
+
|
|
850
|
+
Stripe and similar APIs require nested arrays/objects to be encoded using bracket
|
|
851
|
+
notation when using application/x-www-form-urlencoded content type.
|
|
852
|
+
|
|
853
|
+
Args:
|
|
854
|
+
data: Nested dict with arrays/objects to flatten
|
|
855
|
+
parent_key: Parent key for nested structures (used in recursion)
|
|
856
|
+
|
|
857
|
+
Returns:
|
|
858
|
+
Flattened dict with bracket notation keys
|
|
859
|
+
|
|
860
|
+
Examples:
|
|
861
|
+
>>> _flatten_form_data({"items": [{"price": "p1", "qty": 1}]})
|
|
862
|
+
{"items[0][price]": "p1", "items[0][qty]": 1}
|
|
863
|
+
|
|
864
|
+
>>> _flatten_form_data({"customer": "cus_123", "metadata": {"key": "value"}})
|
|
865
|
+
{"customer": "cus_123", "metadata[key]": "value"}
|
|
866
|
+
"""
|
|
867
|
+
flattened = {}
|
|
868
|
+
|
|
869
|
+
for key, value in data.items():
|
|
870
|
+
new_key = f"{parent_key}[{key}]" if parent_key else key
|
|
871
|
+
|
|
872
|
+
if isinstance(value, dict):
|
|
873
|
+
# Recursively flatten nested dicts
|
|
874
|
+
flattened.update(self._flatten_form_data(value, new_key))
|
|
875
|
+
elif isinstance(value, list):
|
|
876
|
+
# Flatten arrays with indexed bracket notation
|
|
877
|
+
for i, item in enumerate(value):
|
|
878
|
+
indexed_key = f"{new_key}[{i}]"
|
|
879
|
+
if isinstance(item, dict):
|
|
880
|
+
# Nested dict in array - recurse
|
|
881
|
+
flattened.update(self._flatten_form_data(item, indexed_key))
|
|
882
|
+
elif isinstance(item, list):
|
|
883
|
+
# Nested list in array - recurse
|
|
884
|
+
flattened.update(self._flatten_form_data({str(i): item}, new_key))
|
|
885
|
+
else:
|
|
886
|
+
# Primitive value in array
|
|
887
|
+
flattened[indexed_key] = item
|
|
888
|
+
else:
|
|
889
|
+
# Primitive value - add directly
|
|
890
|
+
flattened[new_key] = value
|
|
891
|
+
|
|
892
|
+
return flattened
|
|
893
|
+
|
|
845
894
|
def _determine_request_format(self, endpoint: EndpointDefinition, body: dict[str, Any] | None) -> dict[str, Any]:
|
|
846
895
|
"""Determine json/data parameters for HTTP request.
|
|
847
896
|
|
|
848
897
|
GraphQL always uses JSON, regardless of content_type setting.
|
|
898
|
+
For form-encoded requests, nested structures are flattened into bracket notation.
|
|
849
899
|
|
|
850
900
|
Args:
|
|
851
901
|
endpoint: Endpoint definition
|
|
@@ -862,7 +912,9 @@ class LocalExecutor:
|
|
|
862
912
|
if is_graphql or endpoint.content_type.value == "application/json":
|
|
863
913
|
return {"json": body}
|
|
864
914
|
elif endpoint.content_type.value == "application/x-www-form-urlencoded":
|
|
865
|
-
|
|
915
|
+
# Flatten nested structures for form encoding
|
|
916
|
+
flattened_body = self._flatten_form_data(body)
|
|
917
|
+
return {"data": flattened_body}
|
|
866
918
|
|
|
867
919
|
return {}
|
|
868
920
|
|
|
@@ -903,12 +955,18 @@ class LocalExecutor:
|
|
|
903
955
|
|
|
904
956
|
return query
|
|
905
957
|
|
|
906
|
-
def _build_graphql_body(
|
|
958
|
+
def _build_graphql_body(
|
|
959
|
+
self,
|
|
960
|
+
graphql_config: dict[str, Any],
|
|
961
|
+
params: dict[str, Any],
|
|
962
|
+
param_defaults: dict[str, Any] | None = None,
|
|
963
|
+
) -> dict[str, Any]:
|
|
907
964
|
"""Build GraphQL request body with variable substitution and field selection.
|
|
908
965
|
|
|
909
966
|
Args:
|
|
910
967
|
graphql_config: GraphQL configuration from x-airbyte-body-type extension
|
|
911
968
|
params: Parameters from execute() call
|
|
969
|
+
param_defaults: Default values for params from query_params_schema
|
|
912
970
|
|
|
913
971
|
Returns:
|
|
914
972
|
GraphQL request body: {"query": "...", "variables": {...}}
|
|
@@ -922,7 +980,7 @@ class LocalExecutor:
|
|
|
922
980
|
|
|
923
981
|
# Substitute variables from params
|
|
924
982
|
if "variables" in graphql_config and graphql_config["variables"]:
|
|
925
|
-
body["variables"] = self._interpolate_variables(graphql_config["variables"], params)
|
|
983
|
+
body["variables"] = self._interpolate_variables(graphql_config["variables"], params, param_defaults)
|
|
926
984
|
|
|
927
985
|
# Add operation name if specified
|
|
928
986
|
if "operationName" in graphql_config:
|
|
@@ -981,7 +1039,12 @@ class LocalExecutor:
|
|
|
981
1039
|
fields_str = " ".join(graphql_fields)
|
|
982
1040
|
return query.replace("{{ fields }}", fields_str)
|
|
983
1041
|
|
|
984
|
-
def _interpolate_variables(
|
|
1042
|
+
def _interpolate_variables(
|
|
1043
|
+
self,
|
|
1044
|
+
variables: dict[str, Any],
|
|
1045
|
+
params: dict[str, Any],
|
|
1046
|
+
param_defaults: dict[str, Any] | None = None,
|
|
1047
|
+
) -> dict[str, Any]:
|
|
985
1048
|
"""Recursively interpolate variables using params.
|
|
986
1049
|
|
|
987
1050
|
Preserves types (doesn't stringify everything).
|
|
@@ -990,15 +1053,18 @@ class LocalExecutor:
|
|
|
990
1053
|
- Direct replacement: "{{ owner }}" → params["owner"] (preserves type)
|
|
991
1054
|
- Nested objects: {"input": {"name": "{{ name }}"}}
|
|
992
1055
|
- Arrays: [{"id": "{{ id }}"}]
|
|
993
|
-
-
|
|
1056
|
+
- Default values: "{{ per_page }}" → param_defaults["per_page"] if not in params
|
|
1057
|
+
- Unsubstituted placeholders: "{{ states }}" → None (for optional params without defaults)
|
|
994
1058
|
|
|
995
1059
|
Args:
|
|
996
1060
|
variables: Variables dict with template placeholders
|
|
997
1061
|
params: Parameters to substitute
|
|
1062
|
+
param_defaults: Default values for params from query_params_schema
|
|
998
1063
|
|
|
999
1064
|
Returns:
|
|
1000
1065
|
Interpolated variables dict with types preserved
|
|
1001
1066
|
"""
|
|
1067
|
+
defaults = param_defaults or {}
|
|
1002
1068
|
|
|
1003
1069
|
def interpolate_value(value: Any) -> Any:
|
|
1004
1070
|
if isinstance(value, str):
|
|
@@ -1012,8 +1078,15 @@ class LocalExecutor:
|
|
|
1012
1078
|
value = value.replace(placeholder, str(param_value))
|
|
1013
1079
|
|
|
1014
1080
|
# Check if any unsubstituted placeholders remain
|
|
1015
|
-
# If so, return None (treats as "not provided" for optional params)
|
|
1016
1081
|
if re.search(r"\{\{\s*\w+\s*\}\}", value):
|
|
1082
|
+
# Extract placeholder name and check for default value
|
|
1083
|
+
match = re.search(r"\{\{\s*(\w+)\s*\}\}", value)
|
|
1084
|
+
if match:
|
|
1085
|
+
param_name = match.group(1)
|
|
1086
|
+
if param_name in defaults:
|
|
1087
|
+
# Use default value (preserves type)
|
|
1088
|
+
return defaults[param_name]
|
|
1089
|
+
# No default found - return None (for optional params)
|
|
1017
1090
|
return None
|
|
1018
1091
|
|
|
1019
1092
|
return value
|
|
@@ -1056,7 +1129,7 @@ class LocalExecutor:
|
|
|
1056
1129
|
if not action:
|
|
1057
1130
|
return response_data
|
|
1058
1131
|
|
|
1059
|
-
is_array_action = action in (Action.LIST, Action.
|
|
1132
|
+
is_array_action = action in (Action.LIST, Action.API_SEARCH)
|
|
1060
1133
|
|
|
1061
1134
|
try:
|
|
1062
1135
|
# Parse and apply JSONPath expression
|
|
@@ -1151,21 +1224,18 @@ class LocalExecutor:
|
|
|
1151
1224
|
if action not in (Action.CREATE, Action.UPDATE):
|
|
1152
1225
|
return
|
|
1153
1226
|
|
|
1154
|
-
#
|
|
1155
|
-
|
|
1227
|
+
# Get the request schema to find truly required fields
|
|
1228
|
+
request_schema = endpoint.request_schema
|
|
1229
|
+
if not request_schema:
|
|
1156
1230
|
return
|
|
1157
1231
|
|
|
1158
|
-
#
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
missing_fields = []
|
|
1162
|
-
for field in endpoint.body_fields:
|
|
1163
|
-
if field not in params:
|
|
1164
|
-
missing_fields.append(field)
|
|
1232
|
+
# Only validate fields explicitly marked as required in the schema
|
|
1233
|
+
required_fields = request_schema.get("required", [])
|
|
1234
|
+
missing_fields = [field for field in required_fields if field not in params]
|
|
1165
1235
|
|
|
1166
1236
|
if missing_fields:
|
|
1167
1237
|
raise MissingParameterError(
|
|
1168
|
-
f"Missing required body fields for {entity}.{action.value}: {missing_fields}. Provided parameters: {list(params.keys())}"
|
|
1238
|
+
f"Missing required body fields for {entity}.{action.value}: {missing_fields}. " f"Provided parameters: {list(params.keys())}"
|
|
1169
1239
|
)
|
|
1170
1240
|
|
|
1171
1241
|
async def close(self):
|
|
@@ -1189,7 +1259,7 @@ class LocalExecutor:
|
|
|
1189
1259
|
|
|
1190
1260
|
|
|
1191
1261
|
class _StandardOperationHandler:
|
|
1192
|
-
"""Handler for standard REST operations (GET, LIST, CREATE, UPDATE, DELETE,
|
|
1262
|
+
"""Handler for standard REST operations (GET, LIST, CREATE, UPDATE, DELETE, API_SEARCH, AUTHORIZE)."""
|
|
1193
1263
|
|
|
1194
1264
|
def __init__(self, context: _OperationContext):
|
|
1195
1265
|
self.ctx = context
|
|
@@ -1202,7 +1272,7 @@ class _StandardOperationHandler:
|
|
|
1202
1272
|
Action.CREATE,
|
|
1203
1273
|
Action.UPDATE,
|
|
1204
1274
|
Action.DELETE,
|
|
1205
|
-
Action.
|
|
1275
|
+
Action.API_SEARCH,
|
|
1206
1276
|
Action.AUTHORIZE,
|
|
1207
1277
|
}
|
|
1208
1278
|
|
|
@@ -159,6 +159,38 @@ Example:
|
|
|
159
159
|
```
|
|
160
160
|
"""
|
|
161
161
|
|
|
162
|
+
AIRBYTE_STREAM_NAME = "x-airbyte-stream-name"
|
|
163
|
+
"""
|
|
164
|
+
Extension: x-airbyte-stream-name
|
|
165
|
+
Location: Schema object (in components.schemas)
|
|
166
|
+
Type: string
|
|
167
|
+
Required: No
|
|
168
|
+
|
|
169
|
+
Description:
|
|
170
|
+
Specifies the Airbyte stream name for cache lookup purposes. This maps the entity
|
|
171
|
+
to the corresponding Airbyte stream, enabling cache-based data retrieval. When
|
|
172
|
+
specified, the EntityDefinition.stream_name field will be populated with this value.
|
|
173
|
+
|
|
174
|
+
This extension is placed on Schema objects alongside x-airbyte-entity-name, following
|
|
175
|
+
the same pattern. The stream name is an entity-level property (not operation-level)
|
|
176
|
+
since an entity maps to exactly one Airbyte stream.
|
|
177
|
+
|
|
178
|
+
Example:
|
|
179
|
+
```yaml
|
|
180
|
+
components:
|
|
181
|
+
schemas:
|
|
182
|
+
Customer:
|
|
183
|
+
type: object
|
|
184
|
+
x-airbyte-entity-name: customers
|
|
185
|
+
x-airbyte-stream-name: customers
|
|
186
|
+
properties:
|
|
187
|
+
id:
|
|
188
|
+
type: string
|
|
189
|
+
name:
|
|
190
|
+
type: string
|
|
191
|
+
```
|
|
192
|
+
"""
|
|
193
|
+
|
|
162
194
|
AIRBYTE_TOKEN_PATH = "x-airbyte-token-path"
|
|
163
195
|
"""
|
|
164
196
|
Extension: x-airbyte-token-path
|
|
@@ -495,8 +527,8 @@ class ActionType(str, Enum):
|
|
|
495
527
|
DELETE = "delete"
|
|
496
528
|
"""Delete a record"""
|
|
497
529
|
|
|
498
|
-
|
|
499
|
-
"""Search for records matching specific query criteria"""
|
|
530
|
+
API_SEARCH = "api_search"
|
|
531
|
+
"""Search for records matching specific query criteria via API"""
|
|
500
532
|
|
|
501
533
|
DOWNLOAD = "download"
|
|
502
534
|
"""Download file content from a URL specified in the metadata response"""
|
|
@@ -514,7 +546,7 @@ class BodyType(str, Enum):
|
|
|
514
546
|
|
|
515
547
|
|
|
516
548
|
# Type alias for use in Pydantic models
|
|
517
|
-
ActionTypeLiteral = Literal["get", "list", "create", "update", "delete", "
|
|
549
|
+
ActionTypeLiteral = Literal["get", "list", "create", "update", "delete", "api_search", "download"]
|
|
518
550
|
|
|
519
551
|
|
|
520
552
|
# =============================================================================
|
|
@@ -548,6 +580,7 @@ def get_all_extension_names() -> list[str]:
|
|
|
548
580
|
AIRBYTE_ENTITY,
|
|
549
581
|
AIRBYTE_ACTION,
|
|
550
582
|
AIRBYTE_ENTITY_NAME,
|
|
583
|
+
AIRBYTE_STREAM_NAME,
|
|
551
584
|
AIRBYTE_TOKEN_PATH,
|
|
552
585
|
AIRBYTE_BODY_TYPE,
|
|
553
586
|
AIRBYTE_PATH_OVERRIDE,
|
|
@@ -594,6 +627,12 @@ EXTENSION_REGISTRY = {
|
|
|
594
627
|
"required": False,
|
|
595
628
|
"description": "Links schema to an entity/stream",
|
|
596
629
|
},
|
|
630
|
+
AIRBYTE_STREAM_NAME: {
|
|
631
|
+
"location": "schema",
|
|
632
|
+
"type": "string",
|
|
633
|
+
"required": False,
|
|
634
|
+
"description": "Maps entity to Airbyte stream for cache lookup",
|
|
635
|
+
},
|
|
597
636
|
AIRBYTE_TOKEN_PATH: {
|
|
598
637
|
"location": "securityScheme",
|
|
599
638
|
"type": "string",
|
|
@@ -147,6 +147,9 @@ class HTTPClient:
|
|
|
147
147
|
self.base_url = self.base_url.replace(f"{{{var_name}}}", var_value)
|
|
148
148
|
|
|
149
149
|
self.auth_config = auth_config
|
|
150
|
+
assert (
|
|
151
|
+
self.auth_config.type is not None
|
|
152
|
+
), "auth_config.type cannot be None" # Should never be None when instantiated via the local executor flow
|
|
150
153
|
self.secrets = secrets
|
|
151
154
|
self.logger = logger or NullLogger()
|
|
152
155
|
self.metrics = HTTPMetrics()
|
|
@@ -296,12 +299,12 @@ class HTTPClient:
|
|
|
296
299
|
|
|
297
300
|
# Support both sync and async callbacks
|
|
298
301
|
callback_result = self.on_token_refresh(callback_data)
|
|
299
|
-
if hasattr(callback_result, "__await__"):
|
|
302
|
+
if callback_result is not None and hasattr(callback_result, "__await__"):
|
|
300
303
|
await callback_result
|
|
301
304
|
except Exception as callback_error:
|
|
302
305
|
self.logger.log_error(
|
|
303
306
|
request_id=None,
|
|
304
|
-
error=("Token refresh callback failed during initialization:
|
|
307
|
+
error=(f"Token refresh callback failed during initialization: {callback_error!s}"),
|
|
305
308
|
status_code=None,
|
|
306
309
|
)
|
|
307
310
|
|
|
@@ -485,7 +488,7 @@ class HTTPClient:
|
|
|
485
488
|
elif "application/json" in content_type or not content_type:
|
|
486
489
|
response_data = await response.json()
|
|
487
490
|
else:
|
|
488
|
-
error_msg = f"Expected JSON response for {method.upper()} {url},
|
|
491
|
+
error_msg = f"Expected JSON response for {method.upper()} {url}, got content-type: {content_type}"
|
|
489
492
|
raise HTTPClientError(error_msg)
|
|
490
493
|
|
|
491
494
|
except ValueError as e:
|
|
@@ -556,6 +559,7 @@ class HTTPClient:
|
|
|
556
559
|
current_token = self.secrets.get("access_token")
|
|
557
560
|
strategy = AuthStrategyFactory.get_strategy(self.auth_config.type)
|
|
558
561
|
|
|
562
|
+
# Try to refresh credentials
|
|
559
563
|
try:
|
|
560
564
|
result = await strategy.handle_auth_error(
|
|
561
565
|
status_code=status_code,
|
|
@@ -564,53 +568,56 @@ class HTTPClient:
|
|
|
564
568
|
config_values=self.config_values,
|
|
565
569
|
http_client=None, # Let strategy create its own client
|
|
566
570
|
)
|
|
567
|
-
|
|
568
|
-
if result:
|
|
569
|
-
# Notify callback if provided (for persistence)
|
|
570
|
-
# Include both tokens AND extracted values for full persistence
|
|
571
|
-
if self.on_token_refresh is not None:
|
|
572
|
-
try:
|
|
573
|
-
# Build callback data with both tokens and extracted values
|
|
574
|
-
callback_data = dict(result.tokens)
|
|
575
|
-
if result.extracted_values:
|
|
576
|
-
callback_data.update(result.extracted_values)
|
|
577
|
-
|
|
578
|
-
# Support both sync and async callbacks
|
|
579
|
-
callback_result = self.on_token_refresh(callback_data)
|
|
580
|
-
if hasattr(callback_result, "__await__"):
|
|
581
|
-
await callback_result
|
|
582
|
-
except Exception as callback_error:
|
|
583
|
-
self.logger.log_error(
|
|
584
|
-
request_id=request_id,
|
|
585
|
-
error=f"Token refresh callback failed: {str(callback_error)}",
|
|
586
|
-
status_code=status_code,
|
|
587
|
-
)
|
|
588
|
-
|
|
589
|
-
# Update secrets with new tokens (in-memory)
|
|
590
|
-
self.secrets.update(result.tokens)
|
|
591
|
-
|
|
592
|
-
# Update config_values and re-render base_url with extracted values
|
|
593
|
-
if result.extracted_values:
|
|
594
|
-
self._apply_token_extract(result.extracted_values)
|
|
595
|
-
|
|
596
|
-
if self.secrets.get("access_token") != current_token:
|
|
597
|
-
# Retry with new token - this will go through full retry logic
|
|
598
|
-
return await self.request(
|
|
599
|
-
method=method,
|
|
600
|
-
path=path,
|
|
601
|
-
params=params,
|
|
602
|
-
json=json,
|
|
603
|
-
data=data,
|
|
604
|
-
headers=headers,
|
|
605
|
-
)
|
|
606
|
-
|
|
607
571
|
except Exception as refresh_error:
|
|
608
572
|
self.logger.log_error(
|
|
609
573
|
request_id=request_id,
|
|
610
574
|
error=f"Credential refresh failed: {str(refresh_error)}",
|
|
611
575
|
status_code=status_code,
|
|
612
576
|
)
|
|
577
|
+
result = None
|
|
578
|
+
|
|
579
|
+
# If refresh succeeded, update tokens and retry
|
|
580
|
+
if result:
|
|
581
|
+
# Notify callback if provided (for persistence)
|
|
582
|
+
# Include both tokens AND extracted values for full persistence
|
|
583
|
+
if self.on_token_refresh is not None:
|
|
584
|
+
try:
|
|
585
|
+
# Build callback data with both tokens and extracted values
|
|
586
|
+
callback_data = dict(result.tokens)
|
|
587
|
+
if result.extracted_values:
|
|
588
|
+
callback_data.update(result.extracted_values)
|
|
589
|
+
|
|
590
|
+
# Support both sync and async callbacks
|
|
591
|
+
callback_result = self.on_token_refresh(callback_data)
|
|
592
|
+
if callback_result is not None and hasattr(callback_result, "__await__"):
|
|
593
|
+
await callback_result
|
|
594
|
+
except Exception as callback_error:
|
|
595
|
+
self.logger.log_error(
|
|
596
|
+
request_id=request_id,
|
|
597
|
+
error=f"Token refresh callback failed: {str(callback_error)}",
|
|
598
|
+
status_code=status_code,
|
|
599
|
+
)
|
|
600
|
+
|
|
601
|
+
# Update secrets with new tokens (in-memory)
|
|
602
|
+
self.secrets.update(result.tokens)
|
|
603
|
+
|
|
604
|
+
# Update config_values and re-render base_url with extracted values
|
|
605
|
+
if result.extracted_values:
|
|
606
|
+
self._apply_token_extract(result.extracted_values)
|
|
613
607
|
|
|
608
|
+
if self.secrets.get("access_token") != current_token:
|
|
609
|
+
# Retry with new token - this will go through full retry logic
|
|
610
|
+
# Any errors from this retry will propagate to the caller
|
|
611
|
+
return await self.request(
|
|
612
|
+
method=method,
|
|
613
|
+
path=path,
|
|
614
|
+
params=params,
|
|
615
|
+
json=json,
|
|
616
|
+
data=data,
|
|
617
|
+
headers=headers,
|
|
618
|
+
)
|
|
619
|
+
|
|
620
|
+
# Refresh failed or token didn't change, log and let original error propagate
|
|
614
621
|
self.logger.log_error(request_id=request_id, error=str(error), status_code=status_code)
|
|
615
622
|
|
|
616
623
|
async def request(
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Shared introspection utilities for connector metadata.
|
|
3
|
+
|
|
4
|
+
This module provides utilities for introspecting connector metadata,
|
|
5
|
+
generating descriptions, and formatting parameter signatures. These
|
|
6
|
+
functions are used by both the runtime decorators and the generated
|
|
7
|
+
connector code.
|
|
8
|
+
|
|
9
|
+
The module is designed to work with any object conforming to the
|
|
10
|
+
ConnectorModel and EndpointDefinition interfaces from connector_sdk.types.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from typing import Any, Protocol
|
|
16
|
+
|
|
17
|
+
# Constants
|
|
18
|
+
MAX_EXAMPLE_QUESTIONS = 5 # Maximum number of example questions to include in description
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class EndpointProtocol(Protocol):
|
|
22
|
+
"""Protocol defining the expected interface for endpoint parameters.
|
|
23
|
+
|
|
24
|
+
This allows functions to work with any endpoint-like object
|
|
25
|
+
that has these attributes, including EndpointDefinition and mock objects.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
path_params: list[str]
|
|
29
|
+
path_params_schema: dict[str, dict[str, Any]]
|
|
30
|
+
query_params: list[str]
|
|
31
|
+
query_params_schema: dict[str, dict[str, Any]]
|
|
32
|
+
body_fields: list[str]
|
|
33
|
+
request_schema: dict[str, Any] | None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class EntityProtocol(Protocol):
|
|
37
|
+
"""Protocol defining the expected interface for entity definitions."""
|
|
38
|
+
|
|
39
|
+
name: str
|
|
40
|
+
actions: list[Any]
|
|
41
|
+
endpoints: dict[Any, EndpointProtocol]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class ConnectorModelProtocol(Protocol):
|
|
45
|
+
"""Protocol defining the expected interface for connector model parameters.
|
|
46
|
+
|
|
47
|
+
This allows functions to work with any connector-like object
|
|
48
|
+
that has these attributes, including ConnectorModel and mock objects.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def entities(self) -> list[EntityProtocol]: ...
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def openapi_spec(self) -> Any: ...
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def format_param_signature(endpoint: EndpointProtocol) -> str:
|
|
59
|
+
"""Format parameter signature for an endpoint action.
|
|
60
|
+
|
|
61
|
+
Returns a string like: (id*) or (limit?, starting_after?, email?)
|
|
62
|
+
where * = required, ? = optional
|
|
63
|
+
|
|
64
|
+
Args:
|
|
65
|
+
endpoint: Object conforming to EndpointProtocol (e.g., EndpointDefinition)
|
|
66
|
+
|
|
67
|
+
Returns:
|
|
68
|
+
Formatted parameter signature string
|
|
69
|
+
"""
|
|
70
|
+
params = []
|
|
71
|
+
|
|
72
|
+
# Defensive: safely access attributes with defaults for malformed endpoints
|
|
73
|
+
path_params = getattr(endpoint, "path_params", []) or []
|
|
74
|
+
query_params = getattr(endpoint, "query_params", []) or []
|
|
75
|
+
query_params_schema = getattr(endpoint, "query_params_schema", {}) or {}
|
|
76
|
+
body_fields = getattr(endpoint, "body_fields", []) or []
|
|
77
|
+
request_schema = getattr(endpoint, "request_schema", None)
|
|
78
|
+
|
|
79
|
+
# Path params (always required)
|
|
80
|
+
for name in path_params:
|
|
81
|
+
params.append(f"{name}*")
|
|
82
|
+
|
|
83
|
+
# Query params
|
|
84
|
+
for name in query_params:
|
|
85
|
+
schema = query_params_schema.get(name, {})
|
|
86
|
+
required = schema.get("required", False)
|
|
87
|
+
params.append(f"{name}{'*' if required else '?'}")
|
|
88
|
+
|
|
89
|
+
# Body fields
|
|
90
|
+
if request_schema:
|
|
91
|
+
required_fields = set(request_schema.get("required", []))
|
|
92
|
+
for name in body_fields:
|
|
93
|
+
params.append(f"{name}{'*' if name in required_fields else '?'}")
|
|
94
|
+
|
|
95
|
+
return f"({', '.join(params)})" if params else "()"
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def describe_entities(model: ConnectorModelProtocol) -> list[dict[str, Any]]:
|
|
99
|
+
"""Generate entity descriptions from ConnectorModel.
|
|
100
|
+
|
|
101
|
+
Returns a list of entity descriptions with detailed parameter information
|
|
102
|
+
for each action. This is used by generated connectors' describe() method.
|
|
103
|
+
|
|
104
|
+
Args:
|
|
105
|
+
model: Object conforming to ConnectorModelProtocol (e.g., ConnectorModel)
|
|
106
|
+
|
|
107
|
+
Returns:
|
|
108
|
+
List of entity description dicts with keys:
|
|
109
|
+
- entity_name: Name of the entity (e.g., "contacts", "deals")
|
|
110
|
+
- description: Entity description from the first endpoint
|
|
111
|
+
- available_actions: List of actions (e.g., ["list", "get", "create"])
|
|
112
|
+
- parameters: Dict mapping action -> list of parameter dicts
|
|
113
|
+
"""
|
|
114
|
+
entities = []
|
|
115
|
+
for entity_def in model.entities:
|
|
116
|
+
description = ""
|
|
117
|
+
parameters: dict[str, list[dict[str, Any]]] = {}
|
|
118
|
+
|
|
119
|
+
endpoints = getattr(entity_def, "endpoints", {}) or {}
|
|
120
|
+
if endpoints:
|
|
121
|
+
for action, endpoint in endpoints.items():
|
|
122
|
+
# Get description from first endpoint that has one
|
|
123
|
+
if not description:
|
|
124
|
+
endpoint_desc = getattr(endpoint, "description", None)
|
|
125
|
+
if endpoint_desc:
|
|
126
|
+
description = endpoint_desc
|
|
127
|
+
|
|
128
|
+
action_params: list[dict[str, Any]] = []
|
|
129
|
+
|
|
130
|
+
# Defensive: safely access endpoint attributes
|
|
131
|
+
path_params = getattr(endpoint, "path_params", []) or []
|
|
132
|
+
path_params_schema = getattr(endpoint, "path_params_schema", {}) or {}
|
|
133
|
+
query_params = getattr(endpoint, "query_params", []) or []
|
|
134
|
+
query_params_schema = getattr(endpoint, "query_params_schema", {}) or {}
|
|
135
|
+
body_fields = getattr(endpoint, "body_fields", []) or []
|
|
136
|
+
request_schema = getattr(endpoint, "request_schema", None)
|
|
137
|
+
|
|
138
|
+
# Path params (always required)
|
|
139
|
+
for param_name in path_params:
|
|
140
|
+
schema = path_params_schema.get(param_name, {})
|
|
141
|
+
action_params.append(
|
|
142
|
+
{
|
|
143
|
+
"name": param_name,
|
|
144
|
+
"in": "path",
|
|
145
|
+
"required": True,
|
|
146
|
+
"type": schema.get("type", "string"),
|
|
147
|
+
"description": schema.get("description", ""),
|
|
148
|
+
}
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
# Query params
|
|
152
|
+
for param_name in query_params:
|
|
153
|
+
schema = query_params_schema.get(param_name, {})
|
|
154
|
+
action_params.append(
|
|
155
|
+
{
|
|
156
|
+
"name": param_name,
|
|
157
|
+
"in": "query",
|
|
158
|
+
"required": schema.get("required", False),
|
|
159
|
+
"type": schema.get("type", "string"),
|
|
160
|
+
"description": schema.get("description", ""),
|
|
161
|
+
}
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
# Body fields
|
|
165
|
+
if request_schema:
|
|
166
|
+
required_fields = request_schema.get("required", [])
|
|
167
|
+
properties = request_schema.get("properties", {})
|
|
168
|
+
for param_name in body_fields:
|
|
169
|
+
prop = properties.get(param_name, {})
|
|
170
|
+
action_params.append(
|
|
171
|
+
{
|
|
172
|
+
"name": param_name,
|
|
173
|
+
"in": "body",
|
|
174
|
+
"required": param_name in required_fields,
|
|
175
|
+
"type": prop.get("type", "string"),
|
|
176
|
+
"description": prop.get("description", ""),
|
|
177
|
+
}
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
if action_params:
|
|
181
|
+
# Action is an enum, use .value to get string
|
|
182
|
+
action_key = action.value if hasattr(action, "value") else str(action)
|
|
183
|
+
parameters[action_key] = action_params
|
|
184
|
+
|
|
185
|
+
actions = getattr(entity_def, "actions", []) or []
|
|
186
|
+
entities.append(
|
|
187
|
+
{
|
|
188
|
+
"entity_name": entity_def.name,
|
|
189
|
+
"description": description,
|
|
190
|
+
"available_actions": [a.value if hasattr(a, "value") else str(a) for a in actions],
|
|
191
|
+
"parameters": parameters,
|
|
192
|
+
}
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
return entities
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def generate_tool_description(model: ConnectorModelProtocol) -> str:
|
|
199
|
+
"""Generate AI tool description from connector metadata.
|
|
200
|
+
|
|
201
|
+
Produces a detailed description that includes:
|
|
202
|
+
- Per-entity/action parameter signatures with required (*) and optional (?) markers
|
|
203
|
+
- Response structure documentation with pagination hints
|
|
204
|
+
- Example questions if available in the OpenAPI spec
|
|
205
|
+
|
|
206
|
+
This is used by the Connector.describe class method decorator to populate
|
|
207
|
+
function docstrings for AI framework integration.
|
|
208
|
+
|
|
209
|
+
Args:
|
|
210
|
+
model: Object conforming to ConnectorModelProtocol (e.g., ConnectorModel)
|
|
211
|
+
|
|
212
|
+
Returns:
|
|
213
|
+
Formatted description string suitable for AI tool documentation
|
|
214
|
+
"""
|
|
215
|
+
lines = []
|
|
216
|
+
|
|
217
|
+
# Entity/action parameter details (including pagination params like limit, starting_after)
|
|
218
|
+
lines.append("ENTITIES AND PARAMETERS:")
|
|
219
|
+
for entity in model.entities:
|
|
220
|
+
lines.append(f" {entity.name}:")
|
|
221
|
+
actions = getattr(entity, "actions", []) or []
|
|
222
|
+
endpoints = getattr(entity, "endpoints", {}) or {}
|
|
223
|
+
for action in actions:
|
|
224
|
+
action_str = action.value if hasattr(action, "value") else str(action)
|
|
225
|
+
endpoint = endpoints.get(action)
|
|
226
|
+
if endpoint:
|
|
227
|
+
param_sig = format_param_signature(endpoint)
|
|
228
|
+
lines.append(f" - {action_str}{param_sig}")
|
|
229
|
+
else:
|
|
230
|
+
lines.append(f" - {action_str}()")
|
|
231
|
+
|
|
232
|
+
# Response structure (brief, includes pagination hint)
|
|
233
|
+
lines.append("")
|
|
234
|
+
lines.append("RESPONSE STRUCTURE:")
|
|
235
|
+
lines.append(" - list/api_search: {data: [...], meta: {has_more: bool}}")
|
|
236
|
+
lines.append(" - get: Returns entity directly (no envelope)")
|
|
237
|
+
lines.append(" To paginate: pass starting_after=<last_id> while has_more is true")
|
|
238
|
+
|
|
239
|
+
# Add example questions if available in openapi_spec
|
|
240
|
+
openapi_spec = getattr(model, "openapi_spec", None)
|
|
241
|
+
if openapi_spec:
|
|
242
|
+
info = getattr(openapi_spec, "info", None)
|
|
243
|
+
if info:
|
|
244
|
+
example_questions = getattr(info, "x_airbyte_example_questions", None)
|
|
245
|
+
if example_questions:
|
|
246
|
+
supported = getattr(example_questions, "supported", None)
|
|
247
|
+
if supported:
|
|
248
|
+
lines.append("")
|
|
249
|
+
lines.append("EXAMPLE QUESTIONS:")
|
|
250
|
+
for q in supported[:MAX_EXAMPLE_QUESTIONS]:
|
|
251
|
+
lines.append(f" - {q}")
|
|
252
|
+
|
|
253
|
+
# Generic parameter description for function signature
|
|
254
|
+
lines.append("")
|
|
255
|
+
lines.append("FUNCTION PARAMETERS:")
|
|
256
|
+
lines.append(" - entity: Entity name (string)")
|
|
257
|
+
lines.append(" - action: Operation to perform (string)")
|
|
258
|
+
lines.append(" - params: Operation parameters (dict) - see entity details above")
|
|
259
|
+
lines.append("")
|
|
260
|
+
lines.append("Parameter markers: * = required, ? = optional")
|
|
261
|
+
|
|
262
|
+
return "\n".join(lines)
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""Unified configuration for connector-sdk."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import tempfile
|
|
6
|
+
import uuid
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any, Optional
|
|
10
|
+
|
|
11
|
+
import yaml
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
# New config location
|
|
16
|
+
CONFIG_DIR = Path.home() / ".airbyte" / "connector-sdk"
|
|
17
|
+
CONFIG_PATH = CONFIG_DIR / "config.yaml"
|
|
18
|
+
|
|
19
|
+
# Legacy file locations (for migration)
|
|
20
|
+
LEGACY_USER_ID_PATH = Path.home() / ".airbyte" / "ai_sdk_user_id"
|
|
21
|
+
LEGACY_INTERNAL_MARKER_PATH = Path.home() / ".airbyte" / "internal_user"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class SDKConfig:
|
|
26
|
+
"""Connector SDK configuration."""
|
|
27
|
+
|
|
28
|
+
user_id: str = field(default_factory=lambda: str(uuid.uuid4()))
|
|
29
|
+
is_internal_user: bool = False
|
|
30
|
+
|
|
31
|
+
def to_dict(self) -> dict[str, Any]:
|
|
32
|
+
"""Convert to dictionary for YAML serialization."""
|
|
33
|
+
return {
|
|
34
|
+
"user_id": self.user_id,
|
|
35
|
+
"is_internal_user": self.is_internal_user,
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _delete_legacy_files() -> None:
|
|
40
|
+
"""
|
|
41
|
+
Delete legacy config files after successful migration.
|
|
42
|
+
|
|
43
|
+
Removes:
|
|
44
|
+
- ~/.airbyte/ai_sdk_user_id
|
|
45
|
+
- ~/.airbyte/internal_user
|
|
46
|
+
"""
|
|
47
|
+
for legacy_path in [LEGACY_USER_ID_PATH, LEGACY_INTERNAL_MARKER_PATH]:
|
|
48
|
+
try:
|
|
49
|
+
if legacy_path.exists():
|
|
50
|
+
legacy_path.unlink()
|
|
51
|
+
logger.debug(f"Deleted legacy config file: {legacy_path}")
|
|
52
|
+
except Exception as e:
|
|
53
|
+
logger.debug(f"Could not delete legacy file {legacy_path}: {e}")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _migrate_legacy_config() -> Optional[SDKConfig]:
|
|
57
|
+
"""
|
|
58
|
+
Migrate from legacy file-based config to new YAML format.
|
|
59
|
+
|
|
60
|
+
Reads from:
|
|
61
|
+
- ~/.airbyte/ai_sdk_user_id (user_id)
|
|
62
|
+
- ~/.airbyte/internal_user (is_internal_user marker)
|
|
63
|
+
|
|
64
|
+
Returns SDKConfig if migration was successful, None otherwise.
|
|
65
|
+
"""
|
|
66
|
+
user_id = None
|
|
67
|
+
is_internal = False
|
|
68
|
+
|
|
69
|
+
# Try to read legacy user_id
|
|
70
|
+
try:
|
|
71
|
+
if LEGACY_USER_ID_PATH.exists():
|
|
72
|
+
user_id = LEGACY_USER_ID_PATH.read_text().strip()
|
|
73
|
+
if not user_id:
|
|
74
|
+
user_id = None
|
|
75
|
+
except Exception:
|
|
76
|
+
pass
|
|
77
|
+
|
|
78
|
+
# Check legacy internal_user marker
|
|
79
|
+
try:
|
|
80
|
+
is_internal = LEGACY_INTERNAL_MARKER_PATH.exists()
|
|
81
|
+
except Exception:
|
|
82
|
+
pass
|
|
83
|
+
|
|
84
|
+
if user_id or is_internal:
|
|
85
|
+
return SDKConfig(
|
|
86
|
+
user_id=user_id or str(uuid.uuid4()),
|
|
87
|
+
is_internal_user=is_internal,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
return None
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def load_config() -> SDKConfig:
|
|
94
|
+
"""
|
|
95
|
+
Load SDK configuration from config file.
|
|
96
|
+
|
|
97
|
+
Checks (in order):
|
|
98
|
+
1. New config file at ~/.airbyte/connector-sdk/config.yaml
|
|
99
|
+
2. Legacy files at ~/.airbyte/ai_sdk_user_id and ~/.airbyte/internal_user
|
|
100
|
+
3. Creates new config with generated user_id if nothing exists
|
|
101
|
+
|
|
102
|
+
Environment variable AIRBYTE_INTERNAL_USER can override is_internal_user.
|
|
103
|
+
|
|
104
|
+
Returns:
|
|
105
|
+
SDKConfig with user_id and is_internal_user
|
|
106
|
+
"""
|
|
107
|
+
config = None
|
|
108
|
+
|
|
109
|
+
# Try to load from new config file
|
|
110
|
+
try:
|
|
111
|
+
if CONFIG_PATH.exists():
|
|
112
|
+
content = CONFIG_PATH.read_text()
|
|
113
|
+
data = yaml.safe_load(content) or {}
|
|
114
|
+
config = SDKConfig(
|
|
115
|
+
user_id=data.get("user_id", str(uuid.uuid4())),
|
|
116
|
+
is_internal_user=data.get("is_internal_user", False),
|
|
117
|
+
)
|
|
118
|
+
# Always clean up legacy files if they exist (even if new config exists)
|
|
119
|
+
_delete_legacy_files()
|
|
120
|
+
except Exception as e:
|
|
121
|
+
logger.debug(f"Could not load config from {CONFIG_PATH}: {e}")
|
|
122
|
+
|
|
123
|
+
# Try to migrate from legacy files if new config doesn't exist
|
|
124
|
+
if config is None:
|
|
125
|
+
config = _migrate_legacy_config()
|
|
126
|
+
if config:
|
|
127
|
+
# Save migrated config to new location
|
|
128
|
+
try:
|
|
129
|
+
save_config(config)
|
|
130
|
+
logger.debug("Migrated legacy config to new location")
|
|
131
|
+
# Delete legacy files after successful migration
|
|
132
|
+
_delete_legacy_files()
|
|
133
|
+
except Exception as e:
|
|
134
|
+
logger.debug(f"Could not save migrated config: {e}")
|
|
135
|
+
|
|
136
|
+
# Create new config if nothing exists
|
|
137
|
+
if config is None:
|
|
138
|
+
config = SDKConfig()
|
|
139
|
+
try:
|
|
140
|
+
save_config(config)
|
|
141
|
+
except Exception as e:
|
|
142
|
+
logger.debug(f"Could not save new config: {e}")
|
|
143
|
+
|
|
144
|
+
# Environment variable override for is_internal_user
|
|
145
|
+
env_value = os.getenv("AIRBYTE_INTERNAL_USER", "").lower()
|
|
146
|
+
if env_value in ("true", "1", "yes"):
|
|
147
|
+
config.is_internal_user = True
|
|
148
|
+
elif env_value:
|
|
149
|
+
# Any other non-empty value (including "false", "0", "no") defaults to False
|
|
150
|
+
config.is_internal_user = False
|
|
151
|
+
|
|
152
|
+
return config
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def save_config(config: SDKConfig) -> None:
|
|
156
|
+
"""
|
|
157
|
+
Save SDK configuration to config file.
|
|
158
|
+
|
|
159
|
+
Creates the config directory if it doesn't exist.
|
|
160
|
+
Uses atomic writes to prevent corruption from concurrent access.
|
|
161
|
+
|
|
162
|
+
Args:
|
|
163
|
+
config: SDKConfig to save
|
|
164
|
+
"""
|
|
165
|
+
CONFIG_DIR.mkdir(parents=True, exist_ok=True)
|
|
166
|
+
|
|
167
|
+
# Use atomic write: write to temp file then rename (atomic on POSIX)
|
|
168
|
+
fd, temp_path = tempfile.mkstemp(dir=CONFIG_DIR, suffix=".tmp")
|
|
169
|
+
try:
|
|
170
|
+
with os.fdopen(fd, "w") as f:
|
|
171
|
+
yaml.dump(config.to_dict(), f, default_flow_style=False)
|
|
172
|
+
os.rename(temp_path, CONFIG_PATH)
|
|
173
|
+
except Exception:
|
|
174
|
+
# Clean up temp file on failure
|
|
175
|
+
try:
|
|
176
|
+
os.unlink(temp_path)
|
|
177
|
+
except OSError:
|
|
178
|
+
pass
|
|
179
|
+
raise
|
|
@@ -3,46 +3,40 @@
|
|
|
3
3
|
import logging
|
|
4
4
|
import uuid
|
|
5
5
|
from datetime import UTC, datetime
|
|
6
|
-
from pathlib import Path
|
|
7
6
|
from typing import Any, Dict, Optional
|
|
8
7
|
|
|
8
|
+
from .config import SDKConfig, load_config
|
|
9
|
+
|
|
9
10
|
logger = logging.getLogger(__name__)
|
|
10
11
|
|
|
12
|
+
# Cache the config at module level to avoid repeated reads
|
|
13
|
+
_cached_config: Optional[SDKConfig] = None
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _get_config() -> SDKConfig:
|
|
17
|
+
"""Get cached SDK config or load from file."""
|
|
18
|
+
global _cached_config
|
|
19
|
+
if _cached_config is None:
|
|
20
|
+
_cached_config = load_config()
|
|
21
|
+
return _cached_config
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _clear_config_cache() -> None:
|
|
25
|
+
"""Clear the cached config. Used for testing."""
|
|
26
|
+
global _cached_config
|
|
27
|
+
_cached_config = None
|
|
28
|
+
|
|
11
29
|
|
|
12
30
|
def get_persistent_user_id() -> str:
|
|
13
31
|
"""
|
|
14
|
-
Get
|
|
32
|
+
Get the persistent anonymous user ID.
|
|
15
33
|
|
|
16
|
-
|
|
17
|
-
If the file doesn't exist, a new UUID is generated and saved.
|
|
34
|
+
Now reads from ~/.airbyte/connector-sdk/config.yaml
|
|
18
35
|
|
|
19
36
|
Returns:
|
|
20
37
|
An anonymous UUID string that uniquely identifies this user across sessions.
|
|
21
38
|
"""
|
|
22
|
-
|
|
23
|
-
# Create .airbyte directory in home folder if it doesn't exist
|
|
24
|
-
airbyte_dir = Path.home() / ".airbyte"
|
|
25
|
-
airbyte_dir.mkdir(exist_ok=True)
|
|
26
|
-
|
|
27
|
-
# Path to user ID file
|
|
28
|
-
user_id_file = airbyte_dir / "ai_sdk_user_id"
|
|
29
|
-
|
|
30
|
-
# Try to read existing user ID
|
|
31
|
-
if user_id_file.exists():
|
|
32
|
-
user_id = user_id_file.read_text().strip()
|
|
33
|
-
if user_id: # Validate it's not empty
|
|
34
|
-
return user_id
|
|
35
|
-
|
|
36
|
-
# Generate new user ID if file doesn't exist or is empty
|
|
37
|
-
user_id = str(uuid.uuid4())
|
|
38
|
-
user_id_file.write_text(user_id)
|
|
39
|
-
logger.debug(f"Generated new anonymous user ID: {user_id}")
|
|
40
|
-
|
|
41
|
-
return user_id
|
|
42
|
-
except Exception as e:
|
|
43
|
-
# If we can't read/write the file, generate a session-only ID
|
|
44
|
-
logger.debug(f"Could not access anonymous user ID file: {e}")
|
|
45
|
-
return str(uuid.uuid4())
|
|
39
|
+
return _get_config().user_id
|
|
46
40
|
|
|
47
41
|
|
|
48
42
|
def get_public_ip() -> Optional[str]:
|
|
@@ -65,6 +59,18 @@ def get_public_ip() -> Optional[str]:
|
|
|
65
59
|
return None
|
|
66
60
|
|
|
67
61
|
|
|
62
|
+
def get_is_internal_user() -> bool:
|
|
63
|
+
"""
|
|
64
|
+
Check if the current user is an internal Airbyte user.
|
|
65
|
+
|
|
66
|
+
Now reads from ~/.airbyte/connector-sdk/config.yaml
|
|
67
|
+
Environment variable AIRBYTE_INTERNAL_USER can override.
|
|
68
|
+
|
|
69
|
+
Returns False if not set or on any error.
|
|
70
|
+
"""
|
|
71
|
+
return _get_config().is_internal_user
|
|
72
|
+
|
|
73
|
+
|
|
68
74
|
class ObservabilitySession:
|
|
69
75
|
"""Shared session context for both logging and telemetry."""
|
|
70
76
|
|
|
@@ -84,6 +90,7 @@ class ObservabilitySession:
|
|
|
84
90
|
self.operation_count = 0
|
|
85
91
|
self.metadata: Dict[str, Any] = {}
|
|
86
92
|
self.public_ip = get_public_ip()
|
|
93
|
+
self.is_internal_user = get_is_internal_user()
|
|
87
94
|
|
|
88
95
|
def increment_operations(self):
|
|
89
96
|
"""Increment the operation counter."""
|
|
@@ -65,8 +65,9 @@ class Schema(BaseModel):
|
|
|
65
65
|
write_only: Optional[bool] = Field(None, alias="writeOnly")
|
|
66
66
|
deprecated: Optional[bool] = None
|
|
67
67
|
|
|
68
|
-
# Airbyte
|
|
68
|
+
# Airbyte extensions
|
|
69
69
|
x_airbyte_entity_name: Optional[str] = Field(None, alias="x-airbyte-entity-name")
|
|
70
|
+
x_airbyte_stream_name: Optional[str] = Field(None, alias="x-airbyte-stream-name")
|
|
70
71
|
|
|
71
72
|
|
|
72
73
|
class Parameter(BaseModel):
|
|
@@ -61,7 +61,7 @@ class Operation(BaseModel):
|
|
|
61
61
|
description=(
|
|
62
62
|
"JSONPath expression to extract records from API response envelopes. "
|
|
63
63
|
"When specified, executor extracts data at this path instead of returning "
|
|
64
|
-
"full response. Returns array for list/
|
|
64
|
+
"full response. Returns array for list/api_search actions, single record for "
|
|
65
65
|
"get/create/update/delete actions."
|
|
66
66
|
),
|
|
67
67
|
)
|
|
@@ -77,6 +77,10 @@ class AuthConfigOption(BaseModel):
|
|
|
77
77
|
default_factory=dict,
|
|
78
78
|
description="Mapping from auth parameters (e.g., 'username', 'password', 'token') to template strings using ${field} syntax",
|
|
79
79
|
)
|
|
80
|
+
replication_auth_key_mapping: Optional[Dict[str, str]] = Field(
|
|
81
|
+
None,
|
|
82
|
+
description="Mapping from source config paths (e.g., 'credentials.api_key') to auth config keys for direct connectors",
|
|
83
|
+
)
|
|
80
84
|
|
|
81
85
|
|
|
82
86
|
class AirbyteAuthConfig(BaseModel):
|
|
@@ -99,6 +103,12 @@ class AirbyteAuthConfig(BaseModel):
|
|
|
99
103
|
properties: Optional[Dict[str, AuthConfigFieldSpec]] = None
|
|
100
104
|
auth_mapping: Optional[Dict[str, str]] = None
|
|
101
105
|
|
|
106
|
+
# Replication connector auth mapping
|
|
107
|
+
replication_auth_key_mapping: Optional[Dict[str, str]] = Field(
|
|
108
|
+
None,
|
|
109
|
+
description="Mapping from source config paths (e.g., 'credentials.api_key') to auth config keys for direct connectors",
|
|
110
|
+
)
|
|
111
|
+
|
|
102
112
|
# Multiple options (oneOf)
|
|
103
113
|
one_of: Optional[List[AuthConfigOption]] = Field(None, alias="oneOf")
|
|
104
114
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Telemetry event models."""
|
|
2
2
|
|
|
3
|
-
from dataclasses import asdict, dataclass
|
|
3
|
+
from dataclasses import asdict, dataclass, field
|
|
4
4
|
from datetime import datetime
|
|
5
5
|
from typing import Any, Dict, Optional
|
|
6
6
|
|
|
@@ -13,6 +13,7 @@ class BaseEvent:
|
|
|
13
13
|
session_id: str
|
|
14
14
|
user_id: str
|
|
15
15
|
execution_context: str
|
|
16
|
+
is_internal_user: bool = field(default=False, kw_only=True)
|
|
16
17
|
|
|
17
18
|
def to_dict(self) -> Dict[str, Any]:
|
|
18
19
|
"""Convert event to dictionary with ISO formatted timestamp."""
|
|
@@ -59,6 +59,7 @@ class SegmentTracker:
|
|
|
59
59
|
session_id=self.session.session_id,
|
|
60
60
|
user_id=self.session.user_id,
|
|
61
61
|
execution_context=self.session.execution_context,
|
|
62
|
+
is_internal_user=self.session.is_internal_user,
|
|
62
63
|
public_ip=self.session.public_ip,
|
|
63
64
|
connector_name=self.session.connector_name,
|
|
64
65
|
connector_version=connector_version,
|
|
@@ -101,6 +102,7 @@ class SegmentTracker:
|
|
|
101
102
|
session_id=self.session.session_id,
|
|
102
103
|
user_id=self.session.user_id,
|
|
103
104
|
execution_context=self.session.execution_context,
|
|
105
|
+
is_internal_user=self.session.is_internal_user,
|
|
104
106
|
public_ip=self.session.public_ip,
|
|
105
107
|
connector_name=self.session.connector_name,
|
|
106
108
|
entity=entity,
|
|
@@ -130,6 +132,7 @@ class SegmentTracker:
|
|
|
130
132
|
session_id=self.session.session_id,
|
|
131
133
|
user_id=self.session.user_id,
|
|
132
134
|
execution_context=self.session.execution_context,
|
|
135
|
+
is_internal_user=self.session.is_internal_user,
|
|
133
136
|
public_ip=self.session.public_ip,
|
|
134
137
|
connector_name=self.session.connector_name,
|
|
135
138
|
duration_seconds=self.session.duration_seconds(),
|
|
@@ -22,7 +22,7 @@ class Action(str, Enum):
|
|
|
22
22
|
UPDATE = "update"
|
|
23
23
|
DELETE = "delete"
|
|
24
24
|
LIST = "list"
|
|
25
|
-
|
|
25
|
+
API_SEARCH = "api_search"
|
|
26
26
|
DOWNLOAD = "download"
|
|
27
27
|
AUTHORIZE = "authorize"
|
|
28
28
|
|
|
@@ -221,6 +221,10 @@ class EntityDefinition(BaseModel):
|
|
|
221
221
|
model_config = {"populate_by_name": True}
|
|
222
222
|
|
|
223
223
|
name: str
|
|
224
|
+
stream_name: str | None = Field(
|
|
225
|
+
default=None,
|
|
226
|
+
description="Airbyte stream name for cache lookup (from x-airbyte-stream-name schema extension)",
|
|
227
|
+
)
|
|
224
228
|
actions: list[Action]
|
|
225
229
|
endpoints: dict[Action, EndpointDefinition]
|
|
226
230
|
entity_schema: dict[str, Any] | None = Field(default=None, alias="schema")
|
|
@@ -10,20 +10,21 @@ airbyte_agent_mcp/_vendored/__init__.py,sha256=ILl7AHXMui__swyrjxrh9yRa4dLiwBvV6
|
|
|
10
10
|
airbyte_agent_mcp/_vendored/connector_sdk/__init__.py,sha256=T5o7roU6NSpH-lCAGZ338sE5dlh4ZU6i6IkeG1zpems,1949
|
|
11
11
|
airbyte_agent_mcp/_vendored/connector_sdk/auth_strategies.py,sha256=0BfIISVzuvZTAYZjQFOOhKTpw0QuKDlLQBQ1PQo-V2M,39967
|
|
12
12
|
airbyte_agent_mcp/_vendored/connector_sdk/auth_template.py,sha256=vKnyA21Jp33EuDjkIUAf1PGicwk4t9kZAPJuAgAZKzU,4458
|
|
13
|
-
airbyte_agent_mcp/_vendored/connector_sdk/connector_model_loader.py,sha256=
|
|
14
|
-
airbyte_agent_mcp/_vendored/connector_sdk/constants.py,sha256=
|
|
13
|
+
airbyte_agent_mcp/_vendored/connector_sdk/connector_model_loader.py,sha256=nKrXfe-FAyvNMkW7AqGzxrp5wXdaHiqC0yIFJoIVwlY,34890
|
|
14
|
+
airbyte_agent_mcp/_vendored/connector_sdk/constants.py,sha256=AtzOvhDMWbRJgpsQNWl5tkogHD6mWgEY668PgRmgtOY,2737
|
|
15
15
|
airbyte_agent_mcp/_vendored/connector_sdk/exceptions.py,sha256=ss5MGv9eVPmsbLcLWetuu3sDmvturwfo6Pw3M37Oq5k,481
|
|
16
|
-
airbyte_agent_mcp/_vendored/connector_sdk/extensions.py,sha256=
|
|
17
|
-
airbyte_agent_mcp/_vendored/connector_sdk/http_client.py,sha256=
|
|
16
|
+
airbyte_agent_mcp/_vendored/connector_sdk/extensions.py,sha256=Ks4LR1lbJfBoJ-gN6byj7WtFGE1xMuerVTHXGKJcFKk,21391
|
|
17
|
+
airbyte_agent_mcp/_vendored/connector_sdk/http_client.py,sha256=NdccrrBHI5rW56XnXcP54arCwywIVKnMeSQPas6KlOM,27466
|
|
18
|
+
airbyte_agent_mcp/_vendored/connector_sdk/introspection.py,sha256=2CyKXZHT74-1Id97uw1RLeyOi6TV24_hoNbQ6-6y7uI,10335
|
|
18
19
|
airbyte_agent_mcp/_vendored/connector_sdk/secrets.py,sha256=UWcO9fP-vZwcfkAuvlZahlOCTOwdNN860BIwe8X4jxw,6868
|
|
19
|
-
airbyte_agent_mcp/_vendored/connector_sdk/types.py,sha256=
|
|
20
|
+
airbyte_agent_mcp/_vendored/connector_sdk/types.py,sha256=0yY6mF03Dan5W9RNVHIEYlOijxa1j5T28TErjN_kUvQ,8293
|
|
20
21
|
airbyte_agent_mcp/_vendored/connector_sdk/utils.py,sha256=G4LUXOC2HzPoND2v4tQW68R9uuPX9NQyCjaGxb7Kpl0,1958
|
|
21
22
|
airbyte_agent_mcp/_vendored/connector_sdk/validation.py,sha256=CDjCux1eg35a0Y4BegSivzIwZjiTqOxYWotWNLqTSVU,31792
|
|
22
23
|
airbyte_agent_mcp/_vendored/connector_sdk/cloud_utils/__init__.py,sha256=4799Hv9f2zxDVj1aLyQ8JpTEuFTp_oOZMRz-NZCdBJg,134
|
|
23
24
|
airbyte_agent_mcp/_vendored/connector_sdk/cloud_utils/client.py,sha256=HoDgZuEgGHj78P-BGwUf6HGPVWynbdKjGOmjb-JDk58,7188
|
|
24
25
|
airbyte_agent_mcp/_vendored/connector_sdk/executor/__init__.py,sha256=EmG9YQNAjSuYCVB4D5VoLm4qpD1KfeiiOf7bpALj8p8,702
|
|
25
26
|
airbyte_agent_mcp/_vendored/connector_sdk/executor/hosted_executor.py,sha256=YQ-qfT7PZh9izNFHHe7SAcETiZOKrWjTU-okVb0_VL8,7079
|
|
26
|
-
airbyte_agent_mcp/_vendored/connector_sdk/executor/local_executor.py,sha256=
|
|
27
|
+
airbyte_agent_mcp/_vendored/connector_sdk/executor/local_executor.py,sha256=ODbOLjlHjh_63T8XYGEJLuIz9e8QOZOGVdWFBsWWL44,65283
|
|
27
28
|
airbyte_agent_mcp/_vendored/connector_sdk/executor/models.py,sha256=lYVT_bNcw-PoIks4WHNyl2VY-lJVf2FntzINSOBIheE,5845
|
|
28
29
|
airbyte_agent_mcp/_vendored/connector_sdk/http/__init__.py,sha256=y8fbzZn-3yV9OxtYz8Dy6FFGI5v6TOqADd1G3xHH3Hw,911
|
|
29
30
|
airbyte_agent_mcp/_vendored/connector_sdk/http/config.py,sha256=6J7YIIwHC6sRu9i-yKa5XvArwK2KU60rlnmxzDZq3lw,3283
|
|
@@ -36,23 +37,24 @@ airbyte_agent_mcp/_vendored/connector_sdk/logging/__init__.py,sha256=IZoE5yXhwSA
|
|
|
36
37
|
airbyte_agent_mcp/_vendored/connector_sdk/logging/logger.py,sha256=Nh0h3C0aO-rAqZhDIyeEXG6Jd7yj1Gk32ECMPth0wl8,8118
|
|
37
38
|
airbyte_agent_mcp/_vendored/connector_sdk/logging/types.py,sha256=iI-xLoOg33OUGQOp3CeaxKtHh73WXE-oul6ZCNf3Nzc,3209
|
|
38
39
|
airbyte_agent_mcp/_vendored/connector_sdk/observability/__init__.py,sha256=ojx1n9vaWCXFFvb3-90Pwtg845trFdV2v6wcCoO75Ko,269
|
|
40
|
+
airbyte_agent_mcp/_vendored/connector_sdk/observability/config.py,sha256=GJLzILUc7X6hqwL_Ger6sLAhlrecwV3GcjBJzvmXnqg,5385
|
|
39
41
|
airbyte_agent_mcp/_vendored/connector_sdk/observability/models.py,sha256=rF6-SoAAywqL8bhEv7yCbmr_W_w0vmApO-MCxcHq9RI,473
|
|
40
42
|
airbyte_agent_mcp/_vendored/connector_sdk/observability/redactor.py,sha256=HRbSwGxsfQYbYlG4QBVvv8Qnw0d4SMowMv0dTFHsHqQ,2361
|
|
41
|
-
airbyte_agent_mcp/_vendored/connector_sdk/observability/session.py,sha256=
|
|
43
|
+
airbyte_agent_mcp/_vendored/connector_sdk/observability/session.py,sha256=Q1U6qaH_Uj_eZ9sT6_Wrl27YWd95cxjA7S0GYWUrXkU,2941
|
|
42
44
|
airbyte_agent_mcp/_vendored/connector_sdk/performance/__init__.py,sha256=Sp5fSd1LvtIQkv-fnrKqPsM7-6IWp0sSZSK0mhzal_A,200
|
|
43
45
|
airbyte_agent_mcp/_vendored/connector_sdk/performance/instrumentation.py,sha256=_dXvNiqdndIBwDjeDKNViWzn_M5FkSUsMmJtFldrmsM,1504
|
|
44
46
|
airbyte_agent_mcp/_vendored/connector_sdk/performance/metrics.py,sha256=3-wPwlJyfVLUIG3y7ESxk0avhkILk3z8K7zSrnlZf5U,2833
|
|
45
47
|
airbyte_agent_mcp/_vendored/connector_sdk/schema/__init__.py,sha256=Uymu-QuzGJuMxexBagIvUxpVAigIuIhz3KeBl_Vu4Ko,1638
|
|
46
48
|
airbyte_agent_mcp/_vendored/connector_sdk/schema/base.py,sha256=eN6YfHwsYNz_0CE-S715Me8x3gQWTtaRk1vhFjEZF8I,4799
|
|
47
|
-
airbyte_agent_mcp/_vendored/connector_sdk/schema/components.py,sha256=
|
|
49
|
+
airbyte_agent_mcp/_vendored/connector_sdk/schema/components.py,sha256=4clYbIJNqfmFj7Te31ZqC3Rr5E2v33BeXyQx1wp3oj0,8076
|
|
48
50
|
airbyte_agent_mcp/_vendored/connector_sdk/schema/connector.py,sha256=VFBOzIkLgYBR1XUTmyrPGqBkX8PP-zsG8avQcdJUqXs,3864
|
|
49
51
|
airbyte_agent_mcp/_vendored/connector_sdk/schema/extensions.py,sha256=LdoCuMLNdCj68O47qAL4v8xmqGz5tJgRiNZL5JnL9uw,3311
|
|
50
|
-
airbyte_agent_mcp/_vendored/connector_sdk/schema/operations.py,sha256=
|
|
51
|
-
airbyte_agent_mcp/_vendored/connector_sdk/schema/security.py,sha256=
|
|
52
|
+
airbyte_agent_mcp/_vendored/connector_sdk/schema/operations.py,sha256=HV_4mWya-kvSXOWVGPU1uD4GK3GIWneqETFqy9cnrpM,5829
|
|
53
|
+
airbyte_agent_mcp/_vendored/connector_sdk/schema/security.py,sha256=Xo6Z0-NCryMY4EDZgvg8ABSPglskpXANvm_iI34meV8,8588
|
|
52
54
|
airbyte_agent_mcp/_vendored/connector_sdk/telemetry/__init__.py,sha256=RaLgkBU4dfxn1LC5Y0Q9rr2PJbrwjxvPgBLmq8_WafE,211
|
|
53
55
|
airbyte_agent_mcp/_vendored/connector_sdk/telemetry/config.py,sha256=tLmQwAFD0kP1WyBGWBS3ysaudN9H3e-3EopKZi6cGKg,885
|
|
54
|
-
airbyte_agent_mcp/_vendored/connector_sdk/telemetry/events.py,sha256=
|
|
55
|
-
airbyte_agent_mcp/_vendored/connector_sdk/telemetry/tracker.py,sha256=
|
|
56
|
-
airbyte_agent_mcp-0.1.
|
|
57
|
-
airbyte_agent_mcp-0.1.
|
|
58
|
-
airbyte_agent_mcp-0.1.
|
|
56
|
+
airbyte_agent_mcp/_vendored/connector_sdk/telemetry/events.py,sha256=xUiasLicB818vfGVB108U7noVVDxz35g5vMomRKi0eE,1356
|
|
57
|
+
airbyte_agent_mcp/_vendored/connector_sdk/telemetry/tracker.py,sha256=lURbjZVZSWLluX3OTK_0D8xrc3HAzK9xixtMTOEWnQ4,5439
|
|
58
|
+
airbyte_agent_mcp-0.1.53.dist-info/METADATA,sha256=q3PUSDVjjLnevElY4i0oaMRQdifyp4n2Hb92cpiFMB8,3009
|
|
59
|
+
airbyte_agent_mcp-0.1.53.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
60
|
+
airbyte_agent_mcp-0.1.53.dist-info/RECORD,,
|
|
File without changes
|