airbyte-agent-zendesk-support 0.18.18__py3-none-any.whl → 0.18.51__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_agent_zendesk_support/__init__.py +239 -18
- airbyte_agent_zendesk_support/_vendored/connector_sdk/auth_strategies.py +2 -5
- airbyte_agent_zendesk_support/_vendored/connector_sdk/auth_template.py +1 -1
- airbyte_agent_zendesk_support/_vendored/connector_sdk/cloud_utils/__init__.py +5 -0
- airbyte_agent_zendesk_support/_vendored/connector_sdk/cloud_utils/client.py +213 -0
- airbyte_agent_zendesk_support/_vendored/connector_sdk/connector_model_loader.py +35 -8
- airbyte_agent_zendesk_support/_vendored/connector_sdk/constants.py +1 -1
- airbyte_agent_zendesk_support/_vendored/connector_sdk/executor/hosted_executor.py +92 -84
- airbyte_agent_zendesk_support/_vendored/connector_sdk/executor/local_executor.py +274 -54
- airbyte_agent_zendesk_support/_vendored/connector_sdk/extensions.py +43 -5
- airbyte_agent_zendesk_support/_vendored/connector_sdk/http/response.py +2 -0
- airbyte_agent_zendesk_support/_vendored/connector_sdk/http_client.py +63 -49
- airbyte_agent_zendesk_support/_vendored/connector_sdk/introspection.py +262 -0
- airbyte_agent_zendesk_support/_vendored/connector_sdk/logging/logger.py +19 -10
- airbyte_agent_zendesk_support/_vendored/connector_sdk/logging/types.py +11 -10
- airbyte_agent_zendesk_support/_vendored/connector_sdk/observability/config.py +179 -0
- airbyte_agent_zendesk_support/_vendored/connector_sdk/observability/models.py +6 -6
- airbyte_agent_zendesk_support/_vendored/connector_sdk/observability/session.py +41 -32
- airbyte_agent_zendesk_support/_vendored/connector_sdk/performance/metrics.py +3 -3
- airbyte_agent_zendesk_support/_vendored/connector_sdk/schema/base.py +22 -18
- airbyte_agent_zendesk_support/_vendored/connector_sdk/schema/components.py +59 -58
- airbyte_agent_zendesk_support/_vendored/connector_sdk/schema/connector.py +22 -33
- airbyte_agent_zendesk_support/_vendored/connector_sdk/schema/extensions.py +131 -10
- airbyte_agent_zendesk_support/_vendored/connector_sdk/schema/operations.py +32 -32
- airbyte_agent_zendesk_support/_vendored/connector_sdk/schema/security.py +44 -34
- airbyte_agent_zendesk_support/_vendored/connector_sdk/secrets.py +2 -2
- airbyte_agent_zendesk_support/_vendored/connector_sdk/telemetry/events.py +9 -8
- airbyte_agent_zendesk_support/_vendored/connector_sdk/telemetry/tracker.py +9 -5
- airbyte_agent_zendesk_support/_vendored/connector_sdk/types.py +9 -3
- airbyte_agent_zendesk_support/_vendored/connector_sdk/validation.py +12 -6
- airbyte_agent_zendesk_support/connector.py +1170 -171
- airbyte_agent_zendesk_support/connector_model.py +7 -1
- airbyte_agent_zendesk_support/models.py +628 -69
- airbyte_agent_zendesk_support/types.py +3717 -1
- {airbyte_agent_zendesk_support-0.18.18.dist-info → airbyte_agent_zendesk_support-0.18.51.dist-info}/METADATA +55 -31
- {airbyte_agent_zendesk_support-0.18.18.dist-info → airbyte_agent_zendesk_support-0.18.51.dist-info}/RECORD +37 -33
- {airbyte_agent_zendesk_support-0.18.18.dist-info → airbyte_agent_zendesk_support-0.18.51.dist-info}/WHEEL +0 -0
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import asyncio
|
|
6
|
+
import inspect
|
|
6
7
|
import logging
|
|
7
8
|
import os
|
|
8
9
|
import re
|
|
@@ -11,6 +12,7 @@ from collections.abc import AsyncIterator
|
|
|
11
12
|
from typing import Any, Protocol
|
|
12
13
|
from urllib.parse import quote
|
|
13
14
|
|
|
15
|
+
from jinja2 import Environment, StrictUndefined
|
|
14
16
|
from jsonpath_ng import parse as parse_jsonpath
|
|
15
17
|
from opentelemetry import trace
|
|
16
18
|
|
|
@@ -493,6 +495,14 @@ class LocalExecutor:
|
|
|
493
495
|
print(result.data)
|
|
494
496
|
"""
|
|
495
497
|
try:
|
|
498
|
+
# Check for hosted-only actions before converting to Action enum
|
|
499
|
+
if config.action == "search":
|
|
500
|
+
raise NotImplementedError(
|
|
501
|
+
"search is only available in hosted execution mode. "
|
|
502
|
+
"Initialize the connector with external_user_id, airbyte_client_id, "
|
|
503
|
+
"and airbyte_client_secret to use this feature."
|
|
504
|
+
)
|
|
505
|
+
|
|
496
506
|
# Convert config to internal format
|
|
497
507
|
action = Action(config.action) if isinstance(config.action, str) else config.action
|
|
498
508
|
params = config.params or {}
|
|
@@ -506,8 +516,6 @@ class LocalExecutor:
|
|
|
506
516
|
result = handler.execute_operation(config.entity, action, params)
|
|
507
517
|
|
|
508
518
|
# Check if it's an async generator (download) or awaitable (standard)
|
|
509
|
-
import inspect
|
|
510
|
-
|
|
511
519
|
if inspect.isasyncgen(result):
|
|
512
520
|
# Download operation: return generator directly
|
|
513
521
|
return ExecutionResult(
|
|
@@ -674,16 +682,16 @@ class LocalExecutor:
|
|
|
674
682
|
return {key: value for key, value in params.items() if key in allowed_params}
|
|
675
683
|
|
|
676
684
|
def _extract_body(self, allowed_fields: list[str], params: dict[str, Any]) -> dict[str, Any]:
|
|
677
|
-
"""Extract body fields from params.
|
|
685
|
+
"""Extract body fields from params, filtering out None values.
|
|
678
686
|
|
|
679
687
|
Args:
|
|
680
688
|
allowed_fields: List of allowed body field names
|
|
681
689
|
params: All parameters
|
|
682
690
|
|
|
683
691
|
Returns:
|
|
684
|
-
Dictionary of body fields
|
|
692
|
+
Dictionary of body fields with None values filtered out
|
|
685
693
|
"""
|
|
686
|
-
return {key: value for key, value in params.items() if key in allowed_fields}
|
|
694
|
+
return {key: value for key, value in params.items() if key in allowed_fields and value is not None}
|
|
687
695
|
|
|
688
696
|
def _serialize_deep_object_params(self, params: dict[str, Any], deep_object_param_names: list[str]) -> dict[str, Any]:
|
|
689
697
|
"""Serialize deepObject parameters to bracket notation format.
|
|
@@ -814,7 +822,6 @@ class LocalExecutor:
|
|
|
814
822
|
>>> _substitute_file_field_params("attachments[{attachment_index}].url", {"attachment_index": 0})
|
|
815
823
|
"attachments[0].url"
|
|
816
824
|
"""
|
|
817
|
-
from jinja2 import Environment, StrictUndefined
|
|
818
825
|
|
|
819
826
|
# Use custom delimiters to match OpenAPI path parameter syntax {var}
|
|
820
827
|
# StrictUndefined raises clear error if a template variable is missing
|
|
@@ -837,15 +844,65 @@ class LocalExecutor:
|
|
|
837
844
|
Request body dict or None if no body needed
|
|
838
845
|
"""
|
|
839
846
|
if endpoint.graphql_body:
|
|
840
|
-
|
|
847
|
+
# Extract defaults from query_params_schema for GraphQL variable interpolation
|
|
848
|
+
param_defaults = {name: schema.get("default") for name, schema in endpoint.query_params_schema.items() if "default" in schema}
|
|
849
|
+
return self._build_graphql_body(endpoint.graphql_body, params, param_defaults)
|
|
841
850
|
elif endpoint.body_fields:
|
|
842
851
|
return self._extract_body(endpoint.body_fields, params)
|
|
843
852
|
return None
|
|
844
853
|
|
|
854
|
+
def _flatten_form_data(self, data: dict[str, Any], parent_key: str = "") -> dict[str, Any]:
|
|
855
|
+
"""Flatten nested dict/list structures into bracket notation for form encoding.
|
|
856
|
+
|
|
857
|
+
Stripe and similar APIs require nested arrays/objects to be encoded using bracket
|
|
858
|
+
notation when using application/x-www-form-urlencoded content type.
|
|
859
|
+
|
|
860
|
+
Args:
|
|
861
|
+
data: Nested dict with arrays/objects to flatten
|
|
862
|
+
parent_key: Parent key for nested structures (used in recursion)
|
|
863
|
+
|
|
864
|
+
Returns:
|
|
865
|
+
Flattened dict with bracket notation keys
|
|
866
|
+
|
|
867
|
+
Examples:
|
|
868
|
+
>>> _flatten_form_data({"items": [{"price": "p1", "qty": 1}]})
|
|
869
|
+
{"items[0][price]": "p1", "items[0][qty]": 1}
|
|
870
|
+
|
|
871
|
+
>>> _flatten_form_data({"customer": "cus_123", "metadata": {"key": "value"}})
|
|
872
|
+
{"customer": "cus_123", "metadata[key]": "value"}
|
|
873
|
+
"""
|
|
874
|
+
flattened = {}
|
|
875
|
+
|
|
876
|
+
for key, value in data.items():
|
|
877
|
+
new_key = f"{parent_key}[{key}]" if parent_key else key
|
|
878
|
+
|
|
879
|
+
if isinstance(value, dict):
|
|
880
|
+
# Recursively flatten nested dicts
|
|
881
|
+
flattened.update(self._flatten_form_data(value, new_key))
|
|
882
|
+
elif isinstance(value, list):
|
|
883
|
+
# Flatten arrays with indexed bracket notation
|
|
884
|
+
for i, item in enumerate(value):
|
|
885
|
+
indexed_key = f"{new_key}[{i}]"
|
|
886
|
+
if isinstance(item, dict):
|
|
887
|
+
# Nested dict in array - recurse
|
|
888
|
+
flattened.update(self._flatten_form_data(item, indexed_key))
|
|
889
|
+
elif isinstance(item, list):
|
|
890
|
+
# Nested list in array - recurse
|
|
891
|
+
flattened.update(self._flatten_form_data({str(i): item}, new_key))
|
|
892
|
+
else:
|
|
893
|
+
# Primitive value in array
|
|
894
|
+
flattened[indexed_key] = item
|
|
895
|
+
else:
|
|
896
|
+
# Primitive value - add directly
|
|
897
|
+
flattened[new_key] = value
|
|
898
|
+
|
|
899
|
+
return flattened
|
|
900
|
+
|
|
845
901
|
def _determine_request_format(self, endpoint: EndpointDefinition, body: dict[str, Any] | None) -> dict[str, Any]:
|
|
846
902
|
"""Determine json/data parameters for HTTP request.
|
|
847
903
|
|
|
848
904
|
GraphQL always uses JSON, regardless of content_type setting.
|
|
905
|
+
For form-encoded requests, nested structures are flattened into bracket notation.
|
|
849
906
|
|
|
850
907
|
Args:
|
|
851
908
|
endpoint: Endpoint definition
|
|
@@ -862,7 +919,9 @@ class LocalExecutor:
|
|
|
862
919
|
if is_graphql or endpoint.content_type.value == "application/json":
|
|
863
920
|
return {"json": body}
|
|
864
921
|
elif endpoint.content_type.value == "application/x-www-form-urlencoded":
|
|
865
|
-
|
|
922
|
+
# Flatten nested structures for form encoding
|
|
923
|
+
flattened_body = self._flatten_form_data(body)
|
|
924
|
+
return {"data": flattened_body}
|
|
866
925
|
|
|
867
926
|
return {}
|
|
868
927
|
|
|
@@ -903,12 +962,18 @@ class LocalExecutor:
|
|
|
903
962
|
|
|
904
963
|
return query
|
|
905
964
|
|
|
906
|
-
def _build_graphql_body(
|
|
965
|
+
def _build_graphql_body(
|
|
966
|
+
self,
|
|
967
|
+
graphql_config: dict[str, Any],
|
|
968
|
+
params: dict[str, Any],
|
|
969
|
+
param_defaults: dict[str, Any] | None = None,
|
|
970
|
+
) -> dict[str, Any]:
|
|
907
971
|
"""Build GraphQL request body with variable substitution and field selection.
|
|
908
972
|
|
|
909
973
|
Args:
|
|
910
974
|
graphql_config: GraphQL configuration from x-airbyte-body-type extension
|
|
911
975
|
params: Parameters from execute() call
|
|
976
|
+
param_defaults: Default values for params from query_params_schema
|
|
912
977
|
|
|
913
978
|
Returns:
|
|
914
979
|
GraphQL request body: {"query": "...", "variables": {...}}
|
|
@@ -922,7 +987,9 @@ class LocalExecutor:
|
|
|
922
987
|
|
|
923
988
|
# Substitute variables from params
|
|
924
989
|
if "variables" in graphql_config and graphql_config["variables"]:
|
|
925
|
-
|
|
990
|
+
variables = self._interpolate_variables(graphql_config["variables"], params, param_defaults)
|
|
991
|
+
# Filter out None values (optional fields not provided) - matches REST _extract_body() behavior
|
|
992
|
+
body["variables"] = {k: v for k, v in variables.items() if v is not None}
|
|
926
993
|
|
|
927
994
|
# Add operation name if specified
|
|
928
995
|
if "operationName" in graphql_config:
|
|
@@ -981,7 +1048,12 @@ class LocalExecutor:
|
|
|
981
1048
|
fields_str = " ".join(graphql_fields)
|
|
982
1049
|
return query.replace("{{ fields }}", fields_str)
|
|
983
1050
|
|
|
984
|
-
def _interpolate_variables(
|
|
1051
|
+
def _interpolate_variables(
|
|
1052
|
+
self,
|
|
1053
|
+
variables: dict[str, Any],
|
|
1054
|
+
params: dict[str, Any],
|
|
1055
|
+
param_defaults: dict[str, Any] | None = None,
|
|
1056
|
+
) -> dict[str, Any]:
|
|
985
1057
|
"""Recursively interpolate variables using params.
|
|
986
1058
|
|
|
987
1059
|
Preserves types (doesn't stringify everything).
|
|
@@ -990,15 +1062,18 @@ class LocalExecutor:
|
|
|
990
1062
|
- Direct replacement: "{{ owner }}" → params["owner"] (preserves type)
|
|
991
1063
|
- Nested objects: {"input": {"name": "{{ name }}"}}
|
|
992
1064
|
- Arrays: [{"id": "{{ id }}"}]
|
|
993
|
-
-
|
|
1065
|
+
- Default values: "{{ per_page }}" → param_defaults["per_page"] if not in params
|
|
1066
|
+
- Unsubstituted placeholders: "{{ states }}" → None (for optional params without defaults)
|
|
994
1067
|
|
|
995
1068
|
Args:
|
|
996
1069
|
variables: Variables dict with template placeholders
|
|
997
1070
|
params: Parameters to substitute
|
|
1071
|
+
param_defaults: Default values for params from query_params_schema
|
|
998
1072
|
|
|
999
1073
|
Returns:
|
|
1000
1074
|
Interpolated variables dict with types preserved
|
|
1001
1075
|
"""
|
|
1076
|
+
defaults = param_defaults or {}
|
|
1002
1077
|
|
|
1003
1078
|
def interpolate_value(value: Any) -> Any:
|
|
1004
1079
|
if isinstance(value, str):
|
|
@@ -1012,8 +1087,15 @@ class LocalExecutor:
|
|
|
1012
1087
|
value = value.replace(placeholder, str(param_value))
|
|
1013
1088
|
|
|
1014
1089
|
# Check if any unsubstituted placeholders remain
|
|
1015
|
-
# If so, return None (treats as "not provided" for optional params)
|
|
1016
1090
|
if re.search(r"\{\{\s*\w+\s*\}\}", value):
|
|
1091
|
+
# Extract placeholder name and check for default value
|
|
1092
|
+
match = re.search(r"\{\{\s*(\w+)\s*\}\}", value)
|
|
1093
|
+
if match:
|
|
1094
|
+
param_name = match.group(1)
|
|
1095
|
+
if param_name in defaults:
|
|
1096
|
+
# Use default value (preserves type)
|
|
1097
|
+
return defaults[param_name]
|
|
1098
|
+
# No default found - return None (for optional params)
|
|
1017
1099
|
return None
|
|
1018
1100
|
|
|
1019
1101
|
return value
|
|
@@ -1026,37 +1108,95 @@ class LocalExecutor:
|
|
|
1026
1108
|
|
|
1027
1109
|
return interpolate_value(variables)
|
|
1028
1110
|
|
|
1111
|
+
def _wrap_primitives(self, data: Any) -> dict[str, Any] | list[dict[str, Any]] | None:
|
|
1112
|
+
"""Wrap primitive values in dict format for consistent response structure.
|
|
1113
|
+
|
|
1114
|
+
Transforms primitive API responses into dict format so downstream code
|
|
1115
|
+
can always expect dict-based data structures.
|
|
1116
|
+
|
|
1117
|
+
Args:
|
|
1118
|
+
data: Response data (could be primitive, list, dict, or None)
|
|
1119
|
+
|
|
1120
|
+
Returns:
|
|
1121
|
+
- If data is a primitive (str, int, float, bool): {"value": data}
|
|
1122
|
+
- If data is a list: wraps all non-dict elements as {"value": item}
|
|
1123
|
+
- If data is already a dict or list of dicts: unchanged
|
|
1124
|
+
- If data is None: None
|
|
1125
|
+
|
|
1126
|
+
Examples:
|
|
1127
|
+
>>> executor._wrap_primitives(42)
|
|
1128
|
+
{"value": 42}
|
|
1129
|
+
>>> executor._wrap_primitives([1, 2, 3])
|
|
1130
|
+
[{"value": 1}, {"value": 2}, {"value": 3}]
|
|
1131
|
+
>>> executor._wrap_primitives([1, {"id": 2}, 3])
|
|
1132
|
+
[{"value": 1}, {"id": 2}, {"value": 3}]
|
|
1133
|
+
>>> executor._wrap_primitives([[1, 2], 3])
|
|
1134
|
+
[{"value": [1, 2]}, {"value": 3}]
|
|
1135
|
+
>>> executor._wrap_primitives({"id": 1})
|
|
1136
|
+
{"id": 1} # unchanged
|
|
1137
|
+
"""
|
|
1138
|
+
if data is None:
|
|
1139
|
+
return None
|
|
1140
|
+
|
|
1141
|
+
# Handle primitive scalars
|
|
1142
|
+
if isinstance(data, (bool, str, int, float)):
|
|
1143
|
+
return {"value": data}
|
|
1144
|
+
|
|
1145
|
+
# Handle lists - wrap non-dict elements
|
|
1146
|
+
if isinstance(data, list):
|
|
1147
|
+
if not data:
|
|
1148
|
+
return [] # Empty list unchanged
|
|
1149
|
+
|
|
1150
|
+
wrapped = []
|
|
1151
|
+
for item in data:
|
|
1152
|
+
if isinstance(item, dict):
|
|
1153
|
+
wrapped.append(item)
|
|
1154
|
+
else:
|
|
1155
|
+
wrapped.append({"value": item})
|
|
1156
|
+
return wrapped
|
|
1157
|
+
|
|
1158
|
+
# Dict - return unchanged
|
|
1159
|
+
if isinstance(data, dict):
|
|
1160
|
+
return data
|
|
1161
|
+
|
|
1162
|
+
# Unknown type - wrap for safety
|
|
1163
|
+
return {"value": data}
|
|
1164
|
+
|
|
1029
1165
|
def _extract_records(
|
|
1030
1166
|
self,
|
|
1031
|
-
response_data:
|
|
1167
|
+
response_data: Any,
|
|
1032
1168
|
endpoint: EndpointDefinition,
|
|
1033
|
-
) -> dict[str, Any] | list[Any] | None:
|
|
1169
|
+
) -> dict[str, Any] | list[dict[str, Any]] | None:
|
|
1034
1170
|
"""Extract records from response using record extractor.
|
|
1035
1171
|
|
|
1036
1172
|
Type inference based on action:
|
|
1037
1173
|
- list, search: Returns array ([] if not found)
|
|
1038
1174
|
- get, create, update, delete: Returns single record (None if not found)
|
|
1039
1175
|
|
|
1176
|
+
Automatically wraps primitive values (int, str, float, bool) in {"value": primitive}
|
|
1177
|
+
format to ensure consistent dict-based responses for downstream code.
|
|
1178
|
+
|
|
1040
1179
|
Args:
|
|
1041
|
-
response_data: Full API response
|
|
1180
|
+
response_data: Full API response (can be dict, list, primitive, or None)
|
|
1042
1181
|
endpoint: Endpoint with optional record extractor and action
|
|
1043
1182
|
|
|
1044
1183
|
Returns:
|
|
1045
1184
|
- Extracted data if extractor configured and path found
|
|
1046
1185
|
- [] or None if path not found (based on action)
|
|
1047
1186
|
- Original response if no extractor configured or on error
|
|
1187
|
+
- Primitives are wrapped as {"value": primitive}
|
|
1048
1188
|
"""
|
|
1049
1189
|
# Check if endpoint has record extractor
|
|
1050
1190
|
extractor = endpoint.record_extractor
|
|
1051
1191
|
if not extractor:
|
|
1052
|
-
return response_data
|
|
1192
|
+
return self._wrap_primitives(response_data)
|
|
1053
1193
|
|
|
1054
1194
|
# Determine if this action returns array or single record
|
|
1055
1195
|
action = endpoint.action
|
|
1056
1196
|
if not action:
|
|
1057
|
-
return response_data
|
|
1197
|
+
return self._wrap_primitives(response_data)
|
|
1058
1198
|
|
|
1059
|
-
is_array_action = action in (Action.LIST, Action.
|
|
1199
|
+
is_array_action = action in (Action.LIST, Action.API_SEARCH)
|
|
1060
1200
|
|
|
1061
1201
|
try:
|
|
1062
1202
|
# Parse and apply JSONPath expression
|
|
@@ -1067,30 +1207,39 @@ class LocalExecutor:
|
|
|
1067
1207
|
# Path not found - return empty based on action
|
|
1068
1208
|
return [] if is_array_action else None
|
|
1069
1209
|
|
|
1070
|
-
# Return extracted data
|
|
1210
|
+
# Return extracted data with primitive wrapping
|
|
1071
1211
|
if is_array_action:
|
|
1072
1212
|
# For array actions, return the array (or list of matches)
|
|
1073
|
-
|
|
1213
|
+
result = matches[0] if len(matches) == 1 else matches
|
|
1074
1214
|
else:
|
|
1075
1215
|
# For single record actions, return first match
|
|
1076
|
-
|
|
1216
|
+
result = matches[0]
|
|
1217
|
+
|
|
1218
|
+
return self._wrap_primitives(result)
|
|
1077
1219
|
|
|
1078
1220
|
except Exception as e:
|
|
1079
1221
|
logging.warning(f"Failed to apply record extractor '{extractor}': {e}. Returning original response.")
|
|
1080
|
-
return response_data
|
|
1222
|
+
return self._wrap_primitives(response_data)
|
|
1081
1223
|
|
|
1082
1224
|
def _extract_metadata(
|
|
1083
1225
|
self,
|
|
1084
1226
|
response_data: dict[str, Any],
|
|
1227
|
+
response_headers: dict[str, str],
|
|
1085
1228
|
endpoint: EndpointDefinition,
|
|
1086
1229
|
) -> dict[str, Any] | None:
|
|
1087
1230
|
"""Extract metadata from response using meta extractor.
|
|
1088
1231
|
|
|
1089
|
-
Each field in meta_extractor dict is independently extracted using JSONPath
|
|
1232
|
+
Each field in meta_extractor dict is independently extracted using JSONPath
|
|
1233
|
+
for body extraction, or special prefixes for header extraction:
|
|
1234
|
+
- @link.{rel}: Extract URL from RFC 5988 Link header by rel type
|
|
1235
|
+
- @header.{name}: Extract raw header value by header name
|
|
1236
|
+
- Otherwise: JSONPath expression for body extraction
|
|
1237
|
+
|
|
1090
1238
|
Missing or invalid paths result in None for that field (no crash).
|
|
1091
1239
|
|
|
1092
1240
|
Args:
|
|
1093
1241
|
response_data: Full API response (before record extraction)
|
|
1242
|
+
response_headers: HTTP response headers
|
|
1094
1243
|
endpoint: Endpoint with optional meta extractor configuration
|
|
1095
1244
|
|
|
1096
1245
|
Returns:
|
|
@@ -1101,11 +1250,15 @@ class LocalExecutor:
|
|
|
1101
1250
|
Example:
|
|
1102
1251
|
meta_extractor = {
|
|
1103
1252
|
"pagination": "$.records",
|
|
1104
|
-
"request_id": "$.requestId"
|
|
1253
|
+
"request_id": "$.requestId",
|
|
1254
|
+
"next_page_url": "@link.next",
|
|
1255
|
+
"rate_limit": "@header.X-RateLimit-Remaining"
|
|
1105
1256
|
}
|
|
1106
1257
|
Returns: {
|
|
1107
1258
|
"pagination": {"cursor": "abc", "total": 100},
|
|
1108
|
-
"request_id": "xyz123"
|
|
1259
|
+
"request_id": "xyz123",
|
|
1260
|
+
"next_page_url": "https://api.example.com/data?cursor=abc",
|
|
1261
|
+
"rate_limit": "99"
|
|
1109
1262
|
}
|
|
1110
1263
|
"""
|
|
1111
1264
|
# Check if endpoint has meta extractor
|
|
@@ -1115,26 +1268,96 @@ class LocalExecutor:
|
|
|
1115
1268
|
extracted_meta: dict[str, Any] = {}
|
|
1116
1269
|
|
|
1117
1270
|
# Extract each field independently
|
|
1118
|
-
for field_name,
|
|
1271
|
+
for field_name, extractor_expr in endpoint.meta_extractor.items():
|
|
1119
1272
|
try:
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
#
|
|
1126
|
-
|
|
1273
|
+
if extractor_expr.startswith("@link."):
|
|
1274
|
+
# RFC 5988 Link header extraction
|
|
1275
|
+
rel = extractor_expr[6:]
|
|
1276
|
+
extracted_meta[field_name] = self._extract_link_url(response_headers, rel)
|
|
1277
|
+
elif extractor_expr.startswith("@header."):
|
|
1278
|
+
# Raw header value extraction (case-insensitive lookup)
|
|
1279
|
+
header_name = extractor_expr[8:]
|
|
1280
|
+
extracted_meta[field_name] = self._get_header_value(response_headers, header_name)
|
|
1127
1281
|
else:
|
|
1128
|
-
#
|
|
1129
|
-
|
|
1282
|
+
# JSONPath body extraction
|
|
1283
|
+
jsonpath_expr = parse_jsonpath(extractor_expr)
|
|
1284
|
+
matches = [match.value for match in jsonpath_expr.find(response_data)]
|
|
1285
|
+
|
|
1286
|
+
if matches:
|
|
1287
|
+
# Return first match (most common case)
|
|
1288
|
+
extracted_meta[field_name] = matches[0]
|
|
1289
|
+
else:
|
|
1290
|
+
# Path not found - set to None
|
|
1291
|
+
extracted_meta[field_name] = None
|
|
1130
1292
|
|
|
1131
1293
|
except Exception as e:
|
|
1132
1294
|
# Log error but continue with other fields
|
|
1133
|
-
logging.warning(f"Failed to apply meta extractor for field '{field_name}' with
|
|
1295
|
+
logging.warning(f"Failed to apply meta extractor for field '{field_name}' with expression '{extractor_expr}': {e}. Setting to None.")
|
|
1134
1296
|
extracted_meta[field_name] = None
|
|
1135
1297
|
|
|
1136
1298
|
return extracted_meta
|
|
1137
1299
|
|
|
1300
|
+
@staticmethod
|
|
1301
|
+
def _extract_link_url(headers: dict[str, str], rel: str) -> str | None:
|
|
1302
|
+
"""Extract URL from RFC 5988 Link header by rel type.
|
|
1303
|
+
|
|
1304
|
+
Parses Link header format: <url>; param1="value1"; rel="next"; param2="value2"
|
|
1305
|
+
|
|
1306
|
+
Supports:
|
|
1307
|
+
- Multiple parameters per link in any order
|
|
1308
|
+
- Both quoted and unquoted rel values
|
|
1309
|
+
- Multiple links separated by commas
|
|
1310
|
+
|
|
1311
|
+
Args:
|
|
1312
|
+
headers: Response headers dict
|
|
1313
|
+
rel: The rel type to extract (e.g., "next", "prev", "first", "last")
|
|
1314
|
+
|
|
1315
|
+
Returns:
|
|
1316
|
+
The URL for the specified rel type, or None if not found
|
|
1317
|
+
"""
|
|
1318
|
+
link_header = headers.get("Link") or headers.get("link", "")
|
|
1319
|
+
if not link_header:
|
|
1320
|
+
return None
|
|
1321
|
+
|
|
1322
|
+
for link_segment in re.split(r",(?=\s*<)", link_header):
|
|
1323
|
+
link_segment = link_segment.strip()
|
|
1324
|
+
|
|
1325
|
+
url_match = re.match(r"<([^>]+)>", link_segment)
|
|
1326
|
+
if not url_match:
|
|
1327
|
+
continue
|
|
1328
|
+
|
|
1329
|
+
url = url_match.group(1)
|
|
1330
|
+
params_str = link_segment[url_match.end() :]
|
|
1331
|
+
|
|
1332
|
+
rel_match = re.search(r';\s*rel="?([^";,]+)"?', params_str, re.IGNORECASE)
|
|
1333
|
+
if rel_match and rel_match.group(1).strip() == rel:
|
|
1334
|
+
return url
|
|
1335
|
+
|
|
1336
|
+
return None
|
|
1337
|
+
|
|
1338
|
+
@staticmethod
|
|
1339
|
+
def _get_header_value(headers: dict[str, str], header_name: str) -> str | None:
|
|
1340
|
+
"""Get header value with case-insensitive lookup.
|
|
1341
|
+
|
|
1342
|
+
Args:
|
|
1343
|
+
headers: Response headers dict
|
|
1344
|
+
header_name: Header name to look up
|
|
1345
|
+
|
|
1346
|
+
Returns:
|
|
1347
|
+
Header value or None if not found
|
|
1348
|
+
"""
|
|
1349
|
+
# Try exact match first
|
|
1350
|
+
if header_name in headers:
|
|
1351
|
+
return headers[header_name]
|
|
1352
|
+
|
|
1353
|
+
# Case-insensitive lookup
|
|
1354
|
+
header_name_lower = header_name.lower()
|
|
1355
|
+
for key, value in headers.items():
|
|
1356
|
+
if key.lower() == header_name_lower:
|
|
1357
|
+
return value
|
|
1358
|
+
|
|
1359
|
+
return None
|
|
1360
|
+
|
|
1138
1361
|
def _validate_required_body_fields(self, endpoint: Any, params: dict[str, Any], action: Action, entity: str) -> None:
|
|
1139
1362
|
"""Validate that required body fields are present for CREATE/UPDATE operations.
|
|
1140
1363
|
|
|
@@ -1151,17 +1374,14 @@ class LocalExecutor:
|
|
|
1151
1374
|
if action not in (Action.CREATE, Action.UPDATE):
|
|
1152
1375
|
return
|
|
1153
1376
|
|
|
1154
|
-
#
|
|
1155
|
-
|
|
1377
|
+
# Get the request schema to find truly required fields
|
|
1378
|
+
request_schema = endpoint.request_schema
|
|
1379
|
+
if not request_schema:
|
|
1156
1380
|
return
|
|
1157
1381
|
|
|
1158
|
-
#
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
missing_fields = []
|
|
1162
|
-
for field in endpoint.body_fields:
|
|
1163
|
-
if field not in params:
|
|
1164
|
-
missing_fields.append(field)
|
|
1382
|
+
# Only validate fields explicitly marked as required in the schema
|
|
1383
|
+
required_fields = request_schema.get("required", [])
|
|
1384
|
+
missing_fields = [field for field in required_fields if field not in params]
|
|
1165
1385
|
|
|
1166
1386
|
if missing_fields:
|
|
1167
1387
|
raise MissingParameterError(
|
|
@@ -1189,7 +1409,7 @@ class LocalExecutor:
|
|
|
1189
1409
|
|
|
1190
1410
|
|
|
1191
1411
|
class _StandardOperationHandler:
|
|
1192
|
-
"""Handler for standard REST operations (GET, LIST, CREATE, UPDATE, DELETE,
|
|
1412
|
+
"""Handler for standard REST operations (GET, LIST, CREATE, UPDATE, DELETE, API_SEARCH, AUTHORIZE)."""
|
|
1193
1413
|
|
|
1194
1414
|
def __init__(self, context: _OperationContext):
|
|
1195
1415
|
self.ctx = context
|
|
@@ -1202,7 +1422,7 @@ class _StandardOperationHandler:
|
|
|
1202
1422
|
Action.CREATE,
|
|
1203
1423
|
Action.UPDATE,
|
|
1204
1424
|
Action.DELETE,
|
|
1205
|
-
Action.
|
|
1425
|
+
Action.API_SEARCH,
|
|
1206
1426
|
Action.AUTHORIZE,
|
|
1207
1427
|
}
|
|
1208
1428
|
|
|
@@ -1265,7 +1485,7 @@ class _StandardOperationHandler:
|
|
|
1265
1485
|
request_kwargs = self.ctx.determine_request_format(endpoint, body)
|
|
1266
1486
|
|
|
1267
1487
|
# Execute async HTTP request
|
|
1268
|
-
|
|
1488
|
+
response_data, response_headers = await self.ctx.http_client.request(
|
|
1269
1489
|
method=endpoint.method,
|
|
1270
1490
|
path=path,
|
|
1271
1491
|
params=query_params if query_params else None,
|
|
@@ -1274,10 +1494,10 @@ class _StandardOperationHandler:
|
|
|
1274
1494
|
)
|
|
1275
1495
|
|
|
1276
1496
|
# Extract metadata from original response (before record extraction)
|
|
1277
|
-
metadata = self.ctx.executor._extract_metadata(
|
|
1497
|
+
metadata = self.ctx.executor._extract_metadata(response_data, response_headers, endpoint)
|
|
1278
1498
|
|
|
1279
1499
|
# Extract records if extractor configured
|
|
1280
|
-
response = self.ctx.extract_records(
|
|
1500
|
+
response = self.ctx.extract_records(response_data, endpoint)
|
|
1281
1501
|
|
|
1282
1502
|
# Assume success with 200 status code if no exception raised
|
|
1283
1503
|
status_code = 200
|
|
@@ -1403,7 +1623,7 @@ class _DownloadOperationHandler:
|
|
|
1403
1623
|
request_format = self.ctx.determine_request_format(operation, request_body)
|
|
1404
1624
|
self.ctx.validate_required_body_fields(operation, params, action, entity)
|
|
1405
1625
|
|
|
1406
|
-
metadata_response = await self.ctx.http_client.request(
|
|
1626
|
+
metadata_response, _ = await self.ctx.http_client.request(
|
|
1407
1627
|
method=operation.method,
|
|
1408
1628
|
path=path,
|
|
1409
1629
|
params=query_params,
|
|
@@ -1418,7 +1638,7 @@ class _DownloadOperationHandler:
|
|
|
1418
1638
|
)
|
|
1419
1639
|
|
|
1420
1640
|
# Step 3: Stream file from extracted URL
|
|
1421
|
-
file_response = await self.ctx.http_client.request(
|
|
1641
|
+
file_response, _ = await self.ctx.http_client.request(
|
|
1422
1642
|
method="GET",
|
|
1423
1643
|
path=file_url,
|
|
1424
1644
|
headers=headers,
|
|
@@ -1426,7 +1646,7 @@ class _DownloadOperationHandler:
|
|
|
1426
1646
|
)
|
|
1427
1647
|
else:
|
|
1428
1648
|
# One-step direct download: stream file directly from endpoint
|
|
1429
|
-
file_response = await self.ctx.http_client.request(
|
|
1649
|
+
file_response, _ = await self.ctx.http_client.request(
|
|
1430
1650
|
method=operation.method,
|
|
1431
1651
|
path=path,
|
|
1432
1652
|
params=query_params,
|
|
@@ -159,6 +159,38 @@ Example:
|
|
|
159
159
|
```
|
|
160
160
|
"""
|
|
161
161
|
|
|
162
|
+
AIRBYTE_STREAM_NAME = "x-airbyte-stream-name"
|
|
163
|
+
"""
|
|
164
|
+
Extension: x-airbyte-stream-name
|
|
165
|
+
Location: Schema object (in components.schemas)
|
|
166
|
+
Type: string
|
|
167
|
+
Required: No
|
|
168
|
+
|
|
169
|
+
Description:
|
|
170
|
+
Specifies the Airbyte stream name for cache lookup purposes. This maps the entity
|
|
171
|
+
to the corresponding Airbyte stream, enabling cache-based data retrieval. When
|
|
172
|
+
specified, the EntityDefinition.stream_name field will be populated with this value.
|
|
173
|
+
|
|
174
|
+
This extension is placed on Schema objects alongside x-airbyte-entity-name, following
|
|
175
|
+
the same pattern. The stream name is an entity-level property (not operation-level)
|
|
176
|
+
since an entity maps to exactly one Airbyte stream.
|
|
177
|
+
|
|
178
|
+
Example:
|
|
179
|
+
```yaml
|
|
180
|
+
components:
|
|
181
|
+
schemas:
|
|
182
|
+
Customer:
|
|
183
|
+
type: object
|
|
184
|
+
x-airbyte-entity-name: customers
|
|
185
|
+
x-airbyte-stream-name: customers
|
|
186
|
+
properties:
|
|
187
|
+
id:
|
|
188
|
+
type: string
|
|
189
|
+
name:
|
|
190
|
+
type: string
|
|
191
|
+
```
|
|
192
|
+
"""
|
|
193
|
+
|
|
162
194
|
AIRBYTE_TOKEN_PATH = "x-airbyte-token-path"
|
|
163
195
|
"""
|
|
164
196
|
Extension: x-airbyte-token-path
|
|
@@ -495,8 +527,8 @@ class ActionType(str, Enum):
|
|
|
495
527
|
DELETE = "delete"
|
|
496
528
|
"""Delete a record"""
|
|
497
529
|
|
|
498
|
-
|
|
499
|
-
"""Search for records matching specific query criteria"""
|
|
530
|
+
API_SEARCH = "api_search"
|
|
531
|
+
"""Search for records matching specific query criteria via API"""
|
|
500
532
|
|
|
501
533
|
DOWNLOAD = "download"
|
|
502
534
|
"""Download file content from a URL specified in the metadata response"""
|
|
@@ -514,7 +546,7 @@ class BodyType(str, Enum):
|
|
|
514
546
|
|
|
515
547
|
|
|
516
548
|
# Type alias for use in Pydantic models
|
|
517
|
-
ActionTypeLiteral = Literal["get", "list", "create", "update", "delete", "
|
|
549
|
+
ActionTypeLiteral = Literal["get", "list", "create", "update", "delete", "api_search", "download"]
|
|
518
550
|
|
|
519
551
|
|
|
520
552
|
# =============================================================================
|
|
@@ -548,6 +580,7 @@ def get_all_extension_names() -> list[str]:
|
|
|
548
580
|
AIRBYTE_ENTITY,
|
|
549
581
|
AIRBYTE_ACTION,
|
|
550
582
|
AIRBYTE_ENTITY_NAME,
|
|
583
|
+
AIRBYTE_STREAM_NAME,
|
|
551
584
|
AIRBYTE_TOKEN_PATH,
|
|
552
585
|
AIRBYTE_BODY_TYPE,
|
|
553
586
|
AIRBYTE_PATH_OVERRIDE,
|
|
@@ -594,6 +627,12 @@ EXTENSION_REGISTRY = {
|
|
|
594
627
|
"required": False,
|
|
595
628
|
"description": "Links schema to an entity/stream",
|
|
596
629
|
},
|
|
630
|
+
AIRBYTE_STREAM_NAME: {
|
|
631
|
+
"location": "schema",
|
|
632
|
+
"type": "string",
|
|
633
|
+
"required": False,
|
|
634
|
+
"description": "Maps entity to Airbyte stream for cache lookup",
|
|
635
|
+
},
|
|
597
636
|
AIRBYTE_TOKEN_PATH: {
|
|
598
637
|
"location": "securityScheme",
|
|
599
638
|
"type": "string",
|
|
@@ -627,8 +666,7 @@ EXTENSION_REGISTRY = {
|
|
|
627
666
|
"type": "dict[str, str]",
|
|
628
667
|
"required": False,
|
|
629
668
|
"description": (
|
|
630
|
-
"Dictionary mapping field names to JSONPath expressions for extracting metadata "
|
|
631
|
-
"(pagination, request IDs, etc.) from response envelopes"
|
|
669
|
+
"Dictionary mapping field names to JSONPath expressions for extracting metadata (pagination, request IDs, etc.) from response envelopes"
|
|
632
670
|
),
|
|
633
671
|
},
|
|
634
672
|
AIRBYTE_FILE_URL: {
|