airbyte-cdk 6.60.0.post18.dev16485197212__py3-none-any.whl → 6.60.0.post20.dev16487461641__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/cli/source_declarative_manifest/_run.py +28 -32
- airbyte_cdk/config_observation.py +2 -2
- airbyte_cdk/connector_builder/main.py +5 -7
- airbyte_cdk/destinations/destination.py +4 -3
- airbyte_cdk/entrypoint.py +8 -23
- airbyte_cdk/logger.py +2 -2
- airbyte_cdk/models/__init__.py +2 -3
- airbyte_cdk/models/airbyte_protocol_serializers.py +72 -7
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +2 -2
- airbyte_cdk/sources/streams/http/http_client.py +5 -7
- airbyte_cdk/test/catalog_builder.py +1 -2
- airbyte_cdk/test/entrypoint_wrapper.py +3 -2
- airbyte_cdk/utils/traced_exception.py +2 -2
- {airbyte_cdk-6.60.0.post18.dev16485197212.dist-info → airbyte_cdk-6.60.0.post20.dev16487461641.dist-info}/METADATA +1 -1
- {airbyte_cdk-6.60.0.post18.dev16485197212.dist-info → airbyte_cdk-6.60.0.post20.dev16487461641.dist-info}/RECORD +19 -19
- {airbyte_cdk-6.60.0.post18.dev16485197212.dist-info → airbyte_cdk-6.60.0.post20.dev16487461641.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.60.0.post18.dev16485197212.dist-info → airbyte_cdk-6.60.0.post20.dev16487461641.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.60.0.post18.dev16485197212.dist-info → airbyte_cdk-6.60.0.post20.dev16487461641.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.60.0.post18.dev16485197212.dist-info → airbyte_cdk-6.60.0.post20.dev16487461641.dist-info}/entry_points.txt +0 -0
@@ -32,13 +32,13 @@ from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
|
|
32
32
|
from airbyte_cdk.models import (
|
33
33
|
AirbyteErrorTraceMessage,
|
34
34
|
AirbyteMessage,
|
35
|
-
AirbyteMessageSerializer,
|
36
35
|
AirbyteStateMessage,
|
37
36
|
AirbyteTraceMessage,
|
38
37
|
ConfiguredAirbyteCatalog,
|
39
38
|
ConnectorSpecificationSerializer,
|
40
39
|
TraceType,
|
41
40
|
Type,
|
41
|
+
ab_message_to_string,
|
42
42
|
)
|
43
43
|
from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
|
44
44
|
ConcurrentDeclarativeSource,
|
@@ -105,21 +105,19 @@ def _get_local_yaml_source(args: list[str]) -> SourceLocalYaml:
|
|
105
105
|
)
|
106
106
|
except Exception as error:
|
107
107
|
print(
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
error
|
116
|
-
|
117
|
-
stack_trace=traceback.format_exc(),
|
118
|
-
),
|
108
|
+
ab_message_to_string(
|
109
|
+
AirbyteMessage(
|
110
|
+
type=Type.TRACE,
|
111
|
+
trace=AirbyteTraceMessage(
|
112
|
+
type=TraceType.ERROR,
|
113
|
+
emitted_at=ab_datetime_now().to_epoch_millis(),
|
114
|
+
error=AirbyteErrorTraceMessage(
|
115
|
+
message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
|
116
|
+
stack_trace=traceback.format_exc(),
|
119
117
|
),
|
120
|
-
)
|
121
|
-
)
|
122
|
-
)
|
118
|
+
),
|
119
|
+
),
|
120
|
+
)
|
123
121
|
)
|
124
122
|
raise error
|
125
123
|
|
@@ -153,7 +151,7 @@ def handle_remote_manifest_command(args: list[str]) -> None:
|
|
153
151
|
spec = ConnectorSpecificationSerializer.load(spec_obj)
|
154
152
|
|
155
153
|
message = AirbyteMessage(type=Type.SPEC, spec=spec)
|
156
|
-
print(
|
154
|
+
print(ab_message_to_string(message))
|
157
155
|
else:
|
158
156
|
source = create_declarative_source(args)
|
159
157
|
launch(
|
@@ -215,21 +213,19 @@ def create_declarative_source(
|
|
215
213
|
)
|
216
214
|
except Exception as error:
|
217
215
|
print(
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
error
|
226
|
-
|
227
|
-
stack_trace=traceback.format_exc(),
|
228
|
-
),
|
216
|
+
ab_message_to_string(
|
217
|
+
AirbyteMessage(
|
218
|
+
type=Type.TRACE,
|
219
|
+
trace=AirbyteTraceMessage(
|
220
|
+
type=TraceType.ERROR,
|
221
|
+
emitted_at=ab_datetime_now().to_epoch_millis(),
|
222
|
+
error=AirbyteErrorTraceMessage(
|
223
|
+
message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
|
224
|
+
stack_trace=traceback.format_exc(),
|
229
225
|
),
|
230
|
-
)
|
231
|
-
)
|
232
|
-
)
|
226
|
+
),
|
227
|
+
),
|
228
|
+
),
|
233
229
|
)
|
234
230
|
raise error
|
235
231
|
|
@@ -298,10 +294,10 @@ def _register_components_from_file(filepath: str) -> None:
|
|
298
294
|
spec.loader.exec_module(module)
|
299
295
|
|
300
296
|
|
301
|
-
def run() -> None:
|
297
|
+
def run(args: list[str] | None = None) -> None:
|
302
298
|
"""Run the `source-declarative-manifest` CLI.
|
303
299
|
|
304
300
|
Args are detected from the command line, and the appropriate command is executed.
|
305
301
|
"""
|
306
|
-
args
|
302
|
+
args = args or sys.argv[1:]
|
307
303
|
handle_command(args)
|
@@ -16,9 +16,9 @@ from airbyte_cdk.models import (
|
|
16
16
|
AirbyteControlConnectorConfigMessage,
|
17
17
|
AirbyteControlMessage,
|
18
18
|
AirbyteMessage,
|
19
|
-
AirbyteMessageSerializer,
|
20
19
|
OrchestratorType,
|
21
20
|
Type,
|
21
|
+
ab_message_to_string,
|
22
22
|
)
|
23
23
|
|
24
24
|
|
@@ -92,7 +92,7 @@ def emit_configuration_as_airbyte_control_message(config: MutableMapping[str, An
|
|
92
92
|
See the airbyte_cdk.sources.message package
|
93
93
|
"""
|
94
94
|
airbyte_message = create_connector_config_control_message(config)
|
95
|
-
print(
|
95
|
+
print(ab_message_to_string(airbyte_message))
|
96
96
|
|
97
97
|
|
98
98
|
def create_connector_config_control_message(config: MutableMapping[str, Any]) -> AirbyteMessage:
|
@@ -20,10 +20,10 @@ from airbyte_cdk.connector_builder.connector_builder_handler import (
|
|
20
20
|
from airbyte_cdk.entrypoint import AirbyteEntrypoint
|
21
21
|
from airbyte_cdk.models import (
|
22
22
|
AirbyteMessage,
|
23
|
-
AirbyteMessageSerializer,
|
24
23
|
AirbyteStateMessage,
|
25
24
|
ConfiguredAirbyteCatalog,
|
26
25
|
ConfiguredAirbyteCatalogSerializer,
|
26
|
+
ab_message_to_string,
|
27
27
|
)
|
28
28
|
from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
|
29
29
|
from airbyte_cdk.sources.source import Source
|
@@ -92,11 +92,9 @@ def handle_request(args: List[str]) -> str:
|
|
92
92
|
command, config, catalog, state = get_config_and_catalog_from_args(args)
|
93
93
|
limits = get_limits(config)
|
94
94
|
source = create_source(config, limits)
|
95
|
-
return
|
96
|
-
|
97
|
-
|
98
|
-
)
|
99
|
-
).decode() # type: ignore[no-any-return] # Serializer.dump() always returns AirbyteMessage
|
95
|
+
return ab_message_to_string(
|
96
|
+
handle_connector_builder_request(source, command, config, catalog, state, limits)
|
97
|
+
)
|
100
98
|
|
101
99
|
|
102
100
|
if __name__ == "__main__":
|
@@ -107,4 +105,4 @@ if __name__ == "__main__":
|
|
107
105
|
exc, message=f"Error handling request: {str(exc)}"
|
108
106
|
)
|
109
107
|
m = error.as_airbyte_message()
|
110
|
-
print(
|
108
|
+
print(ab_message_to_string(m))
|
@@ -15,10 +15,11 @@ from airbyte_cdk.connector import Connector
|
|
15
15
|
from airbyte_cdk.exception_handler import init_uncaught_exception_handler
|
16
16
|
from airbyte_cdk.models import (
|
17
17
|
AirbyteMessage,
|
18
|
-
AirbyteMessageSerializer,
|
19
18
|
ConfiguredAirbyteCatalog,
|
20
19
|
ConfiguredAirbyteCatalogSerializer,
|
21
20
|
Type,
|
21
|
+
ab_message_from_string,
|
22
|
+
ab_message_to_string,
|
22
23
|
)
|
23
24
|
from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit
|
24
25
|
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
@@ -46,7 +47,7 @@ class Destination(Connector, ABC):
|
|
46
47
|
"""Reads from stdin, converting to Airbyte messages"""
|
47
48
|
for line in input_stream:
|
48
49
|
try:
|
49
|
-
yield
|
50
|
+
yield ab_message_from_string(line)
|
50
51
|
except orjson.JSONDecodeError:
|
51
52
|
logger.info(
|
52
53
|
f"ignoring input which can't be deserialized as Airbyte Message: {line}"
|
@@ -151,4 +152,4 @@ class Destination(Connector, ABC):
|
|
151
152
|
parsed_args = self.parse_args(args)
|
152
153
|
output_messages = self.run_cmd(parsed_args)
|
153
154
|
for message in output_messages:
|
154
|
-
print(
|
155
|
+
print(ab_message_to_string(message))
|
airbyte_cdk/entrypoint.py
CHANGED
@@ -26,12 +26,12 @@ from airbyte_cdk.logger import PRINT_BUFFER, init_logger
|
|
26
26
|
from airbyte_cdk.models import (
|
27
27
|
AirbyteConnectionStatus,
|
28
28
|
AirbyteMessage,
|
29
|
-
AirbyteMessageSerializer,
|
30
29
|
AirbyteStateStats,
|
31
30
|
ConnectorSpecification,
|
32
31
|
FailureType,
|
33
32
|
Status,
|
34
33
|
Type,
|
34
|
+
ab_message_to_string,
|
35
35
|
)
|
36
36
|
from airbyte_cdk.sources import Source
|
37
37
|
from airbyte_cdk.sources.connector_state_manager import HashableStreamDescriptor
|
@@ -47,7 +47,6 @@ logger = init_logger("airbyte")
|
|
47
47
|
|
48
48
|
VALID_URL_SCHEMES = ["https"]
|
49
49
|
CLOUD_DEPLOYMENT_MODE = "cloud"
|
50
|
-
_HAS_LOGGED_FOR_SERIALIZATION_ERROR = False
|
51
50
|
|
52
51
|
|
53
52
|
class AirbyteEntrypoint(object):
|
@@ -178,26 +177,26 @@ class AirbyteEntrypoint(object):
|
|
178
177
|
if cmd == "spec":
|
179
178
|
message = AirbyteMessage(type=Type.SPEC, spec=source_spec)
|
180
179
|
yield from [
|
181
|
-
|
180
|
+
ab_message_to_string(queued_message)
|
182
181
|
for queued_message in self._emit_queued_messages(self.source)
|
183
182
|
]
|
184
|
-
yield
|
183
|
+
yield ab_message_to_string(message)
|
185
184
|
else:
|
186
185
|
raw_config = self.source.read_config(parsed_args.config)
|
187
186
|
config = self.source.configure(raw_config, temp_dir)
|
188
187
|
|
189
188
|
yield from [
|
190
|
-
|
189
|
+
ab_message_to_string(queued_message)
|
191
190
|
for queued_message in self._emit_queued_messages(self.source)
|
192
191
|
]
|
193
192
|
if cmd == "check":
|
194
193
|
yield from map(
|
195
|
-
|
194
|
+
ab_message_to_string,
|
196
195
|
self.check(source_spec, config),
|
197
196
|
)
|
198
197
|
elif cmd == "discover":
|
199
198
|
yield from map(
|
200
|
-
|
199
|
+
ab_message_to_string,
|
201
200
|
self.discover(source_spec, config),
|
202
201
|
)
|
203
202
|
elif cmd == "read":
|
@@ -205,14 +204,14 @@ class AirbyteEntrypoint(object):
|
|
205
204
|
state = self.source.read_state(parsed_args.state)
|
206
205
|
|
207
206
|
yield from map(
|
208
|
-
|
207
|
+
ab_message_to_string,
|
209
208
|
self.read(source_spec, config, config_catalog, state),
|
210
209
|
)
|
211
210
|
else:
|
212
211
|
raise Exception("Unexpected command " + cmd)
|
213
212
|
finally:
|
214
213
|
yield from [
|
215
|
-
|
214
|
+
ab_message_to_string(queued_message)
|
216
215
|
for queued_message in self._emit_queued_messages(self.source)
|
217
216
|
]
|
218
217
|
|
@@ -327,20 +326,6 @@ class AirbyteEntrypoint(object):
|
|
327
326
|
config_secrets = get_secrets(connection_specification, config)
|
328
327
|
update_secrets(config_secrets)
|
329
328
|
|
330
|
-
@staticmethod
|
331
|
-
def airbyte_message_to_string(airbyte_message: AirbyteMessage) -> str:
|
332
|
-
global _HAS_LOGGED_FOR_SERIALIZATION_ERROR
|
333
|
-
serialized_message = AirbyteMessageSerializer.dump(airbyte_message)
|
334
|
-
try:
|
335
|
-
return orjson.dumps(serialized_message).decode()
|
336
|
-
except Exception as exception:
|
337
|
-
if not _HAS_LOGGED_FOR_SERIALIZATION_ERROR:
|
338
|
-
logger.warning(
|
339
|
-
f"There was an error during the serialization of an AirbyteMessage: `{exception}`. This might impact the sync performances."
|
340
|
-
)
|
341
|
-
_HAS_LOGGED_FOR_SERIALIZATION_ERROR = True
|
342
|
-
return json.dumps(serialized_message)
|
343
|
-
|
344
329
|
@classmethod
|
345
330
|
def extract_state(cls, args: List[str]) -> Optional[Any]:
|
346
331
|
parsed_args = cls.parse_args(args)
|
airbyte_cdk/logger.py
CHANGED
@@ -12,9 +12,9 @@ import orjson
|
|
12
12
|
from airbyte_cdk.models import (
|
13
13
|
AirbyteLogMessage,
|
14
14
|
AirbyteMessage,
|
15
|
-
AirbyteMessageSerializer,
|
16
15
|
Level,
|
17
16
|
Type,
|
17
|
+
ab_message_to_string,
|
18
18
|
)
|
19
19
|
from airbyte_cdk.utils import PrintBuffer
|
20
20
|
from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
|
@@ -81,7 +81,7 @@ class AirbyteLogFormatter(logging.Formatter):
|
|
81
81
|
log_message = AirbyteMessage(
|
82
82
|
type=Type.LOG, log=AirbyteLogMessage(level=airbyte_level, message=message)
|
83
83
|
)
|
84
|
-
return
|
84
|
+
return ab_message_to_string(log_message)
|
85
85
|
|
86
86
|
@staticmethod
|
87
87
|
def extract_extra_args_from_record(record: logging.LogRecord) -> Mapping[str, Any]:
|
airbyte_cdk/models/__init__.py
CHANGED
@@ -50,12 +50,11 @@ from .airbyte_protocol import (
|
|
50
50
|
Type,
|
51
51
|
)
|
52
52
|
from .airbyte_protocol_serializers import (
|
53
|
-
AirbyteMessageSerializer,
|
54
53
|
AirbyteStateMessageSerializer,
|
55
|
-
AirbyteStreamStateSerializer,
|
56
54
|
ConfiguredAirbyteCatalogSerializer,
|
57
|
-
ConfiguredAirbyteStreamSerializer,
|
58
55
|
ConnectorSpecificationSerializer,
|
56
|
+
ab_message_from_string,
|
57
|
+
ab_message_to_string,
|
59
58
|
)
|
60
59
|
from .well_known_types import (
|
61
60
|
BinaryData,
|
@@ -1,10 +1,14 @@
|
|
1
1
|
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
|
+
import json
|
2
3
|
import sys
|
3
|
-
from
|
4
|
+
from enum import Enum
|
5
|
+
from typing import Any, Callable, Dict, Type, TypeVar, cast
|
4
6
|
|
5
7
|
import orjson
|
6
8
|
from pydantic import ValidationError
|
7
9
|
|
10
|
+
from airbyte_cdk.logger import init_logger
|
11
|
+
|
8
12
|
from .airbyte_protocol import ( # type: ignore[attr-defined] # all classes are imported to airbyte_protocol via *
|
9
13
|
AirbyteCatalog,
|
10
14
|
AirbyteMessage,
|
@@ -20,9 +24,15 @@ from .airbyte_protocol import ( # type: ignore[attr-defined] # all classes are
|
|
20
24
|
USE_RUST_BACKEND = sys.platform != "emscripten"
|
21
25
|
"""When run in WASM, use the pure Python backend for serpyco."""
|
22
26
|
|
27
|
+
_HAS_LOGGED_FOR_SERIALIZATION_ERROR = False
|
28
|
+
"""Track if we have logged an error for serialization issues."""
|
23
29
|
|
24
30
|
T = TypeVar("T")
|
25
31
|
|
32
|
+
|
33
|
+
logger = init_logger("airbyte")
|
34
|
+
|
35
|
+
|
26
36
|
class CustomSerializer:
|
27
37
|
"""Custom serializer that mimics serpyco-rs Serializer API"""
|
28
38
|
|
@@ -101,11 +111,6 @@ custom_type_resolver = None
|
|
101
111
|
# return {"type": "object"}
|
102
112
|
|
103
113
|
# Create serializer instances maintaining the same API
|
104
|
-
AirbyteCatalogSerializer = SERIALIZER(AirbyteCatalog, omit_none=True)
|
105
|
-
AirbyteStreamSerializer = SERIALIZER(AirbyteStream, omit_none=True)
|
106
|
-
AirbyteStreamStateSerializer = SERIALIZER(
|
107
|
-
AirbyteStreamState, omit_none=True, custom_type_resolver=custom_type_resolver
|
108
|
-
)
|
109
114
|
AirbyteStateMessageSerializer = SERIALIZER(
|
110
115
|
AirbyteStateMessage, omit_none=True, custom_type_resolver=custom_type_resolver
|
111
116
|
)
|
@@ -113,5 +118,65 @@ AirbyteMessageSerializer = SERIALIZER(
|
|
113
118
|
AirbyteMessage, omit_none=True, custom_type_resolver=custom_type_resolver
|
114
119
|
)
|
115
120
|
ConfiguredAirbyteCatalogSerializer = SERIALIZER(ConfiguredAirbyteCatalog, omit_none=True)
|
116
|
-
ConfiguredAirbyteStreamSerializer = SERIALIZER(ConfiguredAirbyteStream, omit_none=True)
|
117
121
|
ConnectorSpecificationSerializer = SERIALIZER(ConnectorSpecification, omit_none=True)
|
122
|
+
|
123
|
+
|
124
|
+
def _custom_json_serializer(val: object) -> str:
|
125
|
+
"""Handle custom serialization needs for AirbyteMessage."""
|
126
|
+
if isinstance(val, Enum):
|
127
|
+
return str(val.value)
|
128
|
+
|
129
|
+
return str(val)
|
130
|
+
|
131
|
+
|
132
|
+
def ab_message_to_string(
|
133
|
+
message: AirbyteMessage,
|
134
|
+
) -> str:
|
135
|
+
"""
|
136
|
+
Convert an AirbyteMessage to a JSON string.
|
137
|
+
|
138
|
+
Args:
|
139
|
+
message (AirbyteMessage): The Airbyte message to convert.
|
140
|
+
|
141
|
+
Returns:
|
142
|
+
str: JSON string representation of the AirbyteMessage.
|
143
|
+
"""
|
144
|
+
global _HAS_LOGGED_FOR_SERIALIZATION_ERROR
|
145
|
+
dict_obj = AirbyteMessageSerializer.dump(message)
|
146
|
+
|
147
|
+
try:
|
148
|
+
return orjson.dumps(
|
149
|
+
dict_obj,
|
150
|
+
default=_custom_json_serializer,
|
151
|
+
).decode()
|
152
|
+
except Exception as exception:
|
153
|
+
if not _HAS_LOGGED_FOR_SERIALIZATION_ERROR:
|
154
|
+
logger.warning(
|
155
|
+
f"There was an error during the serialization of an AirbyteMessage: `{exception}`. This might impact the sync performances."
|
156
|
+
)
|
157
|
+
_HAS_LOGGED_FOR_SERIALIZATION_ERROR = True
|
158
|
+
return json.dumps(
|
159
|
+
dict_obj,
|
160
|
+
default=_custom_json_serializer,
|
161
|
+
)
|
162
|
+
|
163
|
+
|
164
|
+
def ab_message_from_string(
|
165
|
+
message_str: str,
|
166
|
+
) -> AirbyteMessage:
|
167
|
+
"""
|
168
|
+
Convert a JSON string to an AirbyteMessage.
|
169
|
+
|
170
|
+
Args:
|
171
|
+
message_str (str): The JSON string to convert.
|
172
|
+
|
173
|
+
Returns:
|
174
|
+
AirbyteMessage: The deserialized AirbyteMessage.
|
175
|
+
"""
|
176
|
+
try:
|
177
|
+
message_dict = orjson.loads(message_str)
|
178
|
+
return AirbyteMessageSerializer.load(message_dict)
|
179
|
+
except ValidationError as e:
|
180
|
+
raise ValueError(f"Invalid AirbyteMessage format: {e}") from e
|
181
|
+
except orjson.JSONDecodeError as e:
|
182
|
+
raise ValueError(f"Failed to decode JSON: {e}") from e
|
@@ -31,7 +31,7 @@ from airbyte_cdk.models import (
|
|
31
31
|
ConnectorSpecification,
|
32
32
|
FailureType,
|
33
33
|
)
|
34
|
-
from airbyte_cdk.models.airbyte_protocol_serializers import
|
34
|
+
from airbyte_cdk.models.airbyte_protocol_serializers import ab_message_to_string
|
35
35
|
from airbyte_cdk.sources.declarative.checks import COMPONENTS_CHECKER_TYPE_MAPPING
|
36
36
|
from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
|
37
37
|
from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
|
@@ -234,7 +234,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
234
234
|
)
|
235
235
|
# We have no mechanism for consuming the queue, so we print the messages to stdout
|
236
236
|
for message in self.message_repository.consume_queue():
|
237
|
-
print(
|
237
|
+
print(ab_message_to_string(message))
|
238
238
|
self._spec_component.transform_config(mutable_config)
|
239
239
|
return mutable_config
|
240
240
|
|
@@ -14,12 +14,12 @@ import requests_cache
|
|
14
14
|
from requests.auth import AuthBase
|
15
15
|
|
16
16
|
from airbyte_cdk.models import (
|
17
|
-
AirbyteMessageSerializer,
|
18
17
|
AirbyteStreamStatus,
|
19
18
|
AirbyteStreamStatusReason,
|
20
19
|
AirbyteStreamStatusReasonType,
|
21
20
|
Level,
|
22
21
|
StreamDescriptor,
|
22
|
+
ab_message_to_string,
|
23
23
|
)
|
24
24
|
from airbyte_cdk.sources.http_config import MAX_CONNECTION_POOL_SIZE
|
25
25
|
from airbyte_cdk.sources.message import MessageRepository
|
@@ -396,13 +396,11 @@ class HttpClient:
|
|
396
396
|
if error_resolution.response_action == ResponseAction.RATE_LIMITED:
|
397
397
|
# TODO: Update to handle with message repository when concurrent message repository is ready
|
398
398
|
reasons = [AirbyteStreamStatusReason(type=AirbyteStreamStatusReasonType.RATE_LIMITED)]
|
399
|
-
message =
|
400
|
-
|
401
|
-
|
402
|
-
StreamDescriptor(name=self._name), AirbyteStreamStatus.RUNNING, reasons
|
403
|
-
)
|
399
|
+
message = ab_message_to_string(
|
400
|
+
stream_status_as_airbyte_message(
|
401
|
+
StreamDescriptor(name=self._name), AirbyteStreamStatus.RUNNING, reasons
|
404
402
|
)
|
405
|
-
)
|
403
|
+
)
|
406
404
|
|
407
405
|
# Simply printing the stream status is a temporary solution and can cause future issues. Currently, the _send method is
|
408
406
|
# wrapped with backoff decorators, and we can only emit messages by iterating record_iterator in the abstract source at the
|
@@ -5,7 +5,6 @@ from typing import Any, Dict, List, Union, overload
|
|
5
5
|
from airbyte_cdk.models import (
|
6
6
|
ConfiguredAirbyteCatalog,
|
7
7
|
ConfiguredAirbyteStream,
|
8
|
-
ConfiguredAirbyteStreamSerializer,
|
9
8
|
SyncMode,
|
10
9
|
)
|
11
10
|
|
@@ -42,7 +41,7 @@ class ConfiguredAirbyteStreamBuilder:
|
|
42
41
|
return self
|
43
42
|
|
44
43
|
def build(self) -> ConfiguredAirbyteStream:
|
45
|
-
return
|
44
|
+
return ConfiguredAirbyteStream(**self._stream)
|
46
45
|
|
47
46
|
|
48
47
|
class CatalogBuilder:
|
@@ -36,7 +36,6 @@ from airbyte_cdk.logger import AirbyteLogFormatter
|
|
36
36
|
from airbyte_cdk.models import (
|
37
37
|
AirbyteLogMessage,
|
38
38
|
AirbyteMessage,
|
39
|
-
AirbyteMessageSerializer,
|
40
39
|
AirbyteStateMessage,
|
41
40
|
AirbyteStateMessageSerializer,
|
42
41
|
AirbyteStreamState,
|
@@ -46,6 +45,8 @@ from airbyte_cdk.models import (
|
|
46
45
|
Level,
|
47
46
|
TraceType,
|
48
47
|
Type,
|
48
|
+
ab_message_from_string,
|
49
|
+
ab_message_to_string,
|
49
50
|
)
|
50
51
|
from airbyte_cdk.sources import Source
|
51
52
|
from airbyte_cdk.test.models.scenario import ExpectedOutcome
|
@@ -124,7 +125,7 @@ class EntrypointOutput:
|
|
124
125
|
@staticmethod
|
125
126
|
def _parse_message(message: str) -> AirbyteMessage:
|
126
127
|
try:
|
127
|
-
return
|
128
|
+
return ab_message_from_string(message)
|
128
129
|
except JsonValidationErrors:
|
129
130
|
# The platform assumes that logs that are not of AirbyteMessage format are log messages
|
130
131
|
return AirbyteMessage(
|
@@ -11,12 +11,12 @@ from airbyte_cdk.models import (
|
|
11
11
|
AirbyteConnectionStatus,
|
12
12
|
AirbyteErrorTraceMessage,
|
13
13
|
AirbyteMessage,
|
14
|
-
AirbyteMessageSerializer,
|
15
14
|
AirbyteTraceMessage,
|
16
15
|
FailureType,
|
17
16
|
Status,
|
18
17
|
StreamDescriptor,
|
19
18
|
TraceType,
|
19
|
+
ab_message_to_string,
|
20
20
|
)
|
21
21
|
from airbyte_cdk.models import Type as MessageType
|
22
22
|
from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
|
@@ -95,7 +95,7 @@ class AirbyteTracedException(Exception):
|
|
95
95
|
Prints the exception as an AirbyteTraceMessage.
|
96
96
|
Note that this will be called automatically on uncaught exceptions when using the airbyte_cdk entrypoint.
|
97
97
|
"""
|
98
|
-
message =
|
98
|
+
message = ab_message_to_string(self.as_airbyte_message())
|
99
99
|
filtered_message = filter_secrets(message)
|
100
100
|
print(filtered_message)
|
101
101
|
|
@@ -9,14 +9,14 @@ airbyte_cdk/cli/airbyte_cdk/_version.py,sha256=ohZNIktLFk91sdzqFW5idaNrZAPX2dIRn
|
|
9
9
|
airbyte_cdk/cli/airbyte_cdk/exceptions.py,sha256=bsGmlWN6cXL2jCD1WYAZMqFmK1OLg2xLrcC_60KHSeA,803
|
10
10
|
airbyte_cdk/cli/source_declarative_manifest/README.md,sha256=aviNYFk1qKXGm33NQ2mJtJNyQ1MO0SPrm_fggUs0MVE,2460
|
11
11
|
airbyte_cdk/cli/source_declarative_manifest/__init__.py,sha256=F-DFREvW6Sz71nSu0MwVALaybs9veg678tvsGFi2dYo,143
|
12
|
-
airbyte_cdk/cli/source_declarative_manifest/_run.py,sha256=
|
12
|
+
airbyte_cdk/cli/source_declarative_manifest/_run.py,sha256=45FxXFvpKoIMuptu9nZapzSQKQpGivo34T_HSZkua-w,11056
|
13
13
|
airbyte_cdk/cli/source_declarative_manifest/spec.json,sha256=Earc1L6ngcdIr514oFQlUoOxdF4RHqtUyStSIAquXdY,554
|
14
|
-
airbyte_cdk/config_observation.py,sha256=
|
14
|
+
airbyte_cdk/config_observation.py,sha256=HJznEy6WfwmskCAa_x6edK3z3x_GPbkqK0YL8AA4zr8,3950
|
15
15
|
airbyte_cdk/connector.py,sha256=N6TUlrZOMjLAI85JrNAKkfyTqnO5xfBCw4oEfgjJd9o,4254
|
16
16
|
airbyte_cdk/connector_builder/README.md,sha256=Hw3wvVewuHG9-QgsAq1jDiKuLlStDxKBz52ftyNRnBw,1665
|
17
17
|
airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
18
18
|
airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=OFTzxyfAevI3Um8fXTOLTgoCc4Sx9NzF0boqYkAATfM,6590
|
19
|
-
airbyte_cdk/connector_builder/main.py,sha256=
|
19
|
+
airbyte_cdk/connector_builder/main.py,sha256=gzIdZXNCvciwxCqSSpvmMOqIfBwlOAWAs2rqW7c4Xas,3668
|
20
20
|
airbyte_cdk/connector_builder/models.py,sha256=9pIZ98LW_d6fRS39VdnUOf3cxGt4TkC5MJ0_OrzcCRk,1578
|
21
21
|
airbyte_cdk/connector_builder/test_reader/__init__.py,sha256=iTwBMoI9vaJotEgpqZbFjlxRcbxXYypSVJ9YxeHk7wc,120
|
22
22
|
airbyte_cdk/connector_builder/test_reader/helpers.py,sha256=Iczn-_iczS2CaIAunWwyFcX0uLTra8Wh9JVfzm1Gfxo,26765
|
@@ -24,7 +24,7 @@ airbyte_cdk/connector_builder/test_reader/message_grouper.py,sha256=84BAEPIBHMq3
|
|
24
24
|
airbyte_cdk/connector_builder/test_reader/reader.py,sha256=mP1yHK5vG38KxoKoT2QQ7ZNbkdLA1rMAU3EKpucjHls,21098
|
25
25
|
airbyte_cdk/connector_builder/test_reader/types.py,sha256=hPZG3jO03kBaPyW94NI3JHRS1jxXGSNBcN1HFzOxo5Y,2528
|
26
26
|
airbyte_cdk/destinations/__init__.py,sha256=FyDp28PT_YceJD5HDFhA-mrGfX9AONIyMQ4d68CHNxQ,213
|
27
|
-
airbyte_cdk/destinations/destination.py,sha256=
|
27
|
+
airbyte_cdk/destinations/destination.py,sha256=HReNRGouR3OC5dCaA1rZC9CrukIu81MlzWFYO9-4_3Q,5851
|
28
28
|
airbyte_cdk/destinations/vector_db_based/README.md,sha256=QAe8c_1Afme4r2TCE10cTSaxUE3zgCBuArSuRQqK8tA,2115
|
29
29
|
airbyte_cdk/destinations/vector_db_based/__init__.py,sha256=eAkzwTjBbXBhJ5GfPO5I53Zgpv5xQFLRQS8n4nuyPt0,1006
|
30
30
|
airbyte_cdk/destinations/vector_db_based/config.py,sha256=1u87eibIWLZ_wuaCvE3yp5ayguM9dGhGXbT8agmkUBg,12468
|
@@ -34,9 +34,9 @@ airbyte_cdk/destinations/vector_db_based/indexer.py,sha256=beiSi2Uu67EoTr7yQSaCJ
|
|
34
34
|
airbyte_cdk/destinations/vector_db_based/test_utils.py,sha256=MkqLiOJ5QyKbV4rNiJhe-BHM7FD-ADHQ4bQGf4c5lRY,1932
|
35
35
|
airbyte_cdk/destinations/vector_db_based/utils.py,sha256=FOyEo8Lc-fY8UyhpCivhZtIqBRyxf3cUt6anmK03fUY,1127
|
36
36
|
airbyte_cdk/destinations/vector_db_based/writer.py,sha256=nZ00xPiohElJmYktEZZIhr0m5EDETCHGhg0Lb2S7A20,5095
|
37
|
-
airbyte_cdk/entrypoint.py,sha256=
|
37
|
+
airbyte_cdk/entrypoint.py,sha256=74n08ZAudC1qWFcvnn_u0okQlULw1N2vyaHofGVHUu4,18923
|
38
38
|
airbyte_cdk/exception_handler.py,sha256=D_doVl3Dt60ASXlJsfviOCswxGyKF2q0RL6rif3fNks,2013
|
39
|
-
airbyte_cdk/logger.py,sha256=
|
39
|
+
airbyte_cdk/logger.py,sha256=NJq7p3KoRdcoSIQzJ73sfVZ8QlHW3RBC-nQaTjkLMk8,3708
|
40
40
|
airbyte_cdk/manifest_migrations/README.md,sha256=YX1h0xyc4jHdwH3I25ZHqB7R3hcUUCHMvnexpfzF2E8,3020
|
41
41
|
airbyte_cdk/manifest_migrations/__init__.py,sha256=0eq9ic_6GGXMwzE31eAOSA7PLtBauMfgM9XshjYHF84,61
|
42
42
|
airbyte_cdk/manifest_migrations/exceptions.py,sha256=mmMZaCVEkYSGykVL5jKA0xsDWWkybRdQwnh9pGb7VG0,300
|
@@ -48,9 +48,9 @@ airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_json_data
|
|
48
48
|
airbyte_cdk/manifest_migrations/migrations/http_requester_url_base_to_url.py,sha256=EX1MVYVpoWypA28qoH48wA0SYZjGdlR8bcSixTDzfgo,1346
|
49
49
|
airbyte_cdk/manifest_migrations/migrations/registry.yaml,sha256=F-hdapvl_vZnsI7CQsV00Rb7g7j4Nt2zaM83-Tbwgbg,956
|
50
50
|
airbyte_cdk/manifest_migrations/migrations_registry.py,sha256=zly2fwaOxDukqC7eowzrDlvhA2v71FjW74kDzvRXhSY,2619
|
51
|
-
airbyte_cdk/models/__init__.py,sha256=
|
51
|
+
airbyte_cdk/models/__init__.py,sha256=EUxQ9zcq5PZLsnUo30pxoB-pT4UHumtoCqlljjoqd3g,2065
|
52
52
|
airbyte_cdk/models/airbyte_protocol.py,sha256=R1G1lvnBDRCZDa9_9Mgy5OEeyMXbvGWsDOnwQkImRKA,266
|
53
|
-
airbyte_cdk/models/airbyte_protocol_serializers.py,sha256=
|
53
|
+
airbyte_cdk/models/airbyte_protocol_serializers.py,sha256=LlCiIexUYsYYIzjT40p_xEIii-NMNFxgcGo_x9WNiAw,6122
|
54
54
|
airbyte_cdk/models/connector_metadata.py,sha256=BD6CO8c3mHavxRJAcwP29sHtNNVLVSNFNQLgHOVxrwA,3229
|
55
55
|
airbyte_cdk/models/well_known_types.py,sha256=EquepbisGPuCSrs_D7YVVnMR9-ShhUr21wnFz3COiJs,156
|
56
56
|
airbyte_cdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -128,7 +128,7 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=CQkH
|
|
128
128
|
airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=9IoeuWam3L6GyN10L6U8xNWXmkt9cnahSDNkez1OmFY,982
|
129
129
|
airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=oFGKs3oX0xO6DOL4E9x8rhxwbEoRcgx4HJVIL1RQ9c4,7269
|
130
130
|
airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=RpsAYG75bW0js2fQCzAN1nf3oeGyXwyt0LhJCHnlaUA,6031
|
131
|
-
airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=
|
131
|
+
airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=uZAjGHrtXx9Tp0IXPoQbhtagzdx1KBEwQN1eVRsvFFY,26047
|
132
132
|
airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
133
133
|
airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=V2lpYE9LJKvz6BUViHk4vaRGndxNABmPbDCtyYdkqaE,4013
|
134
134
|
airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
|
@@ -349,7 +349,7 @@ airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py,sha
|
|
349
349
|
airbyte_cdk/sources/streams/http/error_handlers/response_models.py,sha256=xGIVELBFY0TmH9aUq1ikoqJz8oHLr6di2JLvKWVEO-s,2236
|
350
350
|
airbyte_cdk/sources/streams/http/exceptions.py,sha256=njC7MlMJoFYcSGz4mIp6-bqLFTr6vC8ej25X0oSeyjE,1824
|
351
351
|
airbyte_cdk/sources/streams/http/http.py,sha256=0uariNq8OFnlX7iqOHwBhecxA-Hfd5hSY8_XCEgn3jI,28499
|
352
|
-
airbyte_cdk/sources/streams/http/http_client.py,sha256=
|
352
|
+
airbyte_cdk/sources/streams/http/http_client.py,sha256=GGHWzelKKfV5T4vfpkEAr0JSjUMP9Jqc9FUpLIeSZUQ,22929
|
353
353
|
airbyte_cdk/sources/streams/http/rate_limiting.py,sha256=IwdjrHKUnU97XO4qONgYRv4YYW51xQ8SJm4WLafXDB8,6351
|
354
354
|
airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py,sha256=RN0D3nOX1xLgwEwKWu6pkGy3XqBFzKSNZ8Lf6umU2eY,413
|
355
355
|
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=0WfnxuxDwRYeq-PIwdUjJujDnxuJPhNfHlX_8aNHtYU,19663
|
@@ -379,8 +379,8 @@ airbyte_cdk/sql/shared/catalog_providers.py,sha256=lUYtJkkpkIcWUdp4rcL15GUrFy2b8
|
|
379
379
|
airbyte_cdk/sql/shared/sql_processor.py,sha256=jR-hdLZsPf2sNBa_wvWKLvys8ZJ-SQCIiJS71Rszc-w,28215
|
380
380
|
airbyte_cdk/sql/types.py,sha256=XEIhRAo_ASd0kVLBkdLf5bHiRhNple-IJrC9TibcDdY,5880
|
381
381
|
airbyte_cdk/test/__init__.py,sha256=f_XdkOg4_63QT2k3BbKY34209lppwgw-svzfZstQEq4,199
|
382
|
-
airbyte_cdk/test/catalog_builder.py,sha256
|
383
|
-
airbyte_cdk/test/entrypoint_wrapper.py,sha256=
|
382
|
+
airbyte_cdk/test/catalog_builder.py,sha256=2_tdbk5Jk8mUXtxAN3_vhGIzQawYOZ9kg39xgoJbtl4,2963
|
383
|
+
airbyte_cdk/test/entrypoint_wrapper.py,sha256=My3Km9wr2ba4FPMNHBKvRVN_HShoEyOWUMFiZ70XEps,18238
|
384
384
|
airbyte_cdk/test/mock_http/__init__.py,sha256=jE5kC6CQ0OXkTqKhciDnNVZHesBFVIA2YvkdFGwva7k,322
|
385
385
|
airbyte_cdk/test/mock_http/matcher.py,sha256=4Qj8UnJKZIs-eodshryce3SN1Ayc8GZpBETmP6hTEyc,1446
|
386
386
|
airbyte_cdk/test/mock_http/mocker.py,sha256=XgsjMtVoeMpRELPyALgrkHFauH9H5irxrz1Kcxh2yFY,8013
|
@@ -423,10 +423,10 @@ airbyte_cdk/utils/schema_inferrer.py,sha256=_jLzL9PzE4gfR44OSavkIqZNFM9t08c3LuRr
|
|
423
423
|
airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7GrKcg,1264
|
424
424
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
425
425
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
426
|
-
airbyte_cdk/utils/traced_exception.py,sha256=
|
427
|
-
airbyte_cdk-6.60.0.
|
428
|
-
airbyte_cdk-6.60.0.
|
429
|
-
airbyte_cdk-6.60.0.
|
430
|
-
airbyte_cdk-6.60.0.
|
431
|
-
airbyte_cdk-6.60.0.
|
432
|
-
airbyte_cdk-6.60.0.
|
426
|
+
airbyte_cdk/utils/traced_exception.py,sha256=bc5jMk8Z3AnSL-vqsgPCNgHzWqGTKZODHSg7VHiUyj0,6256
|
427
|
+
airbyte_cdk-6.60.0.post20.dev16487461641.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
428
|
+
airbyte_cdk-6.60.0.post20.dev16487461641.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
429
|
+
airbyte_cdk-6.60.0.post20.dev16487461641.dist-info/METADATA,sha256=xVhiS-Eb1DNvtzLXoLyYG66DQSZJOfA2oGVqR4SRbPk,6512
|
430
|
+
airbyte_cdk-6.60.0.post20.dev16487461641.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
431
|
+
airbyte_cdk-6.60.0.post20.dev16487461641.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
|
432
|
+
airbyte_cdk-6.60.0.post20.dev16487461641.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|