airbyte-cdk 6.8.1.dev1__py3-none-any.whl → 6.8.1rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/cli/source_declarative_manifest/_run.py +5 -11
- airbyte_cdk/config_observation.py +1 -1
- airbyte_cdk/connector_builder/main.py +1 -1
- airbyte_cdk/connector_builder/message_grouper.py +10 -10
- airbyte_cdk/destinations/destination.py +1 -1
- airbyte_cdk/destinations/vector_db_based/embedder.py +2 -2
- airbyte_cdk/destinations/vector_db_based/writer.py +4 -12
- airbyte_cdk/entrypoint.py +6 -7
- airbyte_cdk/logger.py +2 -2
- airbyte_cdk/sources/abstract_source.py +1 -1
- airbyte_cdk/sources/config.py +1 -1
- airbyte_cdk/sources/connector_state_manager.py +4 -9
- airbyte_cdk/sources/declarative/auth/oauth.py +1 -1
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +1 -6
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +1 -1
- airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +4 -10
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +17 -16
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +1 -4
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +6 -8
- airbyte_cdk/sources/declarative/interpolation/jinja.py +3 -3
- airbyte_cdk/sources/declarative/interpolation/macros.py +1 -1
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +6 -5
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +7 -13
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +1 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +6 -8
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +1 -1
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +2 -2
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +1 -1
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +2 -5
- airbyte_cdk/sources/declarative/spec/spec.py +1 -1
- airbyte_cdk/sources/embedded/base_integration.py +2 -3
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +4 -12
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +7 -18
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +11 -14
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +3 -3
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +5 -11
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +1 -1
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +2 -2
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +3 -6
- airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +1 -1
- airbyte_cdk/sources/http_logger.py +3 -3
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +2 -5
- airbyte_cdk/sources/streams/concurrent/adapters.py +3 -6
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +3 -9
- airbyte_cdk/sources/streams/concurrent/cursor.py +1 -1
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +2 -2
- airbyte_cdk/sources/streams/core.py +14 -17
- airbyte_cdk/sources/streams/http/http.py +19 -19
- airbyte_cdk/sources/streams/http/http_client.py +34 -3
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +1 -2
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +33 -62
- airbyte_cdk/sources/utils/record_helper.py +1 -1
- airbyte_cdk/sources/utils/schema_helpers.py +1 -1
- airbyte_cdk/sources/utils/transform.py +15 -34
- airbyte_cdk/test/entrypoint_wrapper.py +6 -11
- airbyte_cdk/test/mock_http/response_builder.py +1 -1
- airbyte_cdk/utils/airbyte_secrets_utils.py +1 -1
- airbyte_cdk/utils/event_timing.py +10 -10
- airbyte_cdk/utils/message_utils.py +3 -4
- airbyte_cdk/utils/spec_schema_transformations.py +2 -3
- airbyte_cdk/utils/traced_exception.py +12 -14
- {airbyte_cdk-6.8.1.dev1.dist-info → airbyte_cdk-6.8.1rc1.dist-info}/METADATA +2 -1
- {airbyte_cdk-6.8.1.dev1.dist-info → airbyte_cdk-6.8.1rc1.dist-info}/RECORD +66 -67
- airbyte_cdk/test/utils/manifest_only_fixtures.py +0 -40
- {airbyte_cdk-6.8.1.dev1.dist-info → airbyte_cdk-6.8.1rc1.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.8.1.dev1.dist-info → airbyte_cdk-6.8.1rc1.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.8.1.dev1.dist-info → airbyte_cdk-6.8.1rc1.dist-info}/entry_points.txt +0 -0
@@ -10,7 +10,7 @@ from dataclasses import dataclass
|
|
10
10
|
from functools import cached_property, lru_cache
|
11
11
|
from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Union
|
12
12
|
|
13
|
-
from
|
13
|
+
from deprecated import deprecated
|
14
14
|
|
15
15
|
import airbyte_cdk.sources.utils.casing as casing
|
16
16
|
from airbyte_cdk.models import (
|
@@ -92,8 +92,8 @@ class CheckpointMixin(ABC):
|
|
92
92
|
|
93
93
|
|
94
94
|
@deprecated(
|
95
|
-
"
|
96
|
-
"Deprecated in favor of the
|
95
|
+
version="0.87.0",
|
96
|
+
reason="Deprecated in favor of the CheckpointMixin which offers similar functionality",
|
97
97
|
)
|
98
98
|
class IncrementalMixin(CheckpointMixin, ABC):
|
99
99
|
"""Mixin to make stream incremental.
|
@@ -115,6 +115,12 @@ class StreamClassification:
|
|
115
115
|
has_multiple_slices: bool
|
116
116
|
|
117
117
|
|
118
|
+
# Moved to class declaration since get_updated_state is called on every record for incremental syncs, and thus the @deprecated decorator as well.
|
119
|
+
@deprecated(
|
120
|
+
version="0.1.49",
|
121
|
+
reason="Deprecated method get_updated_state, You should use explicit state property instead, see IncrementalMixin docs.",
|
122
|
+
action="ignore",
|
123
|
+
)
|
118
124
|
class Stream(ABC):
|
119
125
|
"""
|
120
126
|
Base abstract class for an Airbyte Stream. Makes no assumption of the Stream's underlying transport protocol.
|
@@ -216,8 +222,7 @@ class Stream(ABC):
|
|
216
222
|
# Some connectors have streams that implement get_updated_state(), but do not define a cursor_field. This
|
217
223
|
# should be fixed on the stream implementation, but we should also protect against this in the CDK as well
|
218
224
|
stream_state_tracker = self.get_updated_state(
|
219
|
-
stream_state_tracker,
|
220
|
-
record_data, # type: ignore [arg-type]
|
225
|
+
stream_state_tracker, record_data
|
221
226
|
)
|
222
227
|
self._observe_state(checkpoint_reader, stream_state_tracker)
|
223
228
|
record_counter += 1
|
@@ -277,7 +282,7 @@ class Stream(ABC):
|
|
277
282
|
if state
|
278
283
|
else {}, # read() expects MutableMapping instead of Mapping which is used more often
|
279
284
|
state_manager=None,
|
280
|
-
internal_config=InternalConfig(),
|
285
|
+
internal_config=InternalConfig(),
|
281
286
|
)
|
282
287
|
|
283
288
|
@abstractmethod
|
@@ -317,7 +322,7 @@ class Stream(ABC):
|
|
317
322
|
# If we can offer incremental we always should. RFR is always less reliable than incremental which uses a real cursor value
|
318
323
|
if self.supports_incremental:
|
319
324
|
stream.source_defined_cursor = self.source_defined_cursor
|
320
|
-
stream.supported_sync_modes.append(SyncMode.incremental)
|
325
|
+
stream.supported_sync_modes.append(SyncMode.incremental) # type: ignore
|
321
326
|
stream.default_cursor_field = self._wrapped_cursor_field()
|
322
327
|
|
323
328
|
keys = Stream._wrapped_primary_key(self.primary_key)
|
@@ -431,18 +436,10 @@ class Stream(ABC):
|
|
431
436
|
"""
|
432
437
|
return None
|
433
438
|
|
434
|
-
# Commented-out to avoid any runtime penalty, since this is used in a hot per-record codepath.
|
435
|
-
# To be evaluated for re-introduction here: https://github.com/airbytehq/airbyte-python-cdk/issues/116
|
436
|
-
# @deprecated(
|
437
|
-
# "Deprecated method `get_updated_state` as of CDK version 0.1.49. "
|
438
|
-
# "Please use explicit state property instead, see `IncrementalMixin` docs."
|
439
|
-
# )
|
440
439
|
def get_updated_state(
|
441
440
|
self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]
|
442
441
|
) -> MutableMapping[str, Any]:
|
443
|
-
"""
|
444
|
-
|
445
|
-
Override to extract state from the latest record. Needed to implement incremental sync.
|
442
|
+
"""Override to extract state from the latest record. Needed to implement incremental sync.
|
446
443
|
|
447
444
|
Inspects the latest record extracted from the data source and the current state object and return an updated state object.
|
448
445
|
|
@@ -657,7 +654,7 @@ class Stream(ABC):
|
|
657
654
|
# todo: This can be consolidated into one ConnectorStateManager.update_and_create_state_message() method, but I want
|
658
655
|
# to reduce changes right now and this would span concurrent as well
|
659
656
|
state_manager.update_state_for_stream(self.name, self.namespace, stream_state)
|
660
|
-
return state_manager.create_state_message(self.name, self.namespace)
|
657
|
+
return state_manager.create_state_message(self.name, self.namespace)
|
661
658
|
|
662
659
|
@property
|
663
660
|
def configured_json_schema(self) -> Optional[Dict[str, Any]]:
|
@@ -9,8 +9,8 @@ from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optio
|
|
9
9
|
from urllib.parse import urljoin
|
10
10
|
|
11
11
|
import requests
|
12
|
+
from deprecated import deprecated
|
12
13
|
from requests.auth import AuthBase
|
13
|
-
from typing_extensions import deprecated
|
14
14
|
|
15
15
|
from airbyte_cdk.models import AirbyteMessage, FailureType, SyncMode
|
16
16
|
from airbyte_cdk.models import Type as MessageType
|
@@ -121,8 +121,8 @@ class HttpStream(Stream, CheckpointMixin, ABC):
|
|
121
121
|
|
122
122
|
@property
|
123
123
|
@deprecated(
|
124
|
-
"
|
125
|
-
"You should set error_handler explicitly in HttpStream.get_error_handler() instead."
|
124
|
+
version="3.0.0",
|
125
|
+
reason="You should set error_handler explicitly in HttpStream.get_error_handler() instead.",
|
126
126
|
)
|
127
127
|
def raise_on_http_errors(self) -> bool:
|
128
128
|
"""
|
@@ -132,8 +132,8 @@ class HttpStream(Stream, CheckpointMixin, ABC):
|
|
132
132
|
|
133
133
|
@property
|
134
134
|
@deprecated(
|
135
|
-
"
|
136
|
-
"You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead."
|
135
|
+
version="3.0.0",
|
136
|
+
reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.",
|
137
137
|
)
|
138
138
|
def max_retries(self) -> Union[int, None]:
|
139
139
|
"""
|
@@ -143,8 +143,8 @@ class HttpStream(Stream, CheckpointMixin, ABC):
|
|
143
143
|
|
144
144
|
@property
|
145
145
|
@deprecated(
|
146
|
-
"
|
147
|
-
"You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead."
|
146
|
+
version="3.0.0",
|
147
|
+
reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.",
|
148
148
|
)
|
149
149
|
def max_time(self) -> Union[int, None]:
|
150
150
|
"""
|
@@ -154,8 +154,8 @@ class HttpStream(Stream, CheckpointMixin, ABC):
|
|
154
154
|
|
155
155
|
@property
|
156
156
|
@deprecated(
|
157
|
-
"
|
158
|
-
"You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead."
|
157
|
+
version="3.0.0",
|
158
|
+
reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.",
|
159
159
|
)
|
160
160
|
def retry_factor(self) -> float:
|
161
161
|
"""
|
@@ -594,7 +594,7 @@ class HttpSubStream(HttpStream, ABC):
|
|
594
594
|
# Skip non-records (eg AirbyteLogMessage)
|
595
595
|
if isinstance(parent_record, AirbyteMessage):
|
596
596
|
if parent_record.type == MessageType.RECORD:
|
597
|
-
parent_record = parent_record.record.data
|
597
|
+
parent_record = parent_record.record.data
|
598
598
|
else:
|
599
599
|
continue
|
600
600
|
elif isinstance(parent_record, Record):
|
@@ -603,8 +603,8 @@ class HttpSubStream(HttpStream, ABC):
|
|
603
603
|
|
604
604
|
|
605
605
|
@deprecated(
|
606
|
-
"
|
607
|
-
"You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead."
|
606
|
+
version="3.0.0",
|
607
|
+
reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.",
|
608
608
|
)
|
609
609
|
class HttpStreamAdapterBackoffStrategy(BackoffStrategy):
|
610
610
|
def __init__(self, stream: HttpStream):
|
@@ -619,8 +619,8 @@ class HttpStreamAdapterBackoffStrategy(BackoffStrategy):
|
|
619
619
|
|
620
620
|
|
621
621
|
@deprecated(
|
622
|
-
"
|
623
|
-
"You should set error_handler explicitly in HttpStream.get_error_handler() instead."
|
622
|
+
version="3.0.0",
|
623
|
+
reason="You should set error_handler explicitly in HttpStream.get_error_handler() instead.",
|
624
624
|
)
|
625
625
|
class HttpStreamAdapterHttpStatusErrorHandler(HttpStatusErrorHandler):
|
626
626
|
def __init__(self, stream: HttpStream, **kwargs): # type: ignore # noqa
|
@@ -639,15 +639,15 @@ class HttpStreamAdapterHttpStatusErrorHandler(HttpStatusErrorHandler):
|
|
639
639
|
return ErrorResolution(
|
640
640
|
response_action=ResponseAction.RATE_LIMITED,
|
641
641
|
failure_type=FailureType.transient_error,
|
642
|
-
error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
|
642
|
+
error_message=f"Response status code: {response_or_exception.status_code}. Retrying...", # type: ignore[union-attr]
|
643
643
|
)
|
644
644
|
return ErrorResolution(
|
645
645
|
response_action=ResponseAction.RETRY,
|
646
646
|
failure_type=FailureType.transient_error,
|
647
|
-
error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
|
647
|
+
error_message=f"Response status code: {response_or_exception.status_code}. Retrying...", # type: ignore[union-attr]
|
648
648
|
)
|
649
649
|
else:
|
650
|
-
if response_or_exception.ok:
|
650
|
+
if response_or_exception.ok: # type: ignore # noqa
|
651
651
|
return ErrorResolution(
|
652
652
|
response_action=ResponseAction.SUCCESS,
|
653
653
|
failure_type=None,
|
@@ -657,13 +657,13 @@ class HttpStreamAdapterHttpStatusErrorHandler(HttpStatusErrorHandler):
|
|
657
657
|
return ErrorResolution(
|
658
658
|
response_action=ResponseAction.FAIL,
|
659
659
|
failure_type=FailureType.transient_error,
|
660
|
-
error_message=f"Response status code: {response_or_exception.status_code}. Unexpected error. Failed.",
|
660
|
+
error_message=f"Response status code: {response_or_exception.status_code}. Unexpected error. Failed.", # type: ignore[union-attr]
|
661
661
|
)
|
662
662
|
else:
|
663
663
|
return ErrorResolution(
|
664
664
|
response_action=ResponseAction.IGNORE,
|
665
665
|
failure_type=FailureType.transient_error,
|
666
|
-
error_message=f"Response status code: {response_or_exception.status_code}. Ignoring...",
|
666
|
+
error_message=f"Response status code: {response_or_exception.status_code}. Ignoring...", # type: ignore[union-attr]
|
667
667
|
)
|
668
668
|
else:
|
669
669
|
self._logger.error(f"Received unexpected response type: {type(response_or_exception)}")
|
@@ -54,6 +54,7 @@ from airbyte_cdk.utils.stream_status_utils import (
|
|
54
54
|
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
55
55
|
|
56
56
|
BODY_REQUEST_METHODS = ("GET", "POST", "PUT", "PATCH")
|
57
|
+
logger = logging.getLogger("airbyte")
|
57
58
|
|
58
59
|
|
59
60
|
class MessageRepresentationAirbyteTracedErrors(AirbyteTracedException):
|
@@ -142,9 +143,10 @@ class HttpClient:
|
|
142
143
|
sqlite_path = str(Path(cache_dir) / self.cache_filename)
|
143
144
|
else:
|
144
145
|
sqlite_path = "file::memory:?cache=shared"
|
146
|
+
backend = SkipFailureSQLiteCache(sqlite_path)
|
145
147
|
return CachedLimiterSession(
|
146
|
-
sqlite_path, backend=
|
147
|
-
)
|
148
|
+
sqlite_path, backend=backend, api_budget=self._api_budget, match_headers=True
|
149
|
+
) # type: ignore # there are no typeshed stubs for requests_cache
|
148
150
|
else:
|
149
151
|
return LimiterSession(api_budget=self._api_budget)
|
150
152
|
|
@@ -324,7 +326,7 @@ class HttpClient:
|
|
324
326
|
formatter = log_formatter
|
325
327
|
self._message_repository.log_message(
|
326
328
|
Level.DEBUG,
|
327
|
-
lambda: formatter(response),
|
329
|
+
lambda: formatter(response), # type: ignore # log_formatter is always cast to a callable
|
328
330
|
)
|
329
331
|
|
330
332
|
self._handle_error_resolution(
|
@@ -517,3 +519,32 @@ class HttpClient:
|
|
517
519
|
)
|
518
520
|
|
519
521
|
return request, response
|
522
|
+
|
523
|
+
|
524
|
+
class SkipFailureSQLiteDict(requests_cache.backends.sqlite.SQLiteDict):
|
525
|
+
def _write(self, key: str, value: str) -> None:
|
526
|
+
try:
|
527
|
+
super()._write(key, value) # type: ignore # lib is not typed
|
528
|
+
except Exception as exception:
|
529
|
+
logger.warning(exception)
|
530
|
+
|
531
|
+
|
532
|
+
class SkipFailureSQLiteCache(requests_cache.backends.sqlite.SQLiteCache):
|
533
|
+
def __init__( # type: ignore # ignoring as lib is not typed
|
534
|
+
self,
|
535
|
+
db_path="http_cache",
|
536
|
+
serializer=None,
|
537
|
+
**kwargs,
|
538
|
+
) -> None:
|
539
|
+
super().__init__(db_path, serializer, **kwargs)
|
540
|
+
skwargs = {"serializer": serializer, **kwargs} if serializer else kwargs
|
541
|
+
self.responses: requests_cache.backends.sqlite.SQLiteDict = SkipFailureSQLiteDict(
|
542
|
+
db_path, table_name="responses", **skwargs
|
543
|
+
)
|
544
|
+
self.redirects: requests_cache.backends.sqlite.SQLiteDict = SkipFailureSQLiteDict(
|
545
|
+
db_path,
|
546
|
+
table_name="redirects",
|
547
|
+
lock=self.responses._lock,
|
548
|
+
serializer=None,
|
549
|
+
**kwargs,
|
550
|
+
)
|
@@ -5,14 +5,13 @@
|
|
5
5
|
from abc import abstractmethod
|
6
6
|
from typing import Any, Mapping
|
7
7
|
|
8
|
-
import requests
|
9
8
|
from requests.auth import AuthBase
|
10
9
|
|
11
10
|
|
12
11
|
class AbstractHeaderAuthenticator(AuthBase):
|
13
12
|
"""Abstract class for an header-based authenticators that add a header to outgoing HTTP requests."""
|
14
13
|
|
15
|
-
def __call__(self, request
|
14
|
+
def __call__(self, request):
|
16
15
|
"""Attach the HTTP headers required to authenticate on the HTTP request"""
|
17
16
|
request.headers.update(self.get_auth_header())
|
18
17
|
return request
|
@@ -30,12 +30,12 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
|
|
30
30
|
client_id: str,
|
31
31
|
client_secret: str,
|
32
32
|
refresh_token: str,
|
33
|
-
scopes: List[str]
|
34
|
-
token_expiry_date: pendulum.DateTime
|
35
|
-
token_expiry_date_format: str
|
33
|
+
scopes: List[str] = None,
|
34
|
+
token_expiry_date: pendulum.DateTime = None,
|
35
|
+
token_expiry_date_format: str = None,
|
36
36
|
access_token_name: str = "access_token",
|
37
37
|
expires_in_name: str = "expires_in",
|
38
|
-
refresh_request_body: Mapping[str, Any]
|
38
|
+
refresh_request_body: Mapping[str, Any] = None,
|
39
39
|
grant_type: str = "refresh_token",
|
40
40
|
token_expiry_is_time_of_expiration: bool = False,
|
41
41
|
refresh_token_error_status_codes: Tuple[int, ...] = (),
|
@@ -52,7 +52,7 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
|
|
52
52
|
self._refresh_request_body = refresh_request_body
|
53
53
|
self._grant_type = grant_type
|
54
54
|
|
55
|
-
self._token_expiry_date = token_expiry_date or pendulum.now().subtract(days=1)
|
55
|
+
self._token_expiry_date = token_expiry_date or pendulum.now().subtract(days=1)
|
56
56
|
self._token_expiry_date_format = token_expiry_date_format
|
57
57
|
self._token_expiry_is_time_of_expiration = token_expiry_is_time_of_expiration
|
58
58
|
self._access_token = None
|
@@ -75,14 +75,14 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
|
|
75
75
|
def get_access_token_name(self) -> str:
|
76
76
|
return self._access_token_name
|
77
77
|
|
78
|
-
def get_scopes(self) ->
|
79
|
-
return self._scopes
|
78
|
+
def get_scopes(self) -> [str]:
|
79
|
+
return self._scopes
|
80
80
|
|
81
81
|
def get_expires_in_name(self) -> str:
|
82
82
|
return self._expires_in_name
|
83
83
|
|
84
84
|
def get_refresh_request_body(self) -> Mapping[str, Any]:
|
85
|
-
return self._refresh_request_body
|
85
|
+
return self._refresh_request_body
|
86
86
|
|
87
87
|
def get_grant_type(self) -> str:
|
88
88
|
return self._grant_type
|
@@ -90,7 +90,7 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
|
|
90
90
|
def get_token_expiry_date(self) -> pendulum.DateTime:
|
91
91
|
return self._token_expiry_date
|
92
92
|
|
93
|
-
def set_token_expiry_date(self, value: Union[str, int])
|
93
|
+
def set_token_expiry_date(self, value: Union[str, int]):
|
94
94
|
self._token_expiry_date = self._parse_token_expiration_date(value)
|
95
95
|
|
96
96
|
@property
|
@@ -103,11 +103,11 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
|
|
103
103
|
|
104
104
|
@property
|
105
105
|
def access_token(self) -> str:
|
106
|
-
return self._access_token
|
106
|
+
return self._access_token
|
107
107
|
|
108
108
|
@access_token.setter
|
109
|
-
def access_token(self, value: str)
|
110
|
-
self._access_token = value
|
109
|
+
def access_token(self, value: str):
|
110
|
+
self._access_token = value
|
111
111
|
|
112
112
|
|
113
113
|
class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
|
@@ -124,11 +124,11 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
|
|
124
124
|
self,
|
125
125
|
connector_config: Mapping[str, Any],
|
126
126
|
token_refresh_endpoint: str,
|
127
|
-
scopes: List[str]
|
127
|
+
scopes: List[str] = None,
|
128
128
|
access_token_name: str = "access_token",
|
129
129
|
expires_in_name: str = "expires_in",
|
130
130
|
refresh_token_name: str = "refresh_token",
|
131
|
-
refresh_request_body: Mapping[str, Any]
|
131
|
+
refresh_request_body: Mapping[str, Any] = None,
|
132
132
|
grant_type: str = "refresh_token",
|
133
133
|
client_id: Optional[str] = None,
|
134
134
|
client_secret: Optional[str] = None,
|
@@ -162,17 +162,14 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
|
|
162
162
|
message_repository (MessageRepository): the message repository used to emit logs on HTTP requests and control message on config update
|
163
163
|
"""
|
164
164
|
self._client_id = (
|
165
|
-
client_id
|
165
|
+
client_id
|
166
166
|
if client_id is not None
|
167
|
-
else dpath.get(connector_config, ("credentials", "client_id"))
|
167
|
+
else dpath.get(connector_config, ("credentials", "client_id"))
|
168
168
|
)
|
169
169
|
self._client_secret = (
|
170
|
-
client_secret
|
170
|
+
client_secret
|
171
171
|
if client_secret is not None
|
172
|
-
else dpath.get(
|
173
|
-
connector_config, # type: ignore [arg-type]
|
174
|
-
("credentials", "client_secret"),
|
175
|
-
)
|
172
|
+
else dpath.get(connector_config, ("credentials", "client_secret"))
|
176
173
|
)
|
177
174
|
self._access_token_config_path = access_token_config_path
|
178
175
|
self._refresh_token_config_path = refresh_token_config_path
|
@@ -210,50 +207,27 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
|
|
210
207
|
|
211
208
|
@property
|
212
209
|
def access_token(self) -> str:
|
213
|
-
return dpath.get(
|
214
|
-
self._connector_config, # type: ignore [arg-type]
|
215
|
-
self._access_token_config_path,
|
216
|
-
default="",
|
217
|
-
)
|
210
|
+
return dpath.get(self._connector_config, self._access_token_config_path, default="")
|
218
211
|
|
219
212
|
@access_token.setter
|
220
|
-
def access_token(self, new_access_token: str)
|
221
|
-
dpath.new(
|
222
|
-
self._connector_config, # type: ignore [arg-type]
|
223
|
-
self._access_token_config_path,
|
224
|
-
new_access_token,
|
225
|
-
)
|
213
|
+
def access_token(self, new_access_token: str):
|
214
|
+
dpath.new(self._connector_config, self._access_token_config_path, new_access_token)
|
226
215
|
|
227
216
|
def get_refresh_token(self) -> str:
|
228
|
-
return dpath.get(
|
229
|
-
self._connector_config, # type: ignore [arg-type]
|
230
|
-
self._refresh_token_config_path,
|
231
|
-
default="",
|
232
|
-
)
|
217
|
+
return dpath.get(self._connector_config, self._refresh_token_config_path, default="")
|
233
218
|
|
234
|
-
def set_refresh_token(self, new_refresh_token: str)
|
235
|
-
dpath.new(
|
236
|
-
self._connector_config, # type: ignore [arg-type]
|
237
|
-
self._refresh_token_config_path,
|
238
|
-
new_refresh_token,
|
239
|
-
)
|
219
|
+
def set_refresh_token(self, new_refresh_token: str):
|
220
|
+
dpath.new(self._connector_config, self._refresh_token_config_path, new_refresh_token)
|
240
221
|
|
241
222
|
def get_token_expiry_date(self) -> pendulum.DateTime:
|
242
223
|
expiry_date = dpath.get(
|
243
|
-
self._connector_config,
|
244
|
-
self._token_expiry_date_config_path,
|
245
|
-
default="",
|
224
|
+
self._connector_config, self._token_expiry_date_config_path, default=""
|
246
225
|
)
|
247
|
-
return pendulum.now().subtract(days=1) if expiry_date == "" else pendulum.parse(expiry_date)
|
226
|
+
return pendulum.now().subtract(days=1) if expiry_date == "" else pendulum.parse(expiry_date)
|
248
227
|
|
249
|
-
def set_token_expiry_date(
|
250
|
-
self,
|
251
|
-
new_token_expiry_date: pendulum.DateTime,
|
252
|
-
) -> None:
|
228
|
+
def set_token_expiry_date(self, new_token_expiry_date):
|
253
229
|
dpath.new(
|
254
|
-
self._connector_config,
|
255
|
-
self._token_expiry_date_config_path,
|
256
|
-
str(new_token_expiry_date),
|
230
|
+
self._connector_config, self._token_expiry_date_config_path, str(new_token_expiry_date)
|
257
231
|
)
|
258
232
|
|
259
233
|
def token_has_expired(self) -> bool:
|
@@ -262,8 +236,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
|
|
262
236
|
|
263
237
|
@staticmethod
|
264
238
|
def get_new_token_expiry_date(
|
265
|
-
access_token_expires_in: str,
|
266
|
-
token_expiry_date_format: str | None = None,
|
239
|
+
access_token_expires_in: str, token_expiry_date_format: str = None
|
267
240
|
) -> pendulum.DateTime:
|
268
241
|
if token_expiry_date_format:
|
269
242
|
return pendulum.from_format(access_token_expires_in, token_expiry_date_format)
|
@@ -280,7 +253,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
|
|
280
253
|
new_access_token, access_token_expires_in, new_refresh_token = (
|
281
254
|
self.refresh_access_token()
|
282
255
|
)
|
283
|
-
new_token_expiry_date
|
256
|
+
new_token_expiry_date = self.get_new_token_expiry_date(
|
284
257
|
access_token_expires_in, self._token_expiry_date_format
|
285
258
|
)
|
286
259
|
self.access_token = new_access_token
|
@@ -291,15 +264,13 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
|
|
291
264
|
# message directly in the console, this is needed
|
292
265
|
if not isinstance(self._message_repository, NoopMessageRepository):
|
293
266
|
self._message_repository.emit_message(
|
294
|
-
create_connector_config_control_message(self._connector_config)
|
267
|
+
create_connector_config_control_message(self._connector_config)
|
295
268
|
)
|
296
269
|
else:
|
297
|
-
emit_configuration_as_airbyte_control_message(self._connector_config)
|
270
|
+
emit_configuration_as_airbyte_control_message(self._connector_config)
|
298
271
|
return self.access_token
|
299
272
|
|
300
|
-
def refresh_access_token(
|
301
|
-
self,
|
302
|
-
) -> Tuple[str, str, str]:
|
273
|
+
def refresh_access_token(self) -> Tuple[str, str, str]:
|
303
274
|
response_json = self._get_refresh_access_token_response()
|
304
275
|
return (
|
305
276
|
response_json[self.get_access_token_name()],
|
@@ -35,7 +35,7 @@ def stream_data_to_airbyte_message(
|
|
35
35
|
# need it to normalize values against json schema. By default no action
|
36
36
|
# taken unless configured. See
|
37
37
|
# docs/connector-development/cdk-python/schemas.md for details.
|
38
|
-
transformer.transform(data, schema)
|
38
|
+
transformer.transform(data, schema) # type: ignore
|
39
39
|
if is_file_transfer_message:
|
40
40
|
message = AirbyteFileTransferRecordMessage(
|
41
41
|
stream=stream_name, file=data, emitted_at=now_millis, data={}
|
@@ -194,7 +194,7 @@ class InternalConfig(BaseModel):
|
|
194
194
|
def dict(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
|
195
195
|
kwargs["by_alias"] = True
|
196
196
|
kwargs["exclude_unset"] = True
|
197
|
-
return super().dict(*args, **kwargs)
|
197
|
+
return super().dict(*args, **kwargs) # type: ignore[no-any-return]
|
198
198
|
|
199
199
|
def is_limit_reached(self, records_counter: int) -> bool:
|
200
200
|
"""
|
@@ -5,9 +5,9 @@
|
|
5
5
|
import logging
|
6
6
|
from distutils.util import strtobool
|
7
7
|
from enum import Flag, auto
|
8
|
-
from typing import Any, Callable, Dict,
|
8
|
+
from typing import Any, Callable, Dict, Mapping, Optional
|
9
9
|
|
10
|
-
from jsonschema import Draft7Validator,
|
10
|
+
from jsonschema import Draft7Validator, ValidationError, validators
|
11
11
|
|
12
12
|
json_to_python_simple = {
|
13
13
|
"string": str,
|
@@ -30,7 +30,7 @@ class TransformConfig(Flag):
|
|
30
30
|
```
|
31
31
|
"""
|
32
32
|
|
33
|
-
# No action taken, default
|
33
|
+
# No action taken, default behaviour. Cannot be combined with any other options.
|
34
34
|
NoTransform = auto()
|
35
35
|
# Applies default type casting with default_convert method which converts
|
36
36
|
# values by applying simple type casting to specified jsonschema type.
|
@@ -67,15 +67,15 @@ class TypeTransformer:
|
|
67
67
|
)
|
68
68
|
|
69
69
|
def registerCustomTransform(
|
70
|
-
self, normalization_callback: Callable[[Any,
|
71
|
-
) -> Callable
|
70
|
+
self, normalization_callback: Callable[[Any, Dict[str, Any]], Any]
|
71
|
+
) -> Callable:
|
72
72
|
"""
|
73
73
|
Register custom normalization callback.
|
74
74
|
:param normalization_callback function to be used for value
|
75
75
|
normalization. Takes original value and part type schema. Should return
|
76
76
|
normalized value. See docs/connector-development/cdk-python/schemas.md
|
77
77
|
for details.
|
78
|
-
:return Same
|
78
|
+
:return Same callbeck, this is usefull for using registerCustomTransform function as decorator.
|
79
79
|
"""
|
80
80
|
if TransformConfig.CustomSchemaNormalization not in self._config:
|
81
81
|
raise Exception(
|
@@ -141,11 +141,7 @@ class TypeTransformer:
|
|
141
141
|
return original_item
|
142
142
|
return original_item
|
143
143
|
|
144
|
-
def __get_normalizer(
|
145
|
-
self,
|
146
|
-
schema_key: str,
|
147
|
-
original_validator: Callable, # type: ignore[type-arg]
|
148
|
-
) -> Callable[[Any, Any, Any, dict[str, Any]], Generator[Any, Any, None]]:
|
144
|
+
def __get_normalizer(self, schema_key: str, original_validator: Callable):
|
149
145
|
"""
|
150
146
|
Traverse through object fields using native jsonschema validator and apply normalization function.
|
151
147
|
:param schema_key related json schema key that currently being validated/normalized.
|
@@ -153,11 +149,8 @@ class TypeTransformer:
|
|
153
149
|
"""
|
154
150
|
|
155
151
|
def normalizator(
|
156
|
-
validator_instance:
|
157
|
-
|
158
|
-
instance: Any,
|
159
|
-
schema: Dict[str, Any],
|
160
|
-
) -> Generator[Any, Any, None]:
|
152
|
+
validator_instance: Callable, property_value: Any, instance: Any, schema: Dict[str, Any]
|
153
|
+
):
|
161
154
|
"""
|
162
155
|
Jsonschema validator callable it uses for validating instance. We
|
163
156
|
override default Draft7Validator to perform value transformation
|
@@ -170,13 +163,10 @@ class TypeTransformer:
|
|
170
163
|
:
|
171
164
|
"""
|
172
165
|
|
173
|
-
def resolve(subschema
|
166
|
+
def resolve(subschema):
|
174
167
|
if "$ref" in subschema:
|
175
|
-
_, resolved =
|
176
|
-
|
177
|
-
validator_instance.resolver,
|
178
|
-
).resolve(subschema["$ref"])
|
179
|
-
return cast(dict[str, Any], resolved)
|
168
|
+
_, resolved = validator_instance.resolver.resolve(subschema["$ref"])
|
169
|
+
return resolved
|
180
170
|
return subschema
|
181
171
|
|
182
172
|
# Transform object and array values before running json schema type checking for each element.
|
@@ -195,20 +185,11 @@ class TypeTransformer:
|
|
195
185
|
instance[index] = self.__normalize(item, subschema)
|
196
186
|
|
197
187
|
# Running native jsonschema traverse algorithm after field normalization is done.
|
198
|
-
yield from original_validator(
|
199
|
-
validator_instance,
|
200
|
-
property_value,
|
201
|
-
instance,
|
202
|
-
schema,
|
203
|
-
)
|
188
|
+
yield from original_validator(validator_instance, property_value, instance, schema)
|
204
189
|
|
205
190
|
return normalizator
|
206
191
|
|
207
|
-
def transform(
|
208
|
-
self,
|
209
|
-
record: Dict[str, Any],
|
210
|
-
schema: Mapping[str, Any],
|
211
|
-
) -> None:
|
192
|
+
def transform(self, record: Dict[str, Any], schema: Mapping[str, Any]):
|
212
193
|
"""
|
213
194
|
Normalize and validate according to config.
|
214
195
|
:param record: record instance for normalization/transformation. All modification are done by modifying existent object.
|
@@ -220,7 +201,7 @@ class TypeTransformer:
|
|
220
201
|
for e in normalizer.iter_errors(record):
|
221
202
|
"""
|
222
203
|
just calling normalizer.validate() would throw an exception on
|
223
|
-
first validation
|
204
|
+
first validation occurences and stop processing rest of schema.
|
224
205
|
"""
|
225
206
|
logger.warning(self.get_error_message(e))
|
226
207
|
|
@@ -23,7 +23,7 @@ from io import StringIO
|
|
23
23
|
from pathlib import Path
|
24
24
|
from typing import Any, List, Mapping, Optional, Union
|
25
25
|
|
26
|
-
import orjson
|
26
|
+
from orjson import orjson
|
27
27
|
from pydantic import ValidationError as V2ValidationError
|
28
28
|
from serpyco_rs import SchemaValidationError
|
29
29
|
|
@@ -63,7 +63,7 @@ class EntrypointOutput:
|
|
63
63
|
@staticmethod
|
64
64
|
def _parse_message(message: str) -> AirbyteMessage:
|
65
65
|
try:
|
66
|
-
return AirbyteMessageSerializer.load(orjson.loads(message))
|
66
|
+
return AirbyteMessageSerializer.load(orjson.loads(message)) # type: ignore[no-any-return] # Serializer.load() always returns AirbyteMessage
|
67
67
|
except (orjson.JSONDecodeError, SchemaValidationError):
|
68
68
|
# The platform assumes that logs that are not of AirbyteMessage format are log messages
|
69
69
|
return AirbyteMessage(
|
@@ -129,19 +129,14 @@ class EntrypointOutput:
|
|
129
129
|
return [
|
130
130
|
message
|
131
131
|
for message in self._get_message_by_types([Type.TRACE])
|
132
|
-
if message.trace.type == trace_type
|
133
|
-
]
|
132
|
+
if message.trace.type == trace_type
|
133
|
+
] # type: ignore[union-attr] # trace has `type`
|
134
134
|
|
135
135
|
def is_in_logs(self, pattern: str) -> bool:
|
136
136
|
"""Check if any log message case-insensitive matches the pattern."""
|
137
137
|
return any(
|
138
|
-
re.search(
|
139
|
-
|
140
|
-
entry.log.message, # type: ignore[union-attr] # log has `message`
|
141
|
-
flags=re.IGNORECASE,
|
142
|
-
)
|
143
|
-
for entry in self.logs
|
144
|
-
)
|
138
|
+
re.search(pattern, entry.log.message, flags=re.IGNORECASE) for entry in self.logs
|
139
|
+
) # type: ignore[union-attr] # log has `message`
|
145
140
|
|
146
141
|
def is_not_in_logs(self, pattern: str) -> bool:
|
147
142
|
"""Check if no log message matches the case-insensitive pattern."""
|