airbyte-cdk 6.8.1rc9__py3-none-any.whl → 6.8.2.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/cli/source_declarative_manifest/_run.py +11 -5
- airbyte_cdk/config_observation.py +1 -1
- airbyte_cdk/connector_builder/main.py +1 -1
- airbyte_cdk/connector_builder/message_grouper.py +10 -10
- airbyte_cdk/destinations/destination.py +1 -1
- airbyte_cdk/destinations/vector_db_based/embedder.py +2 -2
- airbyte_cdk/destinations/vector_db_based/writer.py +12 -4
- airbyte_cdk/entrypoint.py +7 -6
- airbyte_cdk/logger.py +2 -2
- airbyte_cdk/sources/abstract_source.py +1 -1
- airbyte_cdk/sources/config.py +1 -1
- airbyte_cdk/sources/connector_state_manager.py +9 -4
- airbyte_cdk/sources/declarative/auth/oauth.py +1 -1
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +6 -1
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +76 -28
- airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +10 -4
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +16 -17
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +4 -1
- airbyte_cdk/sources/declarative/extractors/record_filter.py +3 -5
- airbyte_cdk/sources/declarative/incremental/__init__.py +3 -0
- airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +270 -0
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +8 -6
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +9 -0
- airbyte_cdk/sources/declarative/interpolation/jinja.py +35 -36
- airbyte_cdk/sources/declarative/interpolation/macros.py +1 -1
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +71 -17
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +13 -7
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +1 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +8 -6
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +1 -1
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +2 -2
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +1 -1
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +5 -2
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +1 -1
- airbyte_cdk/sources/declarative/spec/spec.py +1 -1
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +0 -1
- airbyte_cdk/sources/embedded/base_integration.py +3 -2
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +12 -4
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +18 -7
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +14 -11
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +3 -3
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +11 -5
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +1 -1
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +2 -2
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +6 -3
- airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +1 -1
- airbyte_cdk/sources/http_logger.py +3 -3
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +5 -2
- airbyte_cdk/sources/streams/concurrent/adapters.py +6 -3
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +9 -3
- airbyte_cdk/sources/streams/concurrent/cursor.py +10 -1
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +2 -2
- airbyte_cdk/sources/streams/core.py +17 -14
- airbyte_cdk/sources/streams/http/http.py +19 -19
- airbyte_cdk/sources/streams/http/http_client.py +4 -48
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +62 -33
- airbyte_cdk/sources/utils/record_helper.py +1 -1
- airbyte_cdk/sources/utils/schema_helpers.py +1 -1
- airbyte_cdk/sources/utils/transform.py +34 -15
- airbyte_cdk/test/entrypoint_wrapper.py +11 -6
- airbyte_cdk/test/mock_http/response_builder.py +1 -1
- airbyte_cdk/utils/airbyte_secrets_utils.py +1 -1
- airbyte_cdk/utils/event_timing.py +10 -10
- airbyte_cdk/utils/message_utils.py +4 -3
- airbyte_cdk/utils/spec_schema_transformations.py +3 -2
- airbyte_cdk/utils/traced_exception.py +14 -12
- airbyte_cdk-6.8.2.dev1.dist-info/METADATA +111 -0
- {airbyte_cdk-6.8.1rc9.dist-info → airbyte_cdk-6.8.2.dev1.dist-info}/RECORD +72 -71
- airbyte_cdk-6.8.1rc9.dist-info/METADATA +0 -307
- {airbyte_cdk-6.8.1rc9.dist-info → airbyte_cdk-6.8.2.dev1.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.8.1rc9.dist-info → airbyte_cdk-6.8.2.dev1.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.8.1rc9.dist-info → airbyte_cdk-6.8.2.dev1.dist-info}/entry_points.txt +0 -0
@@ -10,7 +10,7 @@ from dataclasses import dataclass
|
|
10
10
|
from functools import cached_property, lru_cache
|
11
11
|
from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Union
|
12
12
|
|
13
|
-
from
|
13
|
+
from typing_extensions import deprecated
|
14
14
|
|
15
15
|
import airbyte_cdk.sources.utils.casing as casing
|
16
16
|
from airbyte_cdk.models import (
|
@@ -92,8 +92,8 @@ class CheckpointMixin(ABC):
|
|
92
92
|
|
93
93
|
|
94
94
|
@deprecated(
|
95
|
-
version
|
96
|
-
|
95
|
+
"Deprecated as of CDK version 0.87.0. "
|
96
|
+
"Deprecated in favor of the `CheckpointMixin` which offers similar functionality."
|
97
97
|
)
|
98
98
|
class IncrementalMixin(CheckpointMixin, ABC):
|
99
99
|
"""Mixin to make stream incremental.
|
@@ -115,12 +115,6 @@ class StreamClassification:
|
|
115
115
|
has_multiple_slices: bool
|
116
116
|
|
117
117
|
|
118
|
-
# Moved to class declaration since get_updated_state is called on every record for incremental syncs, and thus the @deprecated decorator as well.
|
119
|
-
@deprecated(
|
120
|
-
version="0.1.49",
|
121
|
-
reason="Deprecated method get_updated_state, You should use explicit state property instead, see IncrementalMixin docs.",
|
122
|
-
action="ignore",
|
123
|
-
)
|
124
118
|
class Stream(ABC):
|
125
119
|
"""
|
126
120
|
Base abstract class for an Airbyte Stream. Makes no assumption of the Stream's underlying transport protocol.
|
@@ -222,7 +216,8 @@ class Stream(ABC):
|
|
222
216
|
# Some connectors have streams that implement get_updated_state(), but do not define a cursor_field. This
|
223
217
|
# should be fixed on the stream implementation, but we should also protect against this in the CDK as well
|
224
218
|
stream_state_tracker = self.get_updated_state(
|
225
|
-
stream_state_tracker,
|
219
|
+
stream_state_tracker,
|
220
|
+
record_data, # type: ignore [arg-type]
|
226
221
|
)
|
227
222
|
self._observe_state(checkpoint_reader, stream_state_tracker)
|
228
223
|
record_counter += 1
|
@@ -282,7 +277,7 @@ class Stream(ABC):
|
|
282
277
|
if state
|
283
278
|
else {}, # read() expects MutableMapping instead of Mapping which is used more often
|
284
279
|
state_manager=None,
|
285
|
-
internal_config=InternalConfig(),
|
280
|
+
internal_config=InternalConfig(), # type: ignore [call-arg]
|
286
281
|
)
|
287
282
|
|
288
283
|
@abstractmethod
|
@@ -322,7 +317,7 @@ class Stream(ABC):
|
|
322
317
|
# If we can offer incremental we always should. RFR is always less reliable than incremental which uses a real cursor value
|
323
318
|
if self.supports_incremental:
|
324
319
|
stream.source_defined_cursor = self.source_defined_cursor
|
325
|
-
stream.supported_sync_modes.append(SyncMode.incremental)
|
320
|
+
stream.supported_sync_modes.append(SyncMode.incremental)
|
326
321
|
stream.default_cursor_field = self._wrapped_cursor_field()
|
327
322
|
|
328
323
|
keys = Stream._wrapped_primary_key(self.primary_key)
|
@@ -436,10 +431,18 @@ class Stream(ABC):
|
|
436
431
|
"""
|
437
432
|
return None
|
438
433
|
|
434
|
+
# Commented-out to avoid any runtime penalty, since this is used in a hot per-record codepath.
|
435
|
+
# To be evaluated for re-introduction here: https://github.com/airbytehq/airbyte-python-cdk/issues/116
|
436
|
+
# @deprecated(
|
437
|
+
# "Deprecated method `get_updated_state` as of CDK version 0.1.49. "
|
438
|
+
# "Please use explicit state property instead, see `IncrementalMixin` docs."
|
439
|
+
# )
|
439
440
|
def get_updated_state(
|
440
441
|
self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]
|
441
442
|
) -> MutableMapping[str, Any]:
|
442
|
-
"""
|
443
|
+
"""DEPRECATED. Please use explicit state property instead, see `IncrementalMixin` docs.
|
444
|
+
|
445
|
+
Override to extract state from the latest record. Needed to implement incremental sync.
|
443
446
|
|
444
447
|
Inspects the latest record extracted from the data source and the current state object and return an updated state object.
|
445
448
|
|
@@ -654,7 +657,7 @@ class Stream(ABC):
|
|
654
657
|
# todo: This can be consolidated into one ConnectorStateManager.update_and_create_state_message() method, but I want
|
655
658
|
# to reduce changes right now and this would span concurrent as well
|
656
659
|
state_manager.update_state_for_stream(self.name, self.namespace, stream_state)
|
657
|
-
return state_manager.create_state_message(self.name, self.namespace)
|
660
|
+
return state_manager.create_state_message(self.name, self.namespace) # type: ignore [no-any-return]
|
658
661
|
|
659
662
|
@property
|
660
663
|
def configured_json_schema(self) -> Optional[Dict[str, Any]]:
|
@@ -9,8 +9,8 @@ from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optio
|
|
9
9
|
from urllib.parse import urljoin
|
10
10
|
|
11
11
|
import requests
|
12
|
-
from deprecated import deprecated
|
13
12
|
from requests.auth import AuthBase
|
13
|
+
from typing_extensions import deprecated
|
14
14
|
|
15
15
|
from airbyte_cdk.models import AirbyteMessage, FailureType, SyncMode
|
16
16
|
from airbyte_cdk.models import Type as MessageType
|
@@ -121,8 +121,8 @@ class HttpStream(Stream, CheckpointMixin, ABC):
|
|
121
121
|
|
122
122
|
@property
|
123
123
|
@deprecated(
|
124
|
-
version
|
125
|
-
|
124
|
+
"Deprecated as of CDK version 3.0.0. "
|
125
|
+
"You should set error_handler explicitly in HttpStream.get_error_handler() instead."
|
126
126
|
)
|
127
127
|
def raise_on_http_errors(self) -> bool:
|
128
128
|
"""
|
@@ -132,8 +132,8 @@ class HttpStream(Stream, CheckpointMixin, ABC):
|
|
132
132
|
|
133
133
|
@property
|
134
134
|
@deprecated(
|
135
|
-
version
|
136
|
-
|
135
|
+
"Deprecated as of CDK version 3.0.0. "
|
136
|
+
"You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead."
|
137
137
|
)
|
138
138
|
def max_retries(self) -> Union[int, None]:
|
139
139
|
"""
|
@@ -143,8 +143,8 @@ class HttpStream(Stream, CheckpointMixin, ABC):
|
|
143
143
|
|
144
144
|
@property
|
145
145
|
@deprecated(
|
146
|
-
version
|
147
|
-
|
146
|
+
"Deprecated as of CDK version 3.0.0. "
|
147
|
+
"You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead."
|
148
148
|
)
|
149
149
|
def max_time(self) -> Union[int, None]:
|
150
150
|
"""
|
@@ -154,8 +154,8 @@ class HttpStream(Stream, CheckpointMixin, ABC):
|
|
154
154
|
|
155
155
|
@property
|
156
156
|
@deprecated(
|
157
|
-
version
|
158
|
-
|
157
|
+
"Deprecated as of CDK version 3.0.0. "
|
158
|
+
"You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead."
|
159
159
|
)
|
160
160
|
def retry_factor(self) -> float:
|
161
161
|
"""
|
@@ -594,7 +594,7 @@ class HttpSubStream(HttpStream, ABC):
|
|
594
594
|
# Skip non-records (eg AirbyteLogMessage)
|
595
595
|
if isinstance(parent_record, AirbyteMessage):
|
596
596
|
if parent_record.type == MessageType.RECORD:
|
597
|
-
parent_record = parent_record.record.data
|
597
|
+
parent_record = parent_record.record.data # type: ignore [assignment, union-attr] # Incorrect type for assignment
|
598
598
|
else:
|
599
599
|
continue
|
600
600
|
elif isinstance(parent_record, Record):
|
@@ -603,8 +603,8 @@ class HttpSubStream(HttpStream, ABC):
|
|
603
603
|
|
604
604
|
|
605
605
|
@deprecated(
|
606
|
-
version
|
607
|
-
|
606
|
+
"Deprecated as of CDK version 3.0.0."
|
607
|
+
"You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead."
|
608
608
|
)
|
609
609
|
class HttpStreamAdapterBackoffStrategy(BackoffStrategy):
|
610
610
|
def __init__(self, stream: HttpStream):
|
@@ -619,8 +619,8 @@ class HttpStreamAdapterBackoffStrategy(BackoffStrategy):
|
|
619
619
|
|
620
620
|
|
621
621
|
@deprecated(
|
622
|
-
version
|
623
|
-
|
622
|
+
"Deprecated as of CDK version 3.0.0. "
|
623
|
+
"You should set error_handler explicitly in HttpStream.get_error_handler() instead."
|
624
624
|
)
|
625
625
|
class HttpStreamAdapterHttpStatusErrorHandler(HttpStatusErrorHandler):
|
626
626
|
def __init__(self, stream: HttpStream, **kwargs): # type: ignore # noqa
|
@@ -639,15 +639,15 @@ class HttpStreamAdapterHttpStatusErrorHandler(HttpStatusErrorHandler):
|
|
639
639
|
return ErrorResolution(
|
640
640
|
response_action=ResponseAction.RATE_LIMITED,
|
641
641
|
failure_type=FailureType.transient_error,
|
642
|
-
error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
|
642
|
+
error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
|
643
643
|
)
|
644
644
|
return ErrorResolution(
|
645
645
|
response_action=ResponseAction.RETRY,
|
646
646
|
failure_type=FailureType.transient_error,
|
647
|
-
error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
|
647
|
+
error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
|
648
648
|
)
|
649
649
|
else:
|
650
|
-
if response_or_exception.ok:
|
650
|
+
if response_or_exception.ok:
|
651
651
|
return ErrorResolution(
|
652
652
|
response_action=ResponseAction.SUCCESS,
|
653
653
|
failure_type=None,
|
@@ -657,13 +657,13 @@ class HttpStreamAdapterHttpStatusErrorHandler(HttpStatusErrorHandler):
|
|
657
657
|
return ErrorResolution(
|
658
658
|
response_action=ResponseAction.FAIL,
|
659
659
|
failure_type=FailureType.transient_error,
|
660
|
-
error_message=f"Response status code: {response_or_exception.status_code}. Unexpected error. Failed.",
|
660
|
+
error_message=f"Response status code: {response_or_exception.status_code}. Unexpected error. Failed.",
|
661
661
|
)
|
662
662
|
else:
|
663
663
|
return ErrorResolution(
|
664
664
|
response_action=ResponseAction.IGNORE,
|
665
665
|
failure_type=FailureType.transient_error,
|
666
|
-
error_message=f"Response status code: {response_or_exception.status_code}. Ignoring...",
|
666
|
+
error_message=f"Response status code: {response_or_exception.status_code}. Ignoring...",
|
667
667
|
)
|
668
668
|
else:
|
669
669
|
self._logger.error(f"Received unexpected response type: {type(response_or_exception)}")
|
@@ -54,7 +54,6 @@ from airbyte_cdk.utils.stream_status_utils import (
|
|
54
54
|
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
55
55
|
|
56
56
|
BODY_REQUEST_METHODS = ("GET", "POST", "PUT", "PATCH")
|
57
|
-
logger = logging.getLogger("airbyte")
|
58
57
|
|
59
58
|
|
60
59
|
class MessageRepresentationAirbyteTracedErrors(AirbyteTracedException):
|
@@ -95,7 +94,6 @@ class HttpClient:
|
|
95
94
|
):
|
96
95
|
self._name = name
|
97
96
|
self._api_budget: APIBudget = api_budget or APIBudget(policies=[])
|
98
|
-
self._logger = logger
|
99
97
|
if session:
|
100
98
|
self._session = session
|
101
99
|
else:
|
@@ -109,6 +107,7 @@ class HttpClient:
|
|
109
107
|
)
|
110
108
|
if isinstance(authenticator, AuthBase):
|
111
109
|
self._session.auth = authenticator
|
110
|
+
self._logger = logger
|
112
111
|
self._error_handler = error_handler or HttpStatusErrorHandler(self._logger)
|
113
112
|
if backoff_strategy is not None:
|
114
113
|
if isinstance(backoff_strategy, list):
|
@@ -142,12 +141,10 @@ class HttpClient:
|
|
142
141
|
if cache_dir:
|
143
142
|
sqlite_path = str(Path(cache_dir) / self.cache_filename)
|
144
143
|
else:
|
145
|
-
self._logger.info("Using memory for cache") # TODO: remove
|
146
144
|
sqlite_path = "file::memory:?cache=shared"
|
147
|
-
backend = SkipFailureSQLiteCache(self._name, sqlite_path) # TODO maybe add a busy timeout
|
148
145
|
return CachedLimiterSession(
|
149
|
-
sqlite_path, backend=
|
150
|
-
)
|
146
|
+
sqlite_path, backend="sqlite", api_budget=self._api_budget, match_headers=True
|
147
|
+
)
|
151
148
|
else:
|
152
149
|
return LimiterSession(api_budget=self._api_budget)
|
153
150
|
|
@@ -327,7 +324,7 @@ class HttpClient:
|
|
327
324
|
formatter = log_formatter
|
328
325
|
self._message_repository.log_message(
|
329
326
|
Level.DEBUG,
|
330
|
-
lambda: formatter(response),
|
327
|
+
lambda: formatter(response),
|
331
328
|
)
|
332
329
|
|
333
330
|
self._handle_error_resolution(
|
@@ -520,44 +517,3 @@ class HttpClient:
|
|
520
517
|
)
|
521
518
|
|
522
519
|
return request, response
|
523
|
-
|
524
|
-
|
525
|
-
class SkipFailureSQLiteDict(requests_cache.backends.sqlite.SQLiteDict):
|
526
|
-
def __getitem__(self, key): # type: ignore # lib is not typed
|
527
|
-
try:
|
528
|
-
return super().__getitem__(key) # type: ignore # lib is not typed
|
529
|
-
except Exception as exception:
|
530
|
-
if not isinstance(exception, KeyError):
|
531
|
-
logger.warning(f"Error while retrieving item from cache: {exception}")
|
532
|
-
else:
|
533
|
-
raise exception
|
534
|
-
|
535
|
-
def _write(self, key: str, value: str) -> None:
|
536
|
-
try:
|
537
|
-
super()._write(key, value) # type: ignore # lib is not typed
|
538
|
-
except Exception as exception:
|
539
|
-
logger.warning(f"Error while saving item to cache: {exception}")
|
540
|
-
|
541
|
-
|
542
|
-
class SkipFailureSQLiteCache(requests_cache.backends.sqlite.SQLiteCache):
|
543
|
-
def __init__( # type: ignore # ignoring as lib is not typed
|
544
|
-
self,
|
545
|
-
table_name="response",
|
546
|
-
db_path="http_cache",
|
547
|
-
serializer=None,
|
548
|
-
**kwargs,
|
549
|
-
) -> None:
|
550
|
-
super().__init__(db_path, serializer, **kwargs)
|
551
|
-
skwargs = {"serializer": serializer, **kwargs} if serializer else kwargs
|
552
|
-
self.responses: requests_cache.backends.sqlite.SQLiteDict = SkipFailureSQLiteDict(
|
553
|
-
db_path, table_name=table_name, fast_save=True, wal=True, **skwargs
|
554
|
-
)
|
555
|
-
self.redirects: requests_cache.backends.sqlite.SQLiteDict = SkipFailureSQLiteDict(
|
556
|
-
db_path,
|
557
|
-
table_name=f"redirects_{table_name}",
|
558
|
-
fast_save=True,
|
559
|
-
wal=True,
|
560
|
-
lock=self.responses._lock,
|
561
|
-
serializer=None,
|
562
|
-
**kwargs,
|
563
|
-
)
|
@@ -5,13 +5,14 @@
|
|
5
5
|
from abc import abstractmethod
|
6
6
|
from typing import Any, Mapping
|
7
7
|
|
8
|
+
import requests
|
8
9
|
from requests.auth import AuthBase
|
9
10
|
|
10
11
|
|
11
12
|
class AbstractHeaderAuthenticator(AuthBase):
|
12
13
|
"""Abstract class for an header-based authenticators that add a header to outgoing HTTP requests."""
|
13
14
|
|
14
|
-
def __call__(self, request):
|
15
|
+
def __call__(self, request: requests.PreparedRequest) -> Any:
|
15
16
|
"""Attach the HTTP headers required to authenticate on the HTTP request"""
|
16
17
|
request.headers.update(self.get_auth_header())
|
17
18
|
return request
|
@@ -30,12 +30,12 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
|
|
30
30
|
client_id: str,
|
31
31
|
client_secret: str,
|
32
32
|
refresh_token: str,
|
33
|
-
scopes: List[str] = None,
|
34
|
-
token_expiry_date: pendulum.DateTime = None,
|
35
|
-
token_expiry_date_format: str = None,
|
33
|
+
scopes: List[str] | None = None,
|
34
|
+
token_expiry_date: pendulum.DateTime | None = None,
|
35
|
+
token_expiry_date_format: str | None = None,
|
36
36
|
access_token_name: str = "access_token",
|
37
37
|
expires_in_name: str = "expires_in",
|
38
|
-
refresh_request_body: Mapping[str, Any] = None,
|
38
|
+
refresh_request_body: Mapping[str, Any] | None = None,
|
39
39
|
grant_type: str = "refresh_token",
|
40
40
|
token_expiry_is_time_of_expiration: bool = False,
|
41
41
|
refresh_token_error_status_codes: Tuple[int, ...] = (),
|
@@ -52,7 +52,7 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
|
|
52
52
|
self._refresh_request_body = refresh_request_body
|
53
53
|
self._grant_type = grant_type
|
54
54
|
|
55
|
-
self._token_expiry_date = token_expiry_date or pendulum.now().subtract(days=1)
|
55
|
+
self._token_expiry_date = token_expiry_date or pendulum.now().subtract(days=1) # type: ignore [no-untyped-call]
|
56
56
|
self._token_expiry_date_format = token_expiry_date_format
|
57
57
|
self._token_expiry_is_time_of_expiration = token_expiry_is_time_of_expiration
|
58
58
|
self._access_token = None
|
@@ -75,14 +75,14 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
|
|
75
75
|
def get_access_token_name(self) -> str:
|
76
76
|
return self._access_token_name
|
77
77
|
|
78
|
-
def get_scopes(self) -> [str]:
|
79
|
-
return self._scopes
|
78
|
+
def get_scopes(self) -> list[str]:
|
79
|
+
return self._scopes # type: ignore [return-value]
|
80
80
|
|
81
81
|
def get_expires_in_name(self) -> str:
|
82
82
|
return self._expires_in_name
|
83
83
|
|
84
84
|
def get_refresh_request_body(self) -> Mapping[str, Any]:
|
85
|
-
return self._refresh_request_body
|
85
|
+
return self._refresh_request_body # type: ignore [return-value]
|
86
86
|
|
87
87
|
def get_grant_type(self) -> str:
|
88
88
|
return self._grant_type
|
@@ -90,7 +90,7 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
|
|
90
90
|
def get_token_expiry_date(self) -> pendulum.DateTime:
|
91
91
|
return self._token_expiry_date
|
92
92
|
|
93
|
-
def set_token_expiry_date(self, value: Union[str, int]):
|
93
|
+
def set_token_expiry_date(self, value: Union[str, int]) -> None:
|
94
94
|
self._token_expiry_date = self._parse_token_expiration_date(value)
|
95
95
|
|
96
96
|
@property
|
@@ -103,11 +103,11 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
|
|
103
103
|
|
104
104
|
@property
|
105
105
|
def access_token(self) -> str:
|
106
|
-
return self._access_token
|
106
|
+
return self._access_token # type: ignore [return-value]
|
107
107
|
|
108
108
|
@access_token.setter
|
109
|
-
def access_token(self, value: str):
|
110
|
-
self._access_token = value
|
109
|
+
def access_token(self, value: str) -> None:
|
110
|
+
self._access_token = value # type: ignore [assignment] # Incorrect type for assignment
|
111
111
|
|
112
112
|
|
113
113
|
class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
|
@@ -124,11 +124,11 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
|
|
124
124
|
self,
|
125
125
|
connector_config: Mapping[str, Any],
|
126
126
|
token_refresh_endpoint: str,
|
127
|
-
scopes: List[str] = None,
|
127
|
+
scopes: List[str] | None = None,
|
128
128
|
access_token_name: str = "access_token",
|
129
129
|
expires_in_name: str = "expires_in",
|
130
130
|
refresh_token_name: str = "refresh_token",
|
131
|
-
refresh_request_body: Mapping[str, Any] = None,
|
131
|
+
refresh_request_body: Mapping[str, Any] | None = None,
|
132
132
|
grant_type: str = "refresh_token",
|
133
133
|
client_id: Optional[str] = None,
|
134
134
|
client_secret: Optional[str] = None,
|
@@ -162,14 +162,17 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
|
|
162
162
|
message_repository (MessageRepository): the message repository used to emit logs on HTTP requests and control message on config update
|
163
163
|
"""
|
164
164
|
self._client_id = (
|
165
|
-
client_id
|
165
|
+
client_id # type: ignore [assignment] # Incorrect type for assignment
|
166
166
|
if client_id is not None
|
167
|
-
else dpath.get(connector_config, ("credentials", "client_id"))
|
167
|
+
else dpath.get(connector_config, ("credentials", "client_id")) # type: ignore [arg-type]
|
168
168
|
)
|
169
169
|
self._client_secret = (
|
170
|
-
client_secret
|
170
|
+
client_secret # type: ignore [assignment] # Incorrect type for assignment
|
171
171
|
if client_secret is not None
|
172
|
-
else dpath.get(
|
172
|
+
else dpath.get(
|
173
|
+
connector_config, # type: ignore [arg-type]
|
174
|
+
("credentials", "client_secret"),
|
175
|
+
)
|
173
176
|
)
|
174
177
|
self._access_token_config_path = access_token_config_path
|
175
178
|
self._refresh_token_config_path = refresh_token_config_path
|
@@ -207,27 +210,50 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
|
|
207
210
|
|
208
211
|
@property
|
209
212
|
def access_token(self) -> str:
|
210
|
-
return dpath.get(
|
213
|
+
return dpath.get( # type: ignore [return-value]
|
214
|
+
self._connector_config, # type: ignore [arg-type]
|
215
|
+
self._access_token_config_path,
|
216
|
+
default="",
|
217
|
+
)
|
211
218
|
|
212
219
|
@access_token.setter
|
213
|
-
def access_token(self, new_access_token: str):
|
214
|
-
dpath.new(
|
220
|
+
def access_token(self, new_access_token: str) -> None:
|
221
|
+
dpath.new(
|
222
|
+
self._connector_config, # type: ignore [arg-type]
|
223
|
+
self._access_token_config_path,
|
224
|
+
new_access_token,
|
225
|
+
)
|
215
226
|
|
216
227
|
def get_refresh_token(self) -> str:
|
217
|
-
return dpath.get(
|
228
|
+
return dpath.get( # type: ignore [return-value]
|
229
|
+
self._connector_config, # type: ignore [arg-type]
|
230
|
+
self._refresh_token_config_path,
|
231
|
+
default="",
|
232
|
+
)
|
218
233
|
|
219
|
-
def set_refresh_token(self, new_refresh_token: str):
|
220
|
-
dpath.new(
|
234
|
+
def set_refresh_token(self, new_refresh_token: str) -> None:
|
235
|
+
dpath.new(
|
236
|
+
self._connector_config, # type: ignore [arg-type]
|
237
|
+
self._refresh_token_config_path,
|
238
|
+
new_refresh_token,
|
239
|
+
)
|
221
240
|
|
222
241
|
def get_token_expiry_date(self) -> pendulum.DateTime:
|
223
242
|
expiry_date = dpath.get(
|
224
|
-
self._connector_config,
|
243
|
+
self._connector_config, # type: ignore [arg-type]
|
244
|
+
self._token_expiry_date_config_path,
|
245
|
+
default="",
|
225
246
|
)
|
226
|
-
return pendulum.now().subtract(days=1) if expiry_date == "" else pendulum.parse(expiry_date)
|
247
|
+
return pendulum.now().subtract(days=1) if expiry_date == "" else pendulum.parse(expiry_date) # type: ignore [arg-type, return-value, no-untyped-call]
|
227
248
|
|
228
|
-
def set_token_expiry_date(
|
249
|
+
def set_token_expiry_date( # type: ignore[override]
|
250
|
+
self,
|
251
|
+
new_token_expiry_date: pendulum.DateTime,
|
252
|
+
) -> None:
|
229
253
|
dpath.new(
|
230
|
-
self._connector_config,
|
254
|
+
self._connector_config, # type: ignore [arg-type]
|
255
|
+
self._token_expiry_date_config_path,
|
256
|
+
str(new_token_expiry_date),
|
231
257
|
)
|
232
258
|
|
233
259
|
def token_has_expired(self) -> bool:
|
@@ -236,7 +262,8 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
|
|
236
262
|
|
237
263
|
@staticmethod
|
238
264
|
def get_new_token_expiry_date(
|
239
|
-
access_token_expires_in: str,
|
265
|
+
access_token_expires_in: str,
|
266
|
+
token_expiry_date_format: str | None = None,
|
240
267
|
) -> pendulum.DateTime:
|
241
268
|
if token_expiry_date_format:
|
242
269
|
return pendulum.from_format(access_token_expires_in, token_expiry_date_format)
|
@@ -253,7 +280,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
|
|
253
280
|
new_access_token, access_token_expires_in, new_refresh_token = (
|
254
281
|
self.refresh_access_token()
|
255
282
|
)
|
256
|
-
new_token_expiry_date = self.get_new_token_expiry_date(
|
283
|
+
new_token_expiry_date: pendulum.DateTime = self.get_new_token_expiry_date(
|
257
284
|
access_token_expires_in, self._token_expiry_date_format
|
258
285
|
)
|
259
286
|
self.access_token = new_access_token
|
@@ -264,13 +291,15 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
|
|
264
291
|
# message directly in the console, this is needed
|
265
292
|
if not isinstance(self._message_repository, NoopMessageRepository):
|
266
293
|
self._message_repository.emit_message(
|
267
|
-
create_connector_config_control_message(self._connector_config)
|
294
|
+
create_connector_config_control_message(self._connector_config) # type: ignore [arg-type]
|
268
295
|
)
|
269
296
|
else:
|
270
|
-
emit_configuration_as_airbyte_control_message(self._connector_config)
|
297
|
+
emit_configuration_as_airbyte_control_message(self._connector_config) # type: ignore [arg-type]
|
271
298
|
return self.access_token
|
272
299
|
|
273
|
-
def refresh_access_token(
|
300
|
+
def refresh_access_token( # type: ignore[override] # Signature doesn't match base class
|
301
|
+
self,
|
302
|
+
) -> Tuple[str, str, str]:
|
274
303
|
response_json = self._get_refresh_access_token_response()
|
275
304
|
return (
|
276
305
|
response_json[self.get_access_token_name()],
|
@@ -35,7 +35,7 @@ def stream_data_to_airbyte_message(
|
|
35
35
|
# need it to normalize values against json schema. By default no action
|
36
36
|
# taken unless configured. See
|
37
37
|
# docs/connector-development/cdk-python/schemas.md for details.
|
38
|
-
transformer.transform(data, schema)
|
38
|
+
transformer.transform(data, schema)
|
39
39
|
if is_file_transfer_message:
|
40
40
|
message = AirbyteFileTransferRecordMessage(
|
41
41
|
stream=stream_name, file=data, emitted_at=now_millis, data={}
|
@@ -194,7 +194,7 @@ class InternalConfig(BaseModel):
|
|
194
194
|
def dict(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
|
195
195
|
kwargs["by_alias"] = True
|
196
196
|
kwargs["exclude_unset"] = True
|
197
|
-
return super().dict(*args, **kwargs)
|
197
|
+
return super().dict(*args, **kwargs)
|
198
198
|
|
199
199
|
def is_limit_reached(self, records_counter: int) -> bool:
|
200
200
|
"""
|
@@ -5,9 +5,9 @@
|
|
5
5
|
import logging
|
6
6
|
from distutils.util import strtobool
|
7
7
|
from enum import Flag, auto
|
8
|
-
from typing import Any, Callable, Dict, Mapping, Optional
|
8
|
+
from typing import Any, Callable, Dict, Generator, Mapping, Optional, cast
|
9
9
|
|
10
|
-
from jsonschema import Draft7Validator, ValidationError, validators
|
10
|
+
from jsonschema import Draft7Validator, RefResolver, ValidationError, Validator, validators
|
11
11
|
|
12
12
|
json_to_python_simple = {
|
13
13
|
"string": str,
|
@@ -30,7 +30,7 @@ class TransformConfig(Flag):
|
|
30
30
|
```
|
31
31
|
"""
|
32
32
|
|
33
|
-
# No action taken, default
|
33
|
+
# No action taken, default behavior. Cannot be combined with any other options.
|
34
34
|
NoTransform = auto()
|
35
35
|
# Applies default type casting with default_convert method which converts
|
36
36
|
# values by applying simple type casting to specified jsonschema type.
|
@@ -67,15 +67,15 @@ class TypeTransformer:
|
|
67
67
|
)
|
68
68
|
|
69
69
|
def registerCustomTransform(
|
70
|
-
self, normalization_callback: Callable[[Any,
|
71
|
-
) -> Callable:
|
70
|
+
self, normalization_callback: Callable[[Any, dict[str, Any]], Any]
|
71
|
+
) -> Callable[[Any, dict[str, Any]], Any]:
|
72
72
|
"""
|
73
73
|
Register custom normalization callback.
|
74
74
|
:param normalization_callback function to be used for value
|
75
75
|
normalization. Takes original value and part type schema. Should return
|
76
76
|
normalized value. See docs/connector-development/cdk-python/schemas.md
|
77
77
|
for details.
|
78
|
-
:return Same
|
78
|
+
:return Same callback, this is useful for using registerCustomTransform function as decorator.
|
79
79
|
"""
|
80
80
|
if TransformConfig.CustomSchemaNormalization not in self._config:
|
81
81
|
raise Exception(
|
@@ -141,7 +141,11 @@ class TypeTransformer:
|
|
141
141
|
return original_item
|
142
142
|
return original_item
|
143
143
|
|
144
|
-
def __get_normalizer(
|
144
|
+
def __get_normalizer(
|
145
|
+
self,
|
146
|
+
schema_key: str,
|
147
|
+
original_validator: Callable, # type: ignore[type-arg]
|
148
|
+
) -> Callable[[Any, Any, Any, dict[str, Any]], Generator[Any, Any, None]]:
|
145
149
|
"""
|
146
150
|
Traverse through object fields using native jsonschema validator and apply normalization function.
|
147
151
|
:param schema_key related json schema key that currently being validated/normalized.
|
@@ -149,8 +153,11 @@ class TypeTransformer:
|
|
149
153
|
"""
|
150
154
|
|
151
155
|
def normalizator(
|
152
|
-
validator_instance:
|
153
|
-
|
156
|
+
validator_instance: Validator,
|
157
|
+
property_value: Any,
|
158
|
+
instance: Any,
|
159
|
+
schema: Dict[str, Any],
|
160
|
+
) -> Generator[Any, Any, None]:
|
154
161
|
"""
|
155
162
|
Jsonschema validator callable it uses for validating instance. We
|
156
163
|
override default Draft7Validator to perform value transformation
|
@@ -163,10 +170,13 @@ class TypeTransformer:
|
|
163
170
|
:
|
164
171
|
"""
|
165
172
|
|
166
|
-
def resolve(subschema):
|
173
|
+
def resolve(subschema: dict[str, Any]) -> dict[str, Any]:
|
167
174
|
if "$ref" in subschema:
|
168
|
-
_, resolved =
|
169
|
-
|
175
|
+
_, resolved = cast(
|
176
|
+
RefResolver,
|
177
|
+
validator_instance.resolver,
|
178
|
+
).resolve(subschema["$ref"])
|
179
|
+
return cast(dict[str, Any], resolved)
|
170
180
|
return subschema
|
171
181
|
|
172
182
|
# Transform object and array values before running json schema type checking for each element.
|
@@ -185,11 +195,20 @@ class TypeTransformer:
|
|
185
195
|
instance[index] = self.__normalize(item, subschema)
|
186
196
|
|
187
197
|
# Running native jsonschema traverse algorithm after field normalization is done.
|
188
|
-
yield from original_validator(
|
198
|
+
yield from original_validator(
|
199
|
+
validator_instance,
|
200
|
+
property_value,
|
201
|
+
instance,
|
202
|
+
schema,
|
203
|
+
)
|
189
204
|
|
190
205
|
return normalizator
|
191
206
|
|
192
|
-
def transform(
|
207
|
+
def transform(
|
208
|
+
self,
|
209
|
+
record: Dict[str, Any],
|
210
|
+
schema: Mapping[str, Any],
|
211
|
+
) -> None:
|
193
212
|
"""
|
194
213
|
Normalize and validate according to config.
|
195
214
|
:param record: record instance for normalization/transformation. All modification are done by modifying existent object.
|
@@ -201,7 +220,7 @@ class TypeTransformer:
|
|
201
220
|
for e in normalizer.iter_errors(record):
|
202
221
|
"""
|
203
222
|
just calling normalizer.validate() would throw an exception on
|
204
|
-
first validation
|
223
|
+
first validation occurrences and stop processing rest of schema.
|
205
224
|
"""
|
206
225
|
logger.warning(self.get_error_message(e))
|
207
226
|
|