airbyte-cdk 6.8.1.dev1__py3-none-any.whl → 6.8.1rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. airbyte_cdk/cli/source_declarative_manifest/_run.py +5 -11
  2. airbyte_cdk/config_observation.py +1 -1
  3. airbyte_cdk/connector_builder/main.py +1 -1
  4. airbyte_cdk/connector_builder/message_grouper.py +10 -10
  5. airbyte_cdk/destinations/destination.py +1 -1
  6. airbyte_cdk/destinations/vector_db_based/embedder.py +2 -2
  7. airbyte_cdk/destinations/vector_db_based/writer.py +4 -12
  8. airbyte_cdk/entrypoint.py +6 -7
  9. airbyte_cdk/logger.py +2 -2
  10. airbyte_cdk/sources/abstract_source.py +1 -1
  11. airbyte_cdk/sources/config.py +1 -1
  12. airbyte_cdk/sources/connector_state_manager.py +4 -9
  13. airbyte_cdk/sources/declarative/auth/oauth.py +1 -1
  14. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +1 -6
  15. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +1 -1
  16. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +4 -10
  17. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +17 -16
  18. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +1 -4
  19. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +6 -8
  20. airbyte_cdk/sources/declarative/interpolation/jinja.py +3 -3
  21. airbyte_cdk/sources/declarative/interpolation/macros.py +1 -1
  22. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +6 -5
  23. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +7 -13
  24. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +1 -1
  25. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +6 -8
  26. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +1 -1
  27. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +2 -2
  28. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +1 -1
  29. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +2 -5
  30. airbyte_cdk/sources/declarative/spec/spec.py +1 -1
  31. airbyte_cdk/sources/embedded/base_integration.py +2 -3
  32. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +4 -12
  33. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +7 -18
  34. airbyte_cdk/sources/file_based/file_types/avro_parser.py +11 -14
  35. airbyte_cdk/sources/file_based/file_types/csv_parser.py +3 -3
  36. airbyte_cdk/sources/file_based/file_types/excel_parser.py +5 -11
  37. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +1 -1
  38. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +2 -2
  39. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +3 -6
  40. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +1 -1
  41. airbyte_cdk/sources/http_logger.py +3 -3
  42. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +2 -5
  43. airbyte_cdk/sources/streams/concurrent/adapters.py +3 -6
  44. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +3 -9
  45. airbyte_cdk/sources/streams/concurrent/cursor.py +1 -1
  46. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +2 -2
  47. airbyte_cdk/sources/streams/core.py +14 -17
  48. airbyte_cdk/sources/streams/http/http.py +19 -19
  49. airbyte_cdk/sources/streams/http/http_client.py +34 -3
  50. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +1 -2
  51. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +33 -62
  52. airbyte_cdk/sources/utils/record_helper.py +1 -1
  53. airbyte_cdk/sources/utils/schema_helpers.py +1 -1
  54. airbyte_cdk/sources/utils/transform.py +15 -34
  55. airbyte_cdk/test/entrypoint_wrapper.py +6 -11
  56. airbyte_cdk/test/mock_http/response_builder.py +1 -1
  57. airbyte_cdk/utils/airbyte_secrets_utils.py +1 -1
  58. airbyte_cdk/utils/event_timing.py +10 -10
  59. airbyte_cdk/utils/message_utils.py +3 -4
  60. airbyte_cdk/utils/spec_schema_transformations.py +2 -3
  61. airbyte_cdk/utils/traced_exception.py +12 -14
  62. {airbyte_cdk-6.8.1.dev1.dist-info → airbyte_cdk-6.8.1rc1.dist-info}/METADATA +2 -1
  63. {airbyte_cdk-6.8.1.dev1.dist-info → airbyte_cdk-6.8.1rc1.dist-info}/RECORD +66 -67
  64. airbyte_cdk/test/utils/manifest_only_fixtures.py +0 -40
  65. {airbyte_cdk-6.8.1.dev1.dist-info → airbyte_cdk-6.8.1rc1.dist-info}/LICENSE.txt +0 -0
  66. {airbyte_cdk-6.8.1.dev1.dist-info → airbyte_cdk-6.8.1rc1.dist-info}/WHEEL +0 -0
  67. {airbyte_cdk-6.8.1.dev1.dist-info → airbyte_cdk-6.8.1rc1.dist-info}/entry_points.txt +0 -0
@@ -10,7 +10,7 @@ from dataclasses import dataclass
10
10
  from functools import cached_property, lru_cache
11
11
  from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Union
12
12
 
13
- from typing_extensions import deprecated
13
+ from deprecated import deprecated
14
14
 
15
15
  import airbyte_cdk.sources.utils.casing as casing
16
16
  from airbyte_cdk.models import (
@@ -92,8 +92,8 @@ class CheckpointMixin(ABC):
92
92
 
93
93
 
94
94
  @deprecated(
95
- "Deprecated as of CDK version 0.87.0. "
96
- "Deprecated in favor of the `CheckpointMixin` which offers similar functionality."
95
+ version="0.87.0",
96
+ reason="Deprecated in favor of the CheckpointMixin which offers similar functionality",
97
97
  )
98
98
  class IncrementalMixin(CheckpointMixin, ABC):
99
99
  """Mixin to make stream incremental.
@@ -115,6 +115,12 @@ class StreamClassification:
115
115
  has_multiple_slices: bool
116
116
 
117
117
 
118
+ # Moved to class declaration since get_updated_state is called on every record for incremental syncs, and thus the @deprecated decorator as well.
119
+ @deprecated(
120
+ version="0.1.49",
121
+ reason="Deprecated method get_updated_state, You should use explicit state property instead, see IncrementalMixin docs.",
122
+ action="ignore",
123
+ )
118
124
  class Stream(ABC):
119
125
  """
120
126
  Base abstract class for an Airbyte Stream. Makes no assumption of the Stream's underlying transport protocol.
@@ -216,8 +222,7 @@ class Stream(ABC):
216
222
  # Some connectors have streams that implement get_updated_state(), but do not define a cursor_field. This
217
223
  # should be fixed on the stream implementation, but we should also protect against this in the CDK as well
218
224
  stream_state_tracker = self.get_updated_state(
219
- stream_state_tracker,
220
- record_data, # type: ignore [arg-type]
225
+ stream_state_tracker, record_data
221
226
  )
222
227
  self._observe_state(checkpoint_reader, stream_state_tracker)
223
228
  record_counter += 1
@@ -277,7 +282,7 @@ class Stream(ABC):
277
282
  if state
278
283
  else {}, # read() expects MutableMapping instead of Mapping which is used more often
279
284
  state_manager=None,
280
- internal_config=InternalConfig(), # type: ignore [call-arg]
285
+ internal_config=InternalConfig(),
281
286
  )
282
287
 
283
288
  @abstractmethod
@@ -317,7 +322,7 @@ class Stream(ABC):
317
322
  # If we can offer incremental we always should. RFR is always less reliable than incremental which uses a real cursor value
318
323
  if self.supports_incremental:
319
324
  stream.source_defined_cursor = self.source_defined_cursor
320
- stream.supported_sync_modes.append(SyncMode.incremental)
325
+ stream.supported_sync_modes.append(SyncMode.incremental) # type: ignore
321
326
  stream.default_cursor_field = self._wrapped_cursor_field()
322
327
 
323
328
  keys = Stream._wrapped_primary_key(self.primary_key)
@@ -431,18 +436,10 @@ class Stream(ABC):
431
436
  """
432
437
  return None
433
438
 
434
- # Commented-out to avoid any runtime penalty, since this is used in a hot per-record codepath.
435
- # To be evaluated for re-introduction here: https://github.com/airbytehq/airbyte-python-cdk/issues/116
436
- # @deprecated(
437
- # "Deprecated method `get_updated_state` as of CDK version 0.1.49. "
438
- # "Please use explicit state property instead, see `IncrementalMixin` docs."
439
- # )
440
439
  def get_updated_state(
441
440
  self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]
442
441
  ) -> MutableMapping[str, Any]:
443
- """DEPRECATED. Please use explicit state property instead, see `IncrementalMixin` docs.
444
-
445
- Override to extract state from the latest record. Needed to implement incremental sync.
442
+ """Override to extract state from the latest record. Needed to implement incremental sync.
446
443
 
447
444
  Inspects the latest record extracted from the data source and the current state object and return an updated state object.
448
445
 
@@ -657,7 +654,7 @@ class Stream(ABC):
657
654
  # todo: This can be consolidated into one ConnectorStateManager.update_and_create_state_message() method, but I want
658
655
  # to reduce changes right now and this would span concurrent as well
659
656
  state_manager.update_state_for_stream(self.name, self.namespace, stream_state)
660
- return state_manager.create_state_message(self.name, self.namespace) # type: ignore [no-any-return]
657
+ return state_manager.create_state_message(self.name, self.namespace)
661
658
 
662
659
  @property
663
660
  def configured_json_schema(self) -> Optional[Dict[str, Any]]:
@@ -9,8 +9,8 @@ from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optio
9
9
  from urllib.parse import urljoin
10
10
 
11
11
  import requests
12
+ from deprecated import deprecated
12
13
  from requests.auth import AuthBase
13
- from typing_extensions import deprecated
14
14
 
15
15
  from airbyte_cdk.models import AirbyteMessage, FailureType, SyncMode
16
16
  from airbyte_cdk.models import Type as MessageType
@@ -121,8 +121,8 @@ class HttpStream(Stream, CheckpointMixin, ABC):
121
121
 
122
122
  @property
123
123
  @deprecated(
124
- "Deprecated as of CDK version 3.0.0. "
125
- "You should set error_handler explicitly in HttpStream.get_error_handler() instead."
124
+ version="3.0.0",
125
+ reason="You should set error_handler explicitly in HttpStream.get_error_handler() instead.",
126
126
  )
127
127
  def raise_on_http_errors(self) -> bool:
128
128
  """
@@ -132,8 +132,8 @@ class HttpStream(Stream, CheckpointMixin, ABC):
132
132
 
133
133
  @property
134
134
  @deprecated(
135
- "Deprecated as of CDK version 3.0.0. "
136
- "You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead."
135
+ version="3.0.0",
136
+ reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.",
137
137
  )
138
138
  def max_retries(self) -> Union[int, None]:
139
139
  """
@@ -143,8 +143,8 @@ class HttpStream(Stream, CheckpointMixin, ABC):
143
143
 
144
144
  @property
145
145
  @deprecated(
146
- "Deprecated as of CDK version 3.0.0. "
147
- "You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead."
146
+ version="3.0.0",
147
+ reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.",
148
148
  )
149
149
  def max_time(self) -> Union[int, None]:
150
150
  """
@@ -154,8 +154,8 @@ class HttpStream(Stream, CheckpointMixin, ABC):
154
154
 
155
155
  @property
156
156
  @deprecated(
157
- "Deprecated as of CDK version 3.0.0. "
158
- "You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead."
157
+ version="3.0.0",
158
+ reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.",
159
159
  )
160
160
  def retry_factor(self) -> float:
161
161
  """
@@ -594,7 +594,7 @@ class HttpSubStream(HttpStream, ABC):
594
594
  # Skip non-records (eg AirbyteLogMessage)
595
595
  if isinstance(parent_record, AirbyteMessage):
596
596
  if parent_record.type == MessageType.RECORD:
597
- parent_record = parent_record.record.data # type: ignore [assignment, union-attr] # Incorrect type for assignment
597
+ parent_record = parent_record.record.data
598
598
  else:
599
599
  continue
600
600
  elif isinstance(parent_record, Record):
@@ -603,8 +603,8 @@ class HttpSubStream(HttpStream, ABC):
603
603
 
604
604
 
605
605
  @deprecated(
606
- "Deprecated as of CDK version 3.0.0."
607
- "You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead."
606
+ version="3.0.0",
607
+ reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.",
608
608
  )
609
609
  class HttpStreamAdapterBackoffStrategy(BackoffStrategy):
610
610
  def __init__(self, stream: HttpStream):
@@ -619,8 +619,8 @@ class HttpStreamAdapterBackoffStrategy(BackoffStrategy):
619
619
 
620
620
 
621
621
  @deprecated(
622
- "Deprecated as of CDK version 3.0.0. "
623
- "You should set error_handler explicitly in HttpStream.get_error_handler() instead."
622
+ version="3.0.0",
623
+ reason="You should set error_handler explicitly in HttpStream.get_error_handler() instead.",
624
624
  )
625
625
  class HttpStreamAdapterHttpStatusErrorHandler(HttpStatusErrorHandler):
626
626
  def __init__(self, stream: HttpStream, **kwargs): # type: ignore # noqa
@@ -639,15 +639,15 @@ class HttpStreamAdapterHttpStatusErrorHandler(HttpStatusErrorHandler):
639
639
  return ErrorResolution(
640
640
  response_action=ResponseAction.RATE_LIMITED,
641
641
  failure_type=FailureType.transient_error,
642
- error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
642
+ error_message=f"Response status code: {response_or_exception.status_code}. Retrying...", # type: ignore[union-attr]
643
643
  )
644
644
  return ErrorResolution(
645
645
  response_action=ResponseAction.RETRY,
646
646
  failure_type=FailureType.transient_error,
647
- error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
647
+ error_message=f"Response status code: {response_or_exception.status_code}. Retrying...", # type: ignore[union-attr]
648
648
  )
649
649
  else:
650
- if response_or_exception.ok:
650
+ if response_or_exception.ok: # type: ignore # noqa
651
651
  return ErrorResolution(
652
652
  response_action=ResponseAction.SUCCESS,
653
653
  failure_type=None,
@@ -657,13 +657,13 @@ class HttpStreamAdapterHttpStatusErrorHandler(HttpStatusErrorHandler):
657
657
  return ErrorResolution(
658
658
  response_action=ResponseAction.FAIL,
659
659
  failure_type=FailureType.transient_error,
660
- error_message=f"Response status code: {response_or_exception.status_code}. Unexpected error. Failed.",
660
+ error_message=f"Response status code: {response_or_exception.status_code}. Unexpected error. Failed.", # type: ignore[union-attr]
661
661
  )
662
662
  else:
663
663
  return ErrorResolution(
664
664
  response_action=ResponseAction.IGNORE,
665
665
  failure_type=FailureType.transient_error,
666
- error_message=f"Response status code: {response_or_exception.status_code}. Ignoring...",
666
+ error_message=f"Response status code: {response_or_exception.status_code}. Ignoring...", # type: ignore[union-attr]
667
667
  )
668
668
  else:
669
669
  self._logger.error(f"Received unexpected response type: {type(response_or_exception)}")
@@ -54,6 +54,7 @@ from airbyte_cdk.utils.stream_status_utils import (
54
54
  from airbyte_cdk.utils.traced_exception import AirbyteTracedException
55
55
 
56
56
  BODY_REQUEST_METHODS = ("GET", "POST", "PUT", "PATCH")
57
+ logger = logging.getLogger("airbyte")
57
58
 
58
59
 
59
60
  class MessageRepresentationAirbyteTracedErrors(AirbyteTracedException):
@@ -142,9 +143,10 @@ class HttpClient:
142
143
  sqlite_path = str(Path(cache_dir) / self.cache_filename)
143
144
  else:
144
145
  sqlite_path = "file::memory:?cache=shared"
146
+ backend = SkipFailureSQLiteCache(sqlite_path)
145
147
  return CachedLimiterSession(
146
- sqlite_path, backend="sqlite", api_budget=self._api_budget, match_headers=True
147
- )
148
+ sqlite_path, backend=backend, api_budget=self._api_budget, match_headers=True
149
+ ) # type: ignore # there are no typeshed stubs for requests_cache
148
150
  else:
149
151
  return LimiterSession(api_budget=self._api_budget)
150
152
 
@@ -324,7 +326,7 @@ class HttpClient:
324
326
  formatter = log_formatter
325
327
  self._message_repository.log_message(
326
328
  Level.DEBUG,
327
- lambda: formatter(response),
329
+ lambda: formatter(response), # type: ignore # log_formatter is always cast to a callable
328
330
  )
329
331
 
330
332
  self._handle_error_resolution(
@@ -517,3 +519,32 @@ class HttpClient:
517
519
  )
518
520
 
519
521
  return request, response
522
+
523
+
524
+ class SkipFailureSQLiteDict(requests_cache.backends.sqlite.SQLiteDict):
525
+ def _write(self, key: str, value: str) -> None:
526
+ try:
527
+ super()._write(key, value) # type: ignore # lib is not typed
528
+ except Exception as exception:
529
+ logger.warning(exception)
530
+
531
+
532
+ class SkipFailureSQLiteCache(requests_cache.backends.sqlite.SQLiteCache):
533
+ def __init__( # type: ignore # ignoring as lib is not typed
534
+ self,
535
+ db_path="http_cache",
536
+ serializer=None,
537
+ **kwargs,
538
+ ) -> None:
539
+ super().__init__(db_path, serializer, **kwargs)
540
+ skwargs = {"serializer": serializer, **kwargs} if serializer else kwargs
541
+ self.responses: requests_cache.backends.sqlite.SQLiteDict = SkipFailureSQLiteDict(
542
+ db_path, table_name="responses", **skwargs
543
+ )
544
+ self.redirects: requests_cache.backends.sqlite.SQLiteDict = SkipFailureSQLiteDict(
545
+ db_path,
546
+ table_name="redirects",
547
+ lock=self.responses._lock,
548
+ serializer=None,
549
+ **kwargs,
550
+ )
@@ -5,14 +5,13 @@
5
5
  from abc import abstractmethod
6
6
  from typing import Any, Mapping
7
7
 
8
- import requests
9
8
  from requests.auth import AuthBase
10
9
 
11
10
 
12
11
  class AbstractHeaderAuthenticator(AuthBase):
13
12
  """Abstract class for an header-based authenticators that add a header to outgoing HTTP requests."""
14
13
 
15
- def __call__(self, request: requests.PreparedRequest) -> Any:
14
+ def __call__(self, request):
16
15
  """Attach the HTTP headers required to authenticate on the HTTP request"""
17
16
  request.headers.update(self.get_auth_header())
18
17
  return request
@@ -30,12 +30,12 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
30
30
  client_id: str,
31
31
  client_secret: str,
32
32
  refresh_token: str,
33
- scopes: List[str] | None = None,
34
- token_expiry_date: pendulum.DateTime | None = None,
35
- token_expiry_date_format: str | None = None,
33
+ scopes: List[str] = None,
34
+ token_expiry_date: pendulum.DateTime = None,
35
+ token_expiry_date_format: str = None,
36
36
  access_token_name: str = "access_token",
37
37
  expires_in_name: str = "expires_in",
38
- refresh_request_body: Mapping[str, Any] | None = None,
38
+ refresh_request_body: Mapping[str, Any] = None,
39
39
  grant_type: str = "refresh_token",
40
40
  token_expiry_is_time_of_expiration: bool = False,
41
41
  refresh_token_error_status_codes: Tuple[int, ...] = (),
@@ -52,7 +52,7 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
52
52
  self._refresh_request_body = refresh_request_body
53
53
  self._grant_type = grant_type
54
54
 
55
- self._token_expiry_date = token_expiry_date or pendulum.now().subtract(days=1) # type: ignore [no-untyped-call]
55
+ self._token_expiry_date = token_expiry_date or pendulum.now().subtract(days=1)
56
56
  self._token_expiry_date_format = token_expiry_date_format
57
57
  self._token_expiry_is_time_of_expiration = token_expiry_is_time_of_expiration
58
58
  self._access_token = None
@@ -75,14 +75,14 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
75
75
  def get_access_token_name(self) -> str:
76
76
  return self._access_token_name
77
77
 
78
- def get_scopes(self) -> list[str]:
79
- return self._scopes # type: ignore [return-value]
78
+ def get_scopes(self) -> [str]:
79
+ return self._scopes
80
80
 
81
81
  def get_expires_in_name(self) -> str:
82
82
  return self._expires_in_name
83
83
 
84
84
  def get_refresh_request_body(self) -> Mapping[str, Any]:
85
- return self._refresh_request_body # type: ignore [return-value]
85
+ return self._refresh_request_body
86
86
 
87
87
  def get_grant_type(self) -> str:
88
88
  return self._grant_type
@@ -90,7 +90,7 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
90
90
  def get_token_expiry_date(self) -> pendulum.DateTime:
91
91
  return self._token_expiry_date
92
92
 
93
- def set_token_expiry_date(self, value: Union[str, int]) -> None:
93
+ def set_token_expiry_date(self, value: Union[str, int]):
94
94
  self._token_expiry_date = self._parse_token_expiration_date(value)
95
95
 
96
96
  @property
@@ -103,11 +103,11 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
103
103
 
104
104
  @property
105
105
  def access_token(self) -> str:
106
- return self._access_token # type: ignore [return-value]
106
+ return self._access_token
107
107
 
108
108
  @access_token.setter
109
- def access_token(self, value: str) -> None:
110
- self._access_token = value # type: ignore [assignment] # Incorrect type for assignment
109
+ def access_token(self, value: str):
110
+ self._access_token = value
111
111
 
112
112
 
113
113
  class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
@@ -124,11 +124,11 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
124
124
  self,
125
125
  connector_config: Mapping[str, Any],
126
126
  token_refresh_endpoint: str,
127
- scopes: List[str] | None = None,
127
+ scopes: List[str] = None,
128
128
  access_token_name: str = "access_token",
129
129
  expires_in_name: str = "expires_in",
130
130
  refresh_token_name: str = "refresh_token",
131
- refresh_request_body: Mapping[str, Any] | None = None,
131
+ refresh_request_body: Mapping[str, Any] = None,
132
132
  grant_type: str = "refresh_token",
133
133
  client_id: Optional[str] = None,
134
134
  client_secret: Optional[str] = None,
@@ -162,17 +162,14 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
162
162
  message_repository (MessageRepository): the message repository used to emit logs on HTTP requests and control message on config update
163
163
  """
164
164
  self._client_id = (
165
- client_id # type: ignore [assignment] # Incorrect type for assignment
165
+ client_id
166
166
  if client_id is not None
167
- else dpath.get(connector_config, ("credentials", "client_id")) # type: ignore [arg-type]
167
+ else dpath.get(connector_config, ("credentials", "client_id"))
168
168
  )
169
169
  self._client_secret = (
170
- client_secret # type: ignore [assignment] # Incorrect type for assignment
170
+ client_secret
171
171
  if client_secret is not None
172
- else dpath.get(
173
- connector_config, # type: ignore [arg-type]
174
- ("credentials", "client_secret"),
175
- )
172
+ else dpath.get(connector_config, ("credentials", "client_secret"))
176
173
  )
177
174
  self._access_token_config_path = access_token_config_path
178
175
  self._refresh_token_config_path = refresh_token_config_path
@@ -210,50 +207,27 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
210
207
 
211
208
  @property
212
209
  def access_token(self) -> str:
213
- return dpath.get( # type: ignore [return-value]
214
- self._connector_config, # type: ignore [arg-type]
215
- self._access_token_config_path,
216
- default="",
217
- )
210
+ return dpath.get(self._connector_config, self._access_token_config_path, default="")
218
211
 
219
212
  @access_token.setter
220
- def access_token(self, new_access_token: str) -> None:
221
- dpath.new(
222
- self._connector_config, # type: ignore [arg-type]
223
- self._access_token_config_path,
224
- new_access_token,
225
- )
213
+ def access_token(self, new_access_token: str):
214
+ dpath.new(self._connector_config, self._access_token_config_path, new_access_token)
226
215
 
227
216
  def get_refresh_token(self) -> str:
228
- return dpath.get( # type: ignore [return-value]
229
- self._connector_config, # type: ignore [arg-type]
230
- self._refresh_token_config_path,
231
- default="",
232
- )
217
+ return dpath.get(self._connector_config, self._refresh_token_config_path, default="")
233
218
 
234
- def set_refresh_token(self, new_refresh_token: str) -> None:
235
- dpath.new(
236
- self._connector_config, # type: ignore [arg-type]
237
- self._refresh_token_config_path,
238
- new_refresh_token,
239
- )
219
+ def set_refresh_token(self, new_refresh_token: str):
220
+ dpath.new(self._connector_config, self._refresh_token_config_path, new_refresh_token)
240
221
 
241
222
  def get_token_expiry_date(self) -> pendulum.DateTime:
242
223
  expiry_date = dpath.get(
243
- self._connector_config, # type: ignore [arg-type]
244
- self._token_expiry_date_config_path,
245
- default="",
224
+ self._connector_config, self._token_expiry_date_config_path, default=""
246
225
  )
247
- return pendulum.now().subtract(days=1) if expiry_date == "" else pendulum.parse(expiry_date) # type: ignore [arg-type, return-value, no-untyped-call]
226
+ return pendulum.now().subtract(days=1) if expiry_date == "" else pendulum.parse(expiry_date)
248
227
 
249
- def set_token_expiry_date( # type: ignore[override]
250
- self,
251
- new_token_expiry_date: pendulum.DateTime,
252
- ) -> None:
228
+ def set_token_expiry_date(self, new_token_expiry_date):
253
229
  dpath.new(
254
- self._connector_config, # type: ignore [arg-type]
255
- self._token_expiry_date_config_path,
256
- str(new_token_expiry_date),
230
+ self._connector_config, self._token_expiry_date_config_path, str(new_token_expiry_date)
257
231
  )
258
232
 
259
233
  def token_has_expired(self) -> bool:
@@ -262,8 +236,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
262
236
 
263
237
  @staticmethod
264
238
  def get_new_token_expiry_date(
265
- access_token_expires_in: str,
266
- token_expiry_date_format: str | None = None,
239
+ access_token_expires_in: str, token_expiry_date_format: str = None
267
240
  ) -> pendulum.DateTime:
268
241
  if token_expiry_date_format:
269
242
  return pendulum.from_format(access_token_expires_in, token_expiry_date_format)
@@ -280,7 +253,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
280
253
  new_access_token, access_token_expires_in, new_refresh_token = (
281
254
  self.refresh_access_token()
282
255
  )
283
- new_token_expiry_date: pendulum.DateTime = self.get_new_token_expiry_date(
256
+ new_token_expiry_date = self.get_new_token_expiry_date(
284
257
  access_token_expires_in, self._token_expiry_date_format
285
258
  )
286
259
  self.access_token = new_access_token
@@ -291,15 +264,13 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
291
264
  # message directly in the console, this is needed
292
265
  if not isinstance(self._message_repository, NoopMessageRepository):
293
266
  self._message_repository.emit_message(
294
- create_connector_config_control_message(self._connector_config) # type: ignore [arg-type]
267
+ create_connector_config_control_message(self._connector_config)
295
268
  )
296
269
  else:
297
- emit_configuration_as_airbyte_control_message(self._connector_config) # type: ignore [arg-type]
270
+ emit_configuration_as_airbyte_control_message(self._connector_config)
298
271
  return self.access_token
299
272
 
300
- def refresh_access_token( # type: ignore[override] # Signature doesn't match base class
301
- self,
302
- ) -> Tuple[str, str, str]:
273
+ def refresh_access_token(self) -> Tuple[str, str, str]:
303
274
  response_json = self._get_refresh_access_token_response()
304
275
  return (
305
276
  response_json[self.get_access_token_name()],
@@ -35,7 +35,7 @@ def stream_data_to_airbyte_message(
35
35
  # need it to normalize values against json schema. By default no action
36
36
  # taken unless configured. See
37
37
  # docs/connector-development/cdk-python/schemas.md for details.
38
- transformer.transform(data, schema)
38
+ transformer.transform(data, schema) # type: ignore
39
39
  if is_file_transfer_message:
40
40
  message = AirbyteFileTransferRecordMessage(
41
41
  stream=stream_name, file=data, emitted_at=now_millis, data={}
@@ -194,7 +194,7 @@ class InternalConfig(BaseModel):
194
194
  def dict(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
195
195
  kwargs["by_alias"] = True
196
196
  kwargs["exclude_unset"] = True
197
- return super().dict(*args, **kwargs)
197
+ return super().dict(*args, **kwargs) # type: ignore[no-any-return]
198
198
 
199
199
  def is_limit_reached(self, records_counter: int) -> bool:
200
200
  """
@@ -5,9 +5,9 @@
5
5
  import logging
6
6
  from distutils.util import strtobool
7
7
  from enum import Flag, auto
8
- from typing import Any, Callable, Dict, Generator, Mapping, Optional, cast
8
+ from typing import Any, Callable, Dict, Mapping, Optional
9
9
 
10
- from jsonschema import Draft7Validator, RefResolver, ValidationError, Validator, validators
10
+ from jsonschema import Draft7Validator, ValidationError, validators
11
11
 
12
12
  json_to_python_simple = {
13
13
  "string": str,
@@ -30,7 +30,7 @@ class TransformConfig(Flag):
30
30
  ```
31
31
  """
32
32
 
33
- # No action taken, default behavior. Cannot be combined with any other options.
33
+ # No action taken, default behaviour. Cannot be combined with any other options.
34
34
  NoTransform = auto()
35
35
  # Applies default type casting with default_convert method which converts
36
36
  # values by applying simple type casting to specified jsonschema type.
@@ -67,15 +67,15 @@ class TypeTransformer:
67
67
  )
68
68
 
69
69
  def registerCustomTransform(
70
- self, normalization_callback: Callable[[Any, dict[str, Any]], Any]
71
- ) -> Callable[[Any, dict[str, Any]], Any]:
70
+ self, normalization_callback: Callable[[Any, Dict[str, Any]], Any]
71
+ ) -> Callable:
72
72
  """
73
73
  Register custom normalization callback.
74
74
  :param normalization_callback function to be used for value
75
75
  normalization. Takes original value and part type schema. Should return
76
76
  normalized value. See docs/connector-development/cdk-python/schemas.md
77
77
  for details.
78
- :return Same callback, this is useful for using registerCustomTransform function as decorator.
78
+ :return Same callbeck, this is usefull for using registerCustomTransform function as decorator.
79
79
  """
80
80
  if TransformConfig.CustomSchemaNormalization not in self._config:
81
81
  raise Exception(
@@ -141,11 +141,7 @@ class TypeTransformer:
141
141
  return original_item
142
142
  return original_item
143
143
 
144
- def __get_normalizer(
145
- self,
146
- schema_key: str,
147
- original_validator: Callable, # type: ignore[type-arg]
148
- ) -> Callable[[Any, Any, Any, dict[str, Any]], Generator[Any, Any, None]]:
144
+ def __get_normalizer(self, schema_key: str, original_validator: Callable):
149
145
  """
150
146
  Traverse through object fields using native jsonschema validator and apply normalization function.
151
147
  :param schema_key related json schema key that currently being validated/normalized.
@@ -153,11 +149,8 @@ class TypeTransformer:
153
149
  """
154
150
 
155
151
  def normalizator(
156
- validator_instance: Validator,
157
- property_value: Any,
158
- instance: Any,
159
- schema: Dict[str, Any],
160
- ) -> Generator[Any, Any, None]:
152
+ validator_instance: Callable, property_value: Any, instance: Any, schema: Dict[str, Any]
153
+ ):
161
154
  """
162
155
  Jsonschema validator callable it uses for validating instance. We
163
156
  override default Draft7Validator to perform value transformation
@@ -170,13 +163,10 @@ class TypeTransformer:
170
163
  :
171
164
  """
172
165
 
173
- def resolve(subschema: dict[str, Any]) -> dict[str, Any]:
166
+ def resolve(subschema):
174
167
  if "$ref" in subschema:
175
- _, resolved = cast(
176
- RefResolver,
177
- validator_instance.resolver,
178
- ).resolve(subschema["$ref"])
179
- return cast(dict[str, Any], resolved)
168
+ _, resolved = validator_instance.resolver.resolve(subschema["$ref"])
169
+ return resolved
180
170
  return subschema
181
171
 
182
172
  # Transform object and array values before running json schema type checking for each element.
@@ -195,20 +185,11 @@ class TypeTransformer:
195
185
  instance[index] = self.__normalize(item, subschema)
196
186
 
197
187
  # Running native jsonschema traverse algorithm after field normalization is done.
198
- yield from original_validator(
199
- validator_instance,
200
- property_value,
201
- instance,
202
- schema,
203
- )
188
+ yield from original_validator(validator_instance, property_value, instance, schema)
204
189
 
205
190
  return normalizator
206
191
 
207
- def transform(
208
- self,
209
- record: Dict[str, Any],
210
- schema: Mapping[str, Any],
211
- ) -> None:
192
+ def transform(self, record: Dict[str, Any], schema: Mapping[str, Any]):
212
193
  """
213
194
  Normalize and validate according to config.
214
195
  :param record: record instance for normalization/transformation. All modification are done by modifying existent object.
@@ -220,7 +201,7 @@ class TypeTransformer:
220
201
  for e in normalizer.iter_errors(record):
221
202
  """
222
203
  just calling normalizer.validate() would throw an exception on
223
- first validation occurrences and stop processing rest of schema.
204
+ first validation occurences and stop processing rest of schema.
224
205
  """
225
206
  logger.warning(self.get_error_message(e))
226
207
 
@@ -23,7 +23,7 @@ from io import StringIO
23
23
  from pathlib import Path
24
24
  from typing import Any, List, Mapping, Optional, Union
25
25
 
26
- import orjson
26
+ from orjson import orjson
27
27
  from pydantic import ValidationError as V2ValidationError
28
28
  from serpyco_rs import SchemaValidationError
29
29
 
@@ -63,7 +63,7 @@ class EntrypointOutput:
63
63
  @staticmethod
64
64
  def _parse_message(message: str) -> AirbyteMessage:
65
65
  try:
66
- return AirbyteMessageSerializer.load(orjson.loads(message))
66
+ return AirbyteMessageSerializer.load(orjson.loads(message)) # type: ignore[no-any-return] # Serializer.load() always returns AirbyteMessage
67
67
  except (orjson.JSONDecodeError, SchemaValidationError):
68
68
  # The platform assumes that logs that are not of AirbyteMessage format are log messages
69
69
  return AirbyteMessage(
@@ -129,19 +129,14 @@ class EntrypointOutput:
129
129
  return [
130
130
  message
131
131
  for message in self._get_message_by_types([Type.TRACE])
132
- if message.trace.type == trace_type # type: ignore[union-attr] # trace has `type`
133
- ]
132
+ if message.trace.type == trace_type
133
+ ] # type: ignore[union-attr] # trace has `type`
134
134
 
135
135
  def is_in_logs(self, pattern: str) -> bool:
136
136
  """Check if any log message case-insensitive matches the pattern."""
137
137
  return any(
138
- re.search(
139
- pattern,
140
- entry.log.message, # type: ignore[union-attr] # log has `message`
141
- flags=re.IGNORECASE,
142
- )
143
- for entry in self.logs
144
- )
138
+ re.search(pattern, entry.log.message, flags=re.IGNORECASE) for entry in self.logs
139
+ ) # type: ignore[union-attr] # log has `message`
145
140
 
146
141
  def is_not_in_logs(self, pattern: str) -> bool:
147
142
  """Check if no log message matches the case-insensitive pattern."""