airbyte-cdk 6.8.1rc9__py3-none-any.whl → 6.8.2.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. airbyte_cdk/cli/source_declarative_manifest/_run.py +11 -5
  2. airbyte_cdk/config_observation.py +1 -1
  3. airbyte_cdk/connector_builder/main.py +1 -1
  4. airbyte_cdk/connector_builder/message_grouper.py +10 -10
  5. airbyte_cdk/destinations/destination.py +1 -1
  6. airbyte_cdk/destinations/vector_db_based/embedder.py +2 -2
  7. airbyte_cdk/destinations/vector_db_based/writer.py +12 -4
  8. airbyte_cdk/entrypoint.py +7 -6
  9. airbyte_cdk/logger.py +2 -2
  10. airbyte_cdk/sources/abstract_source.py +1 -1
  11. airbyte_cdk/sources/config.py +1 -1
  12. airbyte_cdk/sources/connector_state_manager.py +9 -4
  13. airbyte_cdk/sources/declarative/auth/oauth.py +1 -1
  14. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +6 -1
  15. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +76 -28
  16. airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +10 -4
  17. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +16 -17
  18. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +4 -1
  19. airbyte_cdk/sources/declarative/extractors/record_filter.py +3 -5
  20. airbyte_cdk/sources/declarative/incremental/__init__.py +3 -0
  21. airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py +270 -0
  22. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +8 -6
  23. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +9 -0
  24. airbyte_cdk/sources/declarative/interpolation/jinja.py +35 -36
  25. airbyte_cdk/sources/declarative/interpolation/macros.py +1 -1
  26. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +71 -17
  27. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +13 -7
  28. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +1 -1
  29. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +8 -6
  30. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +1 -1
  31. airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +2 -2
  32. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +1 -1
  33. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +5 -2
  34. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +1 -1
  35. airbyte_cdk/sources/declarative/spec/spec.py +1 -1
  36. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +0 -1
  37. airbyte_cdk/sources/embedded/base_integration.py +3 -2
  38. airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +12 -4
  39. airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +18 -7
  40. airbyte_cdk/sources/file_based/file_types/avro_parser.py +14 -11
  41. airbyte_cdk/sources/file_based/file_types/csv_parser.py +3 -3
  42. airbyte_cdk/sources/file_based/file_types/excel_parser.py +11 -5
  43. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +1 -1
  44. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +2 -2
  45. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +6 -3
  46. airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +1 -1
  47. airbyte_cdk/sources/http_logger.py +3 -3
  48. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +5 -2
  49. airbyte_cdk/sources/streams/concurrent/adapters.py +6 -3
  50. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +9 -3
  51. airbyte_cdk/sources/streams/concurrent/cursor.py +10 -1
  52. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +2 -2
  53. airbyte_cdk/sources/streams/core.py +17 -14
  54. airbyte_cdk/sources/streams/http/http.py +19 -19
  55. airbyte_cdk/sources/streams/http/http_client.py +4 -48
  56. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +2 -1
  57. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +62 -33
  58. airbyte_cdk/sources/utils/record_helper.py +1 -1
  59. airbyte_cdk/sources/utils/schema_helpers.py +1 -1
  60. airbyte_cdk/sources/utils/transform.py +34 -15
  61. airbyte_cdk/test/entrypoint_wrapper.py +11 -6
  62. airbyte_cdk/test/mock_http/response_builder.py +1 -1
  63. airbyte_cdk/utils/airbyte_secrets_utils.py +1 -1
  64. airbyte_cdk/utils/event_timing.py +10 -10
  65. airbyte_cdk/utils/message_utils.py +4 -3
  66. airbyte_cdk/utils/spec_schema_transformations.py +3 -2
  67. airbyte_cdk/utils/traced_exception.py +14 -12
  68. airbyte_cdk-6.8.2.dev1.dist-info/METADATA +111 -0
  69. {airbyte_cdk-6.8.1rc9.dist-info → airbyte_cdk-6.8.2.dev1.dist-info}/RECORD +72 -71
  70. airbyte_cdk-6.8.1rc9.dist-info/METADATA +0 -307
  71. {airbyte_cdk-6.8.1rc9.dist-info → airbyte_cdk-6.8.2.dev1.dist-info}/LICENSE.txt +0 -0
  72. {airbyte_cdk-6.8.1rc9.dist-info → airbyte_cdk-6.8.2.dev1.dist-info}/WHEEL +0 -0
  73. {airbyte_cdk-6.8.1rc9.dist-info → airbyte_cdk-6.8.2.dev1.dist-info}/entry_points.txt +0 -0
@@ -10,7 +10,7 @@ from dataclasses import dataclass
10
10
  from functools import cached_property, lru_cache
11
11
  from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Union
12
12
 
13
- from deprecated import deprecated
13
+ from typing_extensions import deprecated
14
14
 
15
15
  import airbyte_cdk.sources.utils.casing as casing
16
16
  from airbyte_cdk.models import (
@@ -92,8 +92,8 @@ class CheckpointMixin(ABC):
92
92
 
93
93
 
94
94
  @deprecated(
95
- version="0.87.0",
96
- reason="Deprecated in favor of the CheckpointMixin which offers similar functionality",
95
+ "Deprecated as of CDK version 0.87.0. "
96
+ "Deprecated in favor of the `CheckpointMixin` which offers similar functionality."
97
97
  )
98
98
  class IncrementalMixin(CheckpointMixin, ABC):
99
99
  """Mixin to make stream incremental.
@@ -115,12 +115,6 @@ class StreamClassification:
115
115
  has_multiple_slices: bool
116
116
 
117
117
 
118
- # Moved to class declaration since get_updated_state is called on every record for incremental syncs, and thus the @deprecated decorator as well.
119
- @deprecated(
120
- version="0.1.49",
121
- reason="Deprecated method get_updated_state, You should use explicit state property instead, see IncrementalMixin docs.",
122
- action="ignore",
123
- )
124
118
  class Stream(ABC):
125
119
  """
126
120
  Base abstract class for an Airbyte Stream. Makes no assumption of the Stream's underlying transport protocol.
@@ -222,7 +216,8 @@ class Stream(ABC):
222
216
  # Some connectors have streams that implement get_updated_state(), but do not define a cursor_field. This
223
217
  # should be fixed on the stream implementation, but we should also protect against this in the CDK as well
224
218
  stream_state_tracker = self.get_updated_state(
225
- stream_state_tracker, record_data
219
+ stream_state_tracker,
220
+ record_data, # type: ignore [arg-type]
226
221
  )
227
222
  self._observe_state(checkpoint_reader, stream_state_tracker)
228
223
  record_counter += 1
@@ -282,7 +277,7 @@ class Stream(ABC):
282
277
  if state
283
278
  else {}, # read() expects MutableMapping instead of Mapping which is used more often
284
279
  state_manager=None,
285
- internal_config=InternalConfig(),
280
+ internal_config=InternalConfig(), # type: ignore [call-arg]
286
281
  )
287
282
 
288
283
  @abstractmethod
@@ -322,7 +317,7 @@ class Stream(ABC):
322
317
  # If we can offer incremental we always should. RFR is always less reliable than incremental which uses a real cursor value
323
318
  if self.supports_incremental:
324
319
  stream.source_defined_cursor = self.source_defined_cursor
325
- stream.supported_sync_modes.append(SyncMode.incremental) # type: ignore
320
+ stream.supported_sync_modes.append(SyncMode.incremental)
326
321
  stream.default_cursor_field = self._wrapped_cursor_field()
327
322
 
328
323
  keys = Stream._wrapped_primary_key(self.primary_key)
@@ -436,10 +431,18 @@ class Stream(ABC):
436
431
  """
437
432
  return None
438
433
 
434
+ # Commented-out to avoid any runtime penalty, since this is used in a hot per-record codepath.
435
+ # To be evaluated for re-introduction here: https://github.com/airbytehq/airbyte-python-cdk/issues/116
436
+ # @deprecated(
437
+ # "Deprecated method `get_updated_state` as of CDK version 0.1.49. "
438
+ # "Please use explicit state property instead, see `IncrementalMixin` docs."
439
+ # )
439
440
  def get_updated_state(
440
441
  self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]
441
442
  ) -> MutableMapping[str, Any]:
442
- """Override to extract state from the latest record. Needed to implement incremental sync.
443
+ """DEPRECATED. Please use explicit state property instead, see `IncrementalMixin` docs.
444
+
445
+ Override to extract state from the latest record. Needed to implement incremental sync.
443
446
 
444
447
  Inspects the latest record extracted from the data source and the current state object and return an updated state object.
445
448
 
@@ -654,7 +657,7 @@ class Stream(ABC):
654
657
  # todo: This can be consolidated into one ConnectorStateManager.update_and_create_state_message() method, but I want
655
658
  # to reduce changes right now and this would span concurrent as well
656
659
  state_manager.update_state_for_stream(self.name, self.namespace, stream_state)
657
- return state_manager.create_state_message(self.name, self.namespace)
660
+ return state_manager.create_state_message(self.name, self.namespace) # type: ignore [no-any-return]
658
661
 
659
662
  @property
660
663
  def configured_json_schema(self) -> Optional[Dict[str, Any]]:
@@ -9,8 +9,8 @@ from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optio
9
9
  from urllib.parse import urljoin
10
10
 
11
11
  import requests
12
- from deprecated import deprecated
13
12
  from requests.auth import AuthBase
13
+ from typing_extensions import deprecated
14
14
 
15
15
  from airbyte_cdk.models import AirbyteMessage, FailureType, SyncMode
16
16
  from airbyte_cdk.models import Type as MessageType
@@ -121,8 +121,8 @@ class HttpStream(Stream, CheckpointMixin, ABC):
121
121
 
122
122
  @property
123
123
  @deprecated(
124
- version="3.0.0",
125
- reason="You should set error_handler explicitly in HttpStream.get_error_handler() instead.",
124
+ "Deprecated as of CDK version 3.0.0. "
125
+ "You should set error_handler explicitly in HttpStream.get_error_handler() instead."
126
126
  )
127
127
  def raise_on_http_errors(self) -> bool:
128
128
  """
@@ -132,8 +132,8 @@ class HttpStream(Stream, CheckpointMixin, ABC):
132
132
 
133
133
  @property
134
134
  @deprecated(
135
- version="3.0.0",
136
- reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.",
135
+ "Deprecated as of CDK version 3.0.0. "
136
+ "You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead."
137
137
  )
138
138
  def max_retries(self) -> Union[int, None]:
139
139
  """
@@ -143,8 +143,8 @@ class HttpStream(Stream, CheckpointMixin, ABC):
143
143
 
144
144
  @property
145
145
  @deprecated(
146
- version="3.0.0",
147
- reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.",
146
+ "Deprecated as of CDK version 3.0.0. "
147
+ "You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead."
148
148
  )
149
149
  def max_time(self) -> Union[int, None]:
150
150
  """
@@ -154,8 +154,8 @@ class HttpStream(Stream, CheckpointMixin, ABC):
154
154
 
155
155
  @property
156
156
  @deprecated(
157
- version="3.0.0",
158
- reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.",
157
+ "Deprecated as of CDK version 3.0.0. "
158
+ "You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead."
159
159
  )
160
160
  def retry_factor(self) -> float:
161
161
  """
@@ -594,7 +594,7 @@ class HttpSubStream(HttpStream, ABC):
594
594
  # Skip non-records (eg AirbyteLogMessage)
595
595
  if isinstance(parent_record, AirbyteMessage):
596
596
  if parent_record.type == MessageType.RECORD:
597
- parent_record = parent_record.record.data
597
+ parent_record = parent_record.record.data # type: ignore [assignment, union-attr] # Incorrect type for assignment
598
598
  else:
599
599
  continue
600
600
  elif isinstance(parent_record, Record):
@@ -603,8 +603,8 @@ class HttpSubStream(HttpStream, ABC):
603
603
 
604
604
 
605
605
  @deprecated(
606
- version="3.0.0",
607
- reason="You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead.",
606
+ "Deprecated as of CDK version 3.0.0."
607
+ "You should set backoff_strategies explicitly in HttpStream.get_backoff_strategy() instead."
608
608
  )
609
609
  class HttpStreamAdapterBackoffStrategy(BackoffStrategy):
610
610
  def __init__(self, stream: HttpStream):
@@ -619,8 +619,8 @@ class HttpStreamAdapterBackoffStrategy(BackoffStrategy):
619
619
 
620
620
 
621
621
  @deprecated(
622
- version="3.0.0",
623
- reason="You should set error_handler explicitly in HttpStream.get_error_handler() instead.",
622
+ "Deprecated as of CDK version 3.0.0. "
623
+ "You should set error_handler explicitly in HttpStream.get_error_handler() instead."
624
624
  )
625
625
  class HttpStreamAdapterHttpStatusErrorHandler(HttpStatusErrorHandler):
626
626
  def __init__(self, stream: HttpStream, **kwargs): # type: ignore # noqa
@@ -639,15 +639,15 @@ class HttpStreamAdapterHttpStatusErrorHandler(HttpStatusErrorHandler):
639
639
  return ErrorResolution(
640
640
  response_action=ResponseAction.RATE_LIMITED,
641
641
  failure_type=FailureType.transient_error,
642
- error_message=f"Response status code: {response_or_exception.status_code}. Retrying...", # type: ignore[union-attr]
642
+ error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
643
643
  )
644
644
  return ErrorResolution(
645
645
  response_action=ResponseAction.RETRY,
646
646
  failure_type=FailureType.transient_error,
647
- error_message=f"Response status code: {response_or_exception.status_code}. Retrying...", # type: ignore[union-attr]
647
+ error_message=f"Response status code: {response_or_exception.status_code}. Retrying...",
648
648
  )
649
649
  else:
650
- if response_or_exception.ok: # type: ignore # noqa
650
+ if response_or_exception.ok:
651
651
  return ErrorResolution(
652
652
  response_action=ResponseAction.SUCCESS,
653
653
  failure_type=None,
@@ -657,13 +657,13 @@ class HttpStreamAdapterHttpStatusErrorHandler(HttpStatusErrorHandler):
657
657
  return ErrorResolution(
658
658
  response_action=ResponseAction.FAIL,
659
659
  failure_type=FailureType.transient_error,
660
- error_message=f"Response status code: {response_or_exception.status_code}. Unexpected error. Failed.", # type: ignore[union-attr]
660
+ error_message=f"Response status code: {response_or_exception.status_code}. Unexpected error. Failed.",
661
661
  )
662
662
  else:
663
663
  return ErrorResolution(
664
664
  response_action=ResponseAction.IGNORE,
665
665
  failure_type=FailureType.transient_error,
666
- error_message=f"Response status code: {response_or_exception.status_code}. Ignoring...", # type: ignore[union-attr]
666
+ error_message=f"Response status code: {response_or_exception.status_code}. Ignoring...",
667
667
  )
668
668
  else:
669
669
  self._logger.error(f"Received unexpected response type: {type(response_or_exception)}")
@@ -54,7 +54,6 @@ from airbyte_cdk.utils.stream_status_utils import (
54
54
  from airbyte_cdk.utils.traced_exception import AirbyteTracedException
55
55
 
56
56
  BODY_REQUEST_METHODS = ("GET", "POST", "PUT", "PATCH")
57
- logger = logging.getLogger("airbyte")
58
57
 
59
58
 
60
59
  class MessageRepresentationAirbyteTracedErrors(AirbyteTracedException):
@@ -95,7 +94,6 @@ class HttpClient:
95
94
  ):
96
95
  self._name = name
97
96
  self._api_budget: APIBudget = api_budget or APIBudget(policies=[])
98
- self._logger = logger
99
97
  if session:
100
98
  self._session = session
101
99
  else:
@@ -109,6 +107,7 @@ class HttpClient:
109
107
  )
110
108
  if isinstance(authenticator, AuthBase):
111
109
  self._session.auth = authenticator
110
+ self._logger = logger
112
111
  self._error_handler = error_handler or HttpStatusErrorHandler(self._logger)
113
112
  if backoff_strategy is not None:
114
113
  if isinstance(backoff_strategy, list):
@@ -142,12 +141,10 @@ class HttpClient:
142
141
  if cache_dir:
143
142
  sqlite_path = str(Path(cache_dir) / self.cache_filename)
144
143
  else:
145
- self._logger.info("Using memory for cache") # TODO: remove
146
144
  sqlite_path = "file::memory:?cache=shared"
147
- backend = SkipFailureSQLiteCache(self._name, sqlite_path) # TODO maybe add a busy timeout
148
145
  return CachedLimiterSession(
149
- sqlite_path, backend=backend, api_budget=self._api_budget, match_headers=True
150
- ) # type: ignore # there are no typeshed stubs for requests_cache
146
+ sqlite_path, backend="sqlite", api_budget=self._api_budget, match_headers=True
147
+ )
151
148
  else:
152
149
  return LimiterSession(api_budget=self._api_budget)
153
150
 
@@ -327,7 +324,7 @@ class HttpClient:
327
324
  formatter = log_formatter
328
325
  self._message_repository.log_message(
329
326
  Level.DEBUG,
330
- lambda: formatter(response), # type: ignore # log_formatter is always cast to a callable
327
+ lambda: formatter(response),
331
328
  )
332
329
 
333
330
  self._handle_error_resolution(
@@ -520,44 +517,3 @@ class HttpClient:
520
517
  )
521
518
 
522
519
  return request, response
523
-
524
-
525
- class SkipFailureSQLiteDict(requests_cache.backends.sqlite.SQLiteDict):
526
- def __getitem__(self, key): # type: ignore # lib is not typed
527
- try:
528
- return super().__getitem__(key) # type: ignore # lib is not typed
529
- except Exception as exception:
530
- if not isinstance(exception, KeyError):
531
- logger.warning(f"Error while retrieving item from cache: {exception}")
532
- else:
533
- raise exception
534
-
535
- def _write(self, key: str, value: str) -> None:
536
- try:
537
- super()._write(key, value) # type: ignore # lib is not typed
538
- except Exception as exception:
539
- logger.warning(f"Error while saving item to cache: {exception}")
540
-
541
-
542
- class SkipFailureSQLiteCache(requests_cache.backends.sqlite.SQLiteCache):
543
- def __init__( # type: ignore # ignoring as lib is not typed
544
- self,
545
- table_name="response",
546
- db_path="http_cache",
547
- serializer=None,
548
- **kwargs,
549
- ) -> None:
550
- super().__init__(db_path, serializer, **kwargs)
551
- skwargs = {"serializer": serializer, **kwargs} if serializer else kwargs
552
- self.responses: requests_cache.backends.sqlite.SQLiteDict = SkipFailureSQLiteDict(
553
- db_path, table_name=table_name, fast_save=True, wal=True, **skwargs
554
- )
555
- self.redirects: requests_cache.backends.sqlite.SQLiteDict = SkipFailureSQLiteDict(
556
- db_path,
557
- table_name=f"redirects_{table_name}",
558
- fast_save=True,
559
- wal=True,
560
- lock=self.responses._lock,
561
- serializer=None,
562
- **kwargs,
563
- )
@@ -5,13 +5,14 @@
5
5
  from abc import abstractmethod
6
6
  from typing import Any, Mapping
7
7
 
8
+ import requests
8
9
  from requests.auth import AuthBase
9
10
 
10
11
 
11
12
  class AbstractHeaderAuthenticator(AuthBase):
12
13
  """Abstract class for an header-based authenticators that add a header to outgoing HTTP requests."""
13
14
 
14
- def __call__(self, request):
15
+ def __call__(self, request: requests.PreparedRequest) -> Any:
15
16
  """Attach the HTTP headers required to authenticate on the HTTP request"""
16
17
  request.headers.update(self.get_auth_header())
17
18
  return request
@@ -30,12 +30,12 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
30
30
  client_id: str,
31
31
  client_secret: str,
32
32
  refresh_token: str,
33
- scopes: List[str] = None,
34
- token_expiry_date: pendulum.DateTime = None,
35
- token_expiry_date_format: str = None,
33
+ scopes: List[str] | None = None,
34
+ token_expiry_date: pendulum.DateTime | None = None,
35
+ token_expiry_date_format: str | None = None,
36
36
  access_token_name: str = "access_token",
37
37
  expires_in_name: str = "expires_in",
38
- refresh_request_body: Mapping[str, Any] = None,
38
+ refresh_request_body: Mapping[str, Any] | None = None,
39
39
  grant_type: str = "refresh_token",
40
40
  token_expiry_is_time_of_expiration: bool = False,
41
41
  refresh_token_error_status_codes: Tuple[int, ...] = (),
@@ -52,7 +52,7 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
52
52
  self._refresh_request_body = refresh_request_body
53
53
  self._grant_type = grant_type
54
54
 
55
- self._token_expiry_date = token_expiry_date or pendulum.now().subtract(days=1)
55
+ self._token_expiry_date = token_expiry_date or pendulum.now().subtract(days=1) # type: ignore [no-untyped-call]
56
56
  self._token_expiry_date_format = token_expiry_date_format
57
57
  self._token_expiry_is_time_of_expiration = token_expiry_is_time_of_expiration
58
58
  self._access_token = None
@@ -75,14 +75,14 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
75
75
  def get_access_token_name(self) -> str:
76
76
  return self._access_token_name
77
77
 
78
- def get_scopes(self) -> [str]:
79
- return self._scopes
78
+ def get_scopes(self) -> list[str]:
79
+ return self._scopes # type: ignore [return-value]
80
80
 
81
81
  def get_expires_in_name(self) -> str:
82
82
  return self._expires_in_name
83
83
 
84
84
  def get_refresh_request_body(self) -> Mapping[str, Any]:
85
- return self._refresh_request_body
85
+ return self._refresh_request_body # type: ignore [return-value]
86
86
 
87
87
  def get_grant_type(self) -> str:
88
88
  return self._grant_type
@@ -90,7 +90,7 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
90
90
  def get_token_expiry_date(self) -> pendulum.DateTime:
91
91
  return self._token_expiry_date
92
92
 
93
- def set_token_expiry_date(self, value: Union[str, int]):
93
+ def set_token_expiry_date(self, value: Union[str, int]) -> None:
94
94
  self._token_expiry_date = self._parse_token_expiration_date(value)
95
95
 
96
96
  @property
@@ -103,11 +103,11 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
103
103
 
104
104
  @property
105
105
  def access_token(self) -> str:
106
- return self._access_token
106
+ return self._access_token # type: ignore [return-value]
107
107
 
108
108
  @access_token.setter
109
- def access_token(self, value: str):
110
- self._access_token = value
109
+ def access_token(self, value: str) -> None:
110
+ self._access_token = value # type: ignore [assignment] # Incorrect type for assignment
111
111
 
112
112
 
113
113
  class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
@@ -124,11 +124,11 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
124
124
  self,
125
125
  connector_config: Mapping[str, Any],
126
126
  token_refresh_endpoint: str,
127
- scopes: List[str] = None,
127
+ scopes: List[str] | None = None,
128
128
  access_token_name: str = "access_token",
129
129
  expires_in_name: str = "expires_in",
130
130
  refresh_token_name: str = "refresh_token",
131
- refresh_request_body: Mapping[str, Any] = None,
131
+ refresh_request_body: Mapping[str, Any] | None = None,
132
132
  grant_type: str = "refresh_token",
133
133
  client_id: Optional[str] = None,
134
134
  client_secret: Optional[str] = None,
@@ -162,14 +162,17 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
162
162
  message_repository (MessageRepository): the message repository used to emit logs on HTTP requests and control message on config update
163
163
  """
164
164
  self._client_id = (
165
- client_id
165
+ client_id # type: ignore [assignment] # Incorrect type for assignment
166
166
  if client_id is not None
167
- else dpath.get(connector_config, ("credentials", "client_id"))
167
+ else dpath.get(connector_config, ("credentials", "client_id")) # type: ignore [arg-type]
168
168
  )
169
169
  self._client_secret = (
170
- client_secret
170
+ client_secret # type: ignore [assignment] # Incorrect type for assignment
171
171
  if client_secret is not None
172
- else dpath.get(connector_config, ("credentials", "client_secret"))
172
+ else dpath.get(
173
+ connector_config, # type: ignore [arg-type]
174
+ ("credentials", "client_secret"),
175
+ )
173
176
  )
174
177
  self._access_token_config_path = access_token_config_path
175
178
  self._refresh_token_config_path = refresh_token_config_path
@@ -207,27 +210,50 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
207
210
 
208
211
  @property
209
212
  def access_token(self) -> str:
210
- return dpath.get(self._connector_config, self._access_token_config_path, default="")
213
+ return dpath.get( # type: ignore [return-value]
214
+ self._connector_config, # type: ignore [arg-type]
215
+ self._access_token_config_path,
216
+ default="",
217
+ )
211
218
 
212
219
  @access_token.setter
213
- def access_token(self, new_access_token: str):
214
- dpath.new(self._connector_config, self._access_token_config_path, new_access_token)
220
+ def access_token(self, new_access_token: str) -> None:
221
+ dpath.new(
222
+ self._connector_config, # type: ignore [arg-type]
223
+ self._access_token_config_path,
224
+ new_access_token,
225
+ )
215
226
 
216
227
  def get_refresh_token(self) -> str:
217
- return dpath.get(self._connector_config, self._refresh_token_config_path, default="")
228
+ return dpath.get( # type: ignore [return-value]
229
+ self._connector_config, # type: ignore [arg-type]
230
+ self._refresh_token_config_path,
231
+ default="",
232
+ )
218
233
 
219
- def set_refresh_token(self, new_refresh_token: str):
220
- dpath.new(self._connector_config, self._refresh_token_config_path, new_refresh_token)
234
+ def set_refresh_token(self, new_refresh_token: str) -> None:
235
+ dpath.new(
236
+ self._connector_config, # type: ignore [arg-type]
237
+ self._refresh_token_config_path,
238
+ new_refresh_token,
239
+ )
221
240
 
222
241
  def get_token_expiry_date(self) -> pendulum.DateTime:
223
242
  expiry_date = dpath.get(
224
- self._connector_config, self._token_expiry_date_config_path, default=""
243
+ self._connector_config, # type: ignore [arg-type]
244
+ self._token_expiry_date_config_path,
245
+ default="",
225
246
  )
226
- return pendulum.now().subtract(days=1) if expiry_date == "" else pendulum.parse(expiry_date)
247
+ return pendulum.now().subtract(days=1) if expiry_date == "" else pendulum.parse(expiry_date) # type: ignore [arg-type, return-value, no-untyped-call]
227
248
 
228
- def set_token_expiry_date(self, new_token_expiry_date):
249
+ def set_token_expiry_date( # type: ignore[override]
250
+ self,
251
+ new_token_expiry_date: pendulum.DateTime,
252
+ ) -> None:
229
253
  dpath.new(
230
- self._connector_config, self._token_expiry_date_config_path, str(new_token_expiry_date)
254
+ self._connector_config, # type: ignore [arg-type]
255
+ self._token_expiry_date_config_path,
256
+ str(new_token_expiry_date),
231
257
  )
232
258
 
233
259
  def token_has_expired(self) -> bool:
@@ -236,7 +262,8 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
236
262
 
237
263
  @staticmethod
238
264
  def get_new_token_expiry_date(
239
- access_token_expires_in: str, token_expiry_date_format: str = None
265
+ access_token_expires_in: str,
266
+ token_expiry_date_format: str | None = None,
240
267
  ) -> pendulum.DateTime:
241
268
  if token_expiry_date_format:
242
269
  return pendulum.from_format(access_token_expires_in, token_expiry_date_format)
@@ -253,7 +280,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
253
280
  new_access_token, access_token_expires_in, new_refresh_token = (
254
281
  self.refresh_access_token()
255
282
  )
256
- new_token_expiry_date = self.get_new_token_expiry_date(
283
+ new_token_expiry_date: pendulum.DateTime = self.get_new_token_expiry_date(
257
284
  access_token_expires_in, self._token_expiry_date_format
258
285
  )
259
286
  self.access_token = new_access_token
@@ -264,13 +291,15 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
264
291
  # message directly in the console, this is needed
265
292
  if not isinstance(self._message_repository, NoopMessageRepository):
266
293
  self._message_repository.emit_message(
267
- create_connector_config_control_message(self._connector_config)
294
+ create_connector_config_control_message(self._connector_config) # type: ignore [arg-type]
268
295
  )
269
296
  else:
270
- emit_configuration_as_airbyte_control_message(self._connector_config)
297
+ emit_configuration_as_airbyte_control_message(self._connector_config) # type: ignore [arg-type]
271
298
  return self.access_token
272
299
 
273
- def refresh_access_token(self) -> Tuple[str, str, str]:
300
+ def refresh_access_token( # type: ignore[override] # Signature doesn't match base class
301
+ self,
302
+ ) -> Tuple[str, str, str]:
274
303
  response_json = self._get_refresh_access_token_response()
275
304
  return (
276
305
  response_json[self.get_access_token_name()],
@@ -35,7 +35,7 @@ def stream_data_to_airbyte_message(
35
35
  # need it to normalize values against json schema. By default no action
36
36
  # taken unless configured. See
37
37
  # docs/connector-development/cdk-python/schemas.md for details.
38
- transformer.transform(data, schema) # type: ignore
38
+ transformer.transform(data, schema)
39
39
  if is_file_transfer_message:
40
40
  message = AirbyteFileTransferRecordMessage(
41
41
  stream=stream_name, file=data, emitted_at=now_millis, data={}
@@ -194,7 +194,7 @@ class InternalConfig(BaseModel):
194
194
  def dict(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
195
195
  kwargs["by_alias"] = True
196
196
  kwargs["exclude_unset"] = True
197
- return super().dict(*args, **kwargs) # type: ignore[no-any-return]
197
+ return super().dict(*args, **kwargs)
198
198
 
199
199
  def is_limit_reached(self, records_counter: int) -> bool:
200
200
  """
@@ -5,9 +5,9 @@
5
5
  import logging
6
6
  from distutils.util import strtobool
7
7
  from enum import Flag, auto
8
- from typing import Any, Callable, Dict, Mapping, Optional
8
+ from typing import Any, Callable, Dict, Generator, Mapping, Optional, cast
9
9
 
10
- from jsonschema import Draft7Validator, ValidationError, validators
10
+ from jsonschema import Draft7Validator, RefResolver, ValidationError, Validator, validators
11
11
 
12
12
  json_to_python_simple = {
13
13
  "string": str,
@@ -30,7 +30,7 @@ class TransformConfig(Flag):
30
30
  ```
31
31
  """
32
32
 
33
- # No action taken, default behaviour. Cannot be combined with any other options.
33
+ # No action taken, default behavior. Cannot be combined with any other options.
34
34
  NoTransform = auto()
35
35
  # Applies default type casting with default_convert method which converts
36
36
  # values by applying simple type casting to specified jsonschema type.
@@ -67,15 +67,15 @@ class TypeTransformer:
67
67
  )
68
68
 
69
69
  def registerCustomTransform(
70
- self, normalization_callback: Callable[[Any, Dict[str, Any]], Any]
71
- ) -> Callable:
70
+ self, normalization_callback: Callable[[Any, dict[str, Any]], Any]
71
+ ) -> Callable[[Any, dict[str, Any]], Any]:
72
72
  """
73
73
  Register custom normalization callback.
74
74
  :param normalization_callback function to be used for value
75
75
  normalization. Takes original value and part type schema. Should return
76
76
  normalized value. See docs/connector-development/cdk-python/schemas.md
77
77
  for details.
78
- :return Same callbeck, this is usefull for using registerCustomTransform function as decorator.
78
+ :return Same callback, this is useful for using registerCustomTransform function as decorator.
79
79
  """
80
80
  if TransformConfig.CustomSchemaNormalization not in self._config:
81
81
  raise Exception(
@@ -141,7 +141,11 @@ class TypeTransformer:
141
141
  return original_item
142
142
  return original_item
143
143
 
144
- def __get_normalizer(self, schema_key: str, original_validator: Callable):
144
+ def __get_normalizer(
145
+ self,
146
+ schema_key: str,
147
+ original_validator: Callable, # type: ignore[type-arg]
148
+ ) -> Callable[[Any, Any, Any, dict[str, Any]], Generator[Any, Any, None]]:
145
149
  """
146
150
  Traverse through object fields using native jsonschema validator and apply normalization function.
147
151
  :param schema_key related json schema key that currently being validated/normalized.
@@ -149,8 +153,11 @@ class TypeTransformer:
149
153
  """
150
154
 
151
155
  def normalizator(
152
- validator_instance: Callable, property_value: Any, instance: Any, schema: Dict[str, Any]
153
- ):
156
+ validator_instance: Validator,
157
+ property_value: Any,
158
+ instance: Any,
159
+ schema: Dict[str, Any],
160
+ ) -> Generator[Any, Any, None]:
154
161
  """
155
162
  Jsonschema validator callable it uses for validating instance. We
156
163
  override default Draft7Validator to perform value transformation
@@ -163,10 +170,13 @@ class TypeTransformer:
163
170
  :
164
171
  """
165
172
 
166
- def resolve(subschema):
173
+ def resolve(subschema: dict[str, Any]) -> dict[str, Any]:
167
174
  if "$ref" in subschema:
168
- _, resolved = validator_instance.resolver.resolve(subschema["$ref"])
169
- return resolved
175
+ _, resolved = cast(
176
+ RefResolver,
177
+ validator_instance.resolver,
178
+ ).resolve(subschema["$ref"])
179
+ return cast(dict[str, Any], resolved)
170
180
  return subschema
171
181
 
172
182
  # Transform object and array values before running json schema type checking for each element.
@@ -185,11 +195,20 @@ class TypeTransformer:
185
195
  instance[index] = self.__normalize(item, subschema)
186
196
 
187
197
  # Running native jsonschema traverse algorithm after field normalization is done.
188
- yield from original_validator(validator_instance, property_value, instance, schema)
198
+ yield from original_validator(
199
+ validator_instance,
200
+ property_value,
201
+ instance,
202
+ schema,
203
+ )
189
204
 
190
205
  return normalizator
191
206
 
192
- def transform(self, record: Dict[str, Any], schema: Mapping[str, Any]):
207
+ def transform(
208
+ self,
209
+ record: Dict[str, Any],
210
+ schema: Mapping[str, Any],
211
+ ) -> None:
193
212
  """
194
213
  Normalize and validate according to config.
195
214
  :param record: record instance for normalization/transformation. All modification are done by modifying existent object.
@@ -201,7 +220,7 @@ class TypeTransformer:
201
220
  for e in normalizer.iter_errors(record):
202
221
  """
203
222
  just calling normalizer.validate() would throw an exception on
204
- first validation occurences and stop processing rest of schema.
223
+ first validation occurrences and stop processing rest of schema.
205
224
  """
206
225
  logger.warning(self.get_error_message(e))
207
226