airbyte-cdk 6.7.1.dev0__py3-none-any.whl → 6.7.1rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. airbyte_cdk/cli/source_declarative_manifest/_run.py +1 -2
  2. airbyte_cdk/config_observation.py +1 -2
  3. airbyte_cdk/connector.py +0 -1
  4. airbyte_cdk/connector_builder/connector_builder_handler.py +1 -1
  5. airbyte_cdk/connector_builder/main.py +1 -2
  6. airbyte_cdk/destinations/destination.py +1 -2
  7. airbyte_cdk/destinations/vector_db_based/config.py +1 -2
  8. airbyte_cdk/destinations/vector_db_based/document_processor.py +3 -4
  9. airbyte_cdk/destinations/vector_db_based/embedder.py +4 -5
  10. airbyte_cdk/entrypoint.py +2 -3
  11. airbyte_cdk/logger.py +1 -2
  12. airbyte_cdk/models/__init__.py +0 -2
  13. airbyte_cdk/models/airbyte_protocol.py +1 -2
  14. airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +3 -3
  15. airbyte_cdk/sources/concurrent_source/concurrent_source.py +1 -1
  16. airbyte_cdk/sources/config.py +1 -2
  17. airbyte_cdk/sources/declarative/auth/jwt.py +0 -1
  18. airbyte_cdk/sources/declarative/auth/oauth.py +0 -1
  19. airbyte_cdk/sources/declarative/auth/selective_authenticator.py +0 -1
  20. airbyte_cdk/sources/declarative/auth/token.py +1 -2
  21. airbyte_cdk/sources/declarative/auth/token_provider.py +2 -3
  22. airbyte_cdk/sources/declarative/concurrent_declarative_source.py +8 -66
  23. airbyte_cdk/sources/declarative/declarative_component_schema.yaml +0 -167
  24. airbyte_cdk/sources/declarative/decoders/json_decoder.py +2 -3
  25. airbyte_cdk/sources/declarative/decoders/noop_decoder.py +0 -1
  26. airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +0 -1
  27. airbyte_cdk/sources/declarative/decoders/xml_decoder.py +0 -1
  28. airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +0 -1
  29. airbyte_cdk/sources/declarative/extractors/http_selector.py +0 -1
  30. airbyte_cdk/sources/declarative/extractors/record_filter.py +48 -6
  31. airbyte_cdk/sources/declarative/extractors/record_selector.py +4 -32
  32. airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +1 -2
  33. airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +1 -2
  34. airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +2 -5
  35. airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +2 -5
  36. airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +3 -1
  37. airbyte_cdk/sources/declarative/interpolation/jinja.py +4 -5
  38. airbyte_cdk/sources/declarative/manifest_declarative_source.py +3 -4
  39. airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +1 -1
  40. airbyte_cdk/sources/declarative/models/declarative_component_schema.py +0 -122
  41. airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +2 -11
  42. airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +0 -1
  43. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +0 -1
  44. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +0 -1
  45. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +0 -1
  46. airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +0 -1
  47. airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +0 -1
  48. airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +0 -1
  49. airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +0 -1
  50. airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +0 -1
  51. airbyte_cdk/sources/declarative/requesters/http_job_repository.py +1 -2
  52. airbyte_cdk/sources/declarative/requesters/http_requester.py +0 -1
  53. airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +0 -1
  54. airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +0 -1
  55. airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +0 -1
  56. airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +0 -1
  57. airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +0 -1
  58. airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +0 -1
  59. airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +0 -1
  60. airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +3 -9
  61. airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +1 -2
  62. airbyte_cdk/sources/declarative/requesters/requester.py +0 -1
  63. airbyte_cdk/sources/declarative/retrievers/async_retriever.py +1 -2
  64. airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +7 -12
  65. airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +4 -7
  66. airbyte_cdk/sources/declarative/transformations/add_fields.py +0 -1
  67. airbyte_cdk/sources/declarative/transformations/remove_fields.py +0 -1
  68. airbyte_cdk/sources/declarative/yaml_declarative_source.py +0 -1
  69. airbyte_cdk/sources/embedded/tools.py +0 -1
  70. airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +1 -2
  71. airbyte_cdk/sources/file_based/config/avro_format.py +1 -2
  72. airbyte_cdk/sources/file_based/config/csv_format.py +1 -2
  73. airbyte_cdk/sources/file_based/config/excel_format.py +1 -2
  74. airbyte_cdk/sources/file_based/config/file_based_stream_config.py +1 -2
  75. airbyte_cdk/sources/file_based/config/jsonl_format.py +1 -2
  76. airbyte_cdk/sources/file_based/config/parquet_format.py +1 -2
  77. airbyte_cdk/sources/file_based/config/unstructured_format.py +1 -2
  78. airbyte_cdk/sources/file_based/file_based_source.py +1 -2
  79. airbyte_cdk/sources/file_based/file_based_stream_reader.py +1 -2
  80. airbyte_cdk/sources/file_based/file_types/avro_parser.py +0 -1
  81. airbyte_cdk/sources/file_based/file_types/csv_parser.py +1 -2
  82. airbyte_cdk/sources/file_based/file_types/excel_parser.py +5 -5
  83. airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +1 -2
  84. airbyte_cdk/sources/file_based/file_types/parquet_parser.py +1 -2
  85. airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +8 -9
  86. airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +1 -2
  87. airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +4 -5
  88. airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +1 -1
  89. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +1 -1
  90. airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +1 -1
  91. airbyte_cdk/sources/http_logger.py +0 -1
  92. airbyte_cdk/sources/streams/call_rate.py +2 -1
  93. airbyte_cdk/sources/streams/concurrent/abstract_stream.py +1 -2
  94. airbyte_cdk/sources/streams/concurrent/adapters.py +4 -8
  95. airbyte_cdk/sources/streams/concurrent/availability_strategy.py +1 -2
  96. airbyte_cdk/sources/streams/concurrent/cursor.py +6 -30
  97. airbyte_cdk/sources/streams/concurrent/default_stream.py +0 -1
  98. airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -1
  99. airbyte_cdk/sources/streams/concurrent/partitions/record.py +35 -0
  100. airbyte_cdk/sources/streams/concurrent/partitions/types.py +1 -1
  101. airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +1 -1
  102. airbyte_cdk/sources/streams/core.py +1 -2
  103. airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +1 -2
  104. airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +0 -1
  105. airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +0 -1
  106. airbyte_cdk/sources/streams/http/error_handlers/response_models.py +1 -2
  107. airbyte_cdk/sources/streams/http/http.py +2 -3
  108. airbyte_cdk/sources/streams/http/http_client.py +8 -49
  109. airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +1 -2
  110. airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +0 -1
  111. airbyte_cdk/sources/types.py +1 -14
  112. airbyte_cdk/sources/utils/schema_helpers.py +2 -3
  113. airbyte_cdk/sql/secrets.py +1 -2
  114. airbyte_cdk/sql/shared/sql_processor.py +6 -8
  115. airbyte_cdk/test/entrypoint_wrapper.py +3 -4
  116. airbyte_cdk/test/mock_http/mocker.py +0 -1
  117. airbyte_cdk/utils/schema_inferrer.py +1 -2
  118. airbyte_cdk/utils/slice_hasher.py +1 -1
  119. airbyte_cdk/utils/traced_exception.py +1 -2
  120. {airbyte_cdk-6.7.1.dev0.dist-info → airbyte_cdk-6.7.1rc1.dist-info}/METADATA +2 -9
  121. {airbyte_cdk-6.7.1.dev0.dist-info → airbyte_cdk-6.7.1rc1.dist-info}/RECORD +124 -123
  122. {airbyte_cdk-6.7.1.dev0.dist-info → airbyte_cdk-6.7.1rc1.dist-info}/LICENSE.txt +0 -0
  123. {airbyte_cdk-6.7.1.dev0.dist-info → airbyte_cdk-6.7.1rc1.dist-info}/WHEEL +0 -0
  124. {airbyte_cdk-6.7.1.dev0.dist-info → airbyte_cdk-6.7.1rc1.dist-info}/entry_points.txt +0 -0
@@ -7,7 +7,6 @@ from dataclasses import dataclass
7
7
  from typing import Any, Generator, MutableMapping
8
8
 
9
9
  import requests
10
-
11
10
  from airbyte_cdk.sources.declarative.decoders import Decoder
12
11
 
13
12
  logger = logging.getLogger("airbyte")
@@ -9,7 +9,6 @@ from xml.parsers.expat import ExpatError
9
9
 
10
10
  import requests
11
11
  import xmltodict
12
-
13
12
  from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
14
13
 
15
14
  logger = logging.getLogger("airbyte")
@@ -7,7 +7,6 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Union
7
7
 
8
8
  import dpath
9
9
  import requests
10
-
11
10
  from airbyte_cdk.sources.declarative.decoders import Decoder, JsonDecoder
12
11
  from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
13
12
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
@@ -6,7 +6,6 @@ from abc import abstractmethod
6
6
  from typing import Any, Iterable, Mapping, Optional
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
11
10
 
12
11
 
@@ -1,6 +1,7 @@
1
1
  #
2
2
  # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
3
  #
4
+ import datetime
4
5
  from dataclasses import InitVar, dataclass
5
6
  from typing import Any, Iterable, Mapping, Optional, Union
6
7
 
@@ -10,7 +11,7 @@ from airbyte_cdk.sources.declarative.incremental import (
10
11
  PerPartitionWithGlobalCursor,
11
12
  )
12
13
  from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
13
- from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
14
+ from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
14
15
 
15
16
 
16
17
  @dataclass
@@ -67,6 +68,20 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
67
68
  self._date_time_based_cursor = date_time_based_cursor
68
69
  self._substream_cursor = substream_cursor
69
70
 
71
+ @property
72
+ def _cursor_field(self) -> str:
73
+ return self._date_time_based_cursor.cursor_field.eval(self._date_time_based_cursor.config) # type: ignore # eval returns a string in this context
74
+
75
+ @property
76
+ def _start_date_from_config(self) -> datetime.datetime:
77
+ return self._date_time_based_cursor._start_datetime.get_datetime(
78
+ self._date_time_based_cursor.config
79
+ )
80
+
81
+ @property
82
+ def _end_datetime(self) -> datetime.datetime:
83
+ return self._date_time_based_cursor.select_best_end_datetime()
84
+
70
85
  def filter_records(
71
86
  self,
72
87
  records: Iterable[Mapping[str, Any]],
@@ -74,14 +89,16 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
74
89
  stream_slice: Optional[StreamSlice] = None,
75
90
  next_page_token: Optional[Mapping[str, Any]] = None,
76
91
  ) -> Iterable[Mapping[str, Any]]:
92
+ state_value = self._get_state_value(
93
+ stream_state, stream_slice or StreamSlice(partition={}, cursor_slice={})
94
+ )
95
+ filter_date: datetime.datetime = self._get_filter_date(state_value)
77
96
  records = (
78
97
  record
79
98
  for record in records
80
- if (self._substream_cursor or self._date_time_based_cursor).should_be_synced(
81
- # Record is created on the fly to align with cursors interface; stream name is ignored as we don't need it here
82
- # Record stream name is empty cause it is not used durig the filtering
83
- Record(data=record, associated_slice=stream_slice, stream_name="")
84
- )
99
+ if self._end_datetime
100
+ >= self._date_time_based_cursor.parse_date(record[self._cursor_field])
101
+ >= filter_date
85
102
  )
86
103
  if self.condition:
87
104
  records = super().filter_records(
@@ -91,3 +108,28 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
91
108
  next_page_token=next_page_token,
92
109
  )
93
110
  yield from records
111
+
112
+ def _get_state_value(
113
+ self, stream_state: StreamState, stream_slice: StreamSlice
114
+ ) -> Optional[str]:
115
+ """
116
+ Return cursor_value or None in case it was not found.
117
+ Cursor_value may be empty if:
118
+ 1. It is an initial sync => no stream_state exist at all.
119
+ 2. In Parent-child stream, and we already make initial sync, so stream_state is present.
120
+ During the second read, we receive one extra record from parent and therefore no stream_state for this record will be found.
121
+
122
+ :param StreamState stream_state: State
123
+ :param StreamSlice stream_slice: Current Stream slice
124
+ :return Optional[str]: cursor_value in case it was found, otherwise None.
125
+ """
126
+ state = (self._substream_cursor or self._date_time_based_cursor).select_state(stream_slice)
127
+
128
+ return state.get(self._cursor_field) if state else None
129
+
130
+ def _get_filter_date(self, state_value: Optional[str]) -> datetime.datetime:
131
+ start_date_parsed = self._start_date_from_config
132
+ if state_value:
133
+ return max(start_date_parsed, self._date_time_based_cursor.parse_date(state_value))
134
+ else:
135
+ return start_date_parsed
@@ -3,14 +3,12 @@
3
3
  #
4
4
 
5
5
  from dataclasses import InitVar, dataclass, field
6
- from typing import Any, Iterable, List, Mapping, Optional, Union
6
+ from typing import Any, Iterable, List, Mapping, Optional
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
11
10
  from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
12
11
  from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
13
- from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
14
12
  from airbyte_cdk.sources.declarative.models import SchemaNormalization
15
13
  from airbyte_cdk.sources.declarative.transformations import RecordTransformation
16
14
  from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
@@ -39,34 +37,11 @@ class RecordSelector(HttpSelector):
39
37
  config: Config
40
38
  parameters: InitVar[Mapping[str, Any]]
41
39
  schema_normalization: TypeTransformer
42
- name: str
43
- _name: Union[InterpolatedString, str] = field(init=False, repr=False, default="")
44
40
  record_filter: Optional[RecordFilter] = None
45
41
  transformations: List[RecordTransformation] = field(default_factory=lambda: [])
46
42
 
47
43
  def __post_init__(self, parameters: Mapping[str, Any]) -> None:
48
44
  self._parameters = parameters
49
- self._name = (
50
- InterpolatedString(self._name, parameters=parameters)
51
- if isinstance(self._name, str)
52
- else self._name
53
- )
54
-
55
- @property # type: ignore
56
- def name(self) -> str:
57
- """
58
- :return: Stream name
59
- """
60
- return (
61
- str(self._name.eval(self.config))
62
- if isinstance(self._name, InterpolatedString)
63
- else self._name
64
- )
65
-
66
- @name.setter
67
- def name(self, value: str) -> None:
68
- if not isinstance(value, property):
69
- self._name = value
70
45
 
71
46
  def select_records(
72
47
  self,
@@ -110,7 +85,7 @@ class RecordSelector(HttpSelector):
110
85
  transformed_data = self._transform(filtered_data, stream_state, stream_slice)
111
86
  normalized_data = self._normalize_by_schema(transformed_data, schema=records_schema)
112
87
  for data in normalized_data:
113
- yield Record(data=data, stream_name=self.name, associated_slice=stream_slice)
88
+ yield Record(data, stream_slice)
114
89
 
115
90
  def _normalize_by_schema(
116
91
  self, records: Iterable[Mapping[str, Any]], schema: Optional[Mapping[str, Any]]
@@ -150,9 +125,6 @@ class RecordSelector(HttpSelector):
150
125
  for record in records:
151
126
  for transformation in self.transformations:
152
127
  transformation.transform(
153
- record, # type: ignore # record has type Mapping[str, Any], but Dict[str, Any] expected
154
- config=self.config,
155
- stream_state=stream_state,
156
- stream_slice=stream_slice,
157
- )
128
+ record, config=self.config, stream_state=stream_state, stream_slice=stream_slice
129
+ ) # type: ignore # record has type Mapping[str, Any], but Dict[str, Any] expected
158
130
  yield record
@@ -10,9 +10,8 @@ from typing import Any, Dict, Iterable, Mapping, Optional, Tuple
10
10
 
11
11
  import pandas as pd
12
12
  import requests
13
- from numpy import nan
14
-
15
13
  from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
14
+ from numpy import nan
16
15
 
17
16
  EMPTY_STR: str = ""
18
17
  DEFAULT_ENCODING: str = "utf-8"
@@ -7,8 +7,6 @@ from dataclasses import InitVar, dataclass, field
7
7
  from datetime import timedelta
8
8
  from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Union
9
9
 
10
- from isodate import Duration, duration_isoformat, parse_duration
11
-
12
10
  from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type
13
11
  from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
14
12
  from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
@@ -21,6 +19,7 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
21
19
  )
22
20
  from airbyte_cdk.sources.message import MessageRepository
23
21
  from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
22
+ from isodate import Duration, duration_isoformat, parse_duration
24
23
 
25
24
 
26
25
  @dataclass
@@ -340,11 +340,8 @@ class GlobalSubstreamCursor(DeclarativeCursor):
340
340
  @staticmethod
341
341
  def _convert_record_to_cursor_record(record: Record) -> Record:
342
342
  return Record(
343
- data=record.data,
344
- stream_name=record.stream_name,
345
- associated_slice=StreamSlice(
346
- partition={}, cursor_slice=record.associated_slice.cursor_slice
347
- )
343
+ record.data,
344
+ StreamSlice(partition={}, cursor_slice=record.associated_slice.cursor_slice)
348
345
  if record.associated_slice
349
346
  else None,
350
347
  )
@@ -325,11 +325,8 @@ class PerPartitionCursor(DeclarativeCursor):
325
325
  @staticmethod
326
326
  def _convert_record_to_cursor_record(record: Record) -> Record:
327
327
  return Record(
328
- data=record.data,
329
- stream_name=record.stream_name,
330
- associated_slice=StreamSlice(
331
- partition={}, cursor_slice=record.associated_slice.cursor_slice
332
- )
328
+ record.data,
329
+ StreamSlice(partition={}, cursor_slice=record.associated_slice.cursor_slice)
333
330
  if record.associated_slice
334
331
  else None,
335
332
  )
@@ -194,7 +194,9 @@ class PerPartitionWithGlobalCursor(DeclarativeCursor):
194
194
  )
195
195
 
196
196
  def should_be_synced(self, record: Record) -> bool:
197
- return self._get_active_cursor().should_be_synced(record)
197
+ return self._global_cursor.should_be_synced(
198
+ record
199
+ ) or self._per_partition_cursor.should_be_synced(record)
198
200
 
199
201
  def is_greater_than_or_equal(self, first: Record, second: Record) -> bool:
200
202
  return self._global_cursor.is_greater_than_or_equal(first, second)
@@ -6,15 +6,14 @@ import ast
6
6
  from functools import cache
7
7
  from typing import Any, Mapping, Optional, Tuple, Type
8
8
 
9
- from jinja2 import meta
10
- from jinja2.environment import Template
11
- from jinja2.exceptions import UndefinedError
12
- from jinja2.sandbox import SandboxedEnvironment
13
-
14
9
  from airbyte_cdk.sources.declarative.interpolation.filters import filters
15
10
  from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
16
11
  from airbyte_cdk.sources.declarative.interpolation.macros import macros
17
12
  from airbyte_cdk.sources.types import Config
13
+ from jinja2 import meta
14
+ from jinja2.environment import Template
15
+ from jinja2.exceptions import UndefinedError
16
+ from jinja2.sandbox import SandboxedEnvironment
18
17
 
19
18
 
20
19
  class StreamPartitionAccessEnvironment(SandboxedEnvironment):
@@ -8,12 +8,9 @@ import pkgutil
8
8
  from copy import deepcopy
9
9
  from importlib import metadata
10
10
  from typing import Any, Dict, Iterator, List, Mapping, Optional
11
+ from packaging.version import Version, InvalidVersion
11
12
 
12
13
  import yaml
13
- from jsonschema.exceptions import ValidationError
14
- from jsonschema.validators import validate
15
- from packaging.version import InvalidVersion, Version
16
-
17
14
  from airbyte_cdk.models import (
18
15
  AirbyteConnectionStatus,
19
16
  AirbyteMessage,
@@ -47,6 +44,8 @@ from airbyte_cdk.sources.utils.slice_logger import (
47
44
  DebugSliceLogger,
48
45
  SliceLogger,
49
46
  )
47
+ from jsonschema.exceptions import ValidationError
48
+ from jsonschema.validators import validate
50
49
 
51
50
 
52
51
  class ManifestDeclarativeSource(DeclarativeSource):
@@ -5,9 +5,9 @@ from typing import Any, Mapping
5
5
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
6
6
  from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
7
7
  from airbyte_cdk.sources.declarative.models import (
8
- CustomIncrementalSync,
9
8
  DatetimeBasedCursor,
10
9
  SubstreamPartitionRouter,
10
+ CustomIncrementalSync,
11
11
  )
12
12
  from airbyte_cdk.sources.declarative.models.declarative_component_schema import ParentStreamConfig
13
13
 
@@ -748,123 +748,6 @@ class NoPagination(BaseModel):
748
748
  type: Literal["NoPagination"]
749
749
 
750
750
 
751
- class State(BaseModel):
752
- class Config:
753
- extra = Extra.allow
754
-
755
- min: int
756
- max: int
757
-
758
-
759
- class OauthConnectorInputSpecification(BaseModel):
760
- class Config:
761
- extra = Extra.allow
762
-
763
- consent_url: str = Field(
764
- ...,
765
- description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.",
766
- examples=[
767
- {
768
- "consent_url": "https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}"
769
- },
770
- {
771
- "consent_url": "https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}"
772
- },
773
- ],
774
- title="DeclarativeOAuth Consent URL",
775
- )
776
- scope: Optional[str] = Field(
777
- None,
778
- description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.",
779
- examples=[{"scope": "user:read user:read_orders workspaces:read"}],
780
- title="(Optional) DeclarativeOAuth Scope",
781
- )
782
- access_token_url: str = Field(
783
- ...,
784
- description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.",
785
- examples=[
786
- {
787
- "access_token_url": "https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
788
- }
789
- ],
790
- title="DeclarativeOAuth Access Token URL",
791
- )
792
- access_token_headers: Optional[Dict[str, Any]] = Field(
793
- None,
794
- description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.",
795
- examples=[
796
- {
797
- "access_token_headers": {
798
- "Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"
799
- }
800
- }
801
- ],
802
- title="(Optional) DeclarativeOAuth Access Token Headers",
803
- )
804
- access_token_params: Optional[Dict[str, Any]] = Field(
805
- None,
806
- description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.",
807
- examples=[
808
- {
809
- "access_token_params": {
810
- "{auth_code_key}": "{{auth_code_key}}",
811
- "{client_id_key}": "{{client_id_key}}",
812
- "{client_secret_key}": "{{client_secret_key}}",
813
- }
814
- }
815
- ],
816
- title="(Optional) DeclarativeOAuth Access Token Query Params (Json Encoded)",
817
- )
818
- extract_output: List[str] = Field(
819
- ...,
820
- description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config. ",
821
- examples=[{"extract_output": ["access_token", "refresh_token", "other_field"]}],
822
- title="DeclarativeOAuth Extract Output",
823
- )
824
- state: Optional[State] = Field(
825
- None,
826
- description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity. ",
827
- examples=[{"state": {"min": 7, "max": 128}}],
828
- title="(Optional) DeclarativeOAuth Configurable State Query Param",
829
- )
830
- client_id_key: Optional[str] = Field(
831
- None,
832
- description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.",
833
- examples=[{"client_id_key": "my_custom_client_id_key_name"}],
834
- title="(Optional) DeclarativeOAuth Client ID Key Override",
835
- )
836
- client_secret_key: Optional[str] = Field(
837
- None,
838
- description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.",
839
- examples=[{"client_secret_key": "my_custom_client_secret_key_name"}],
840
- title="(Optional) DeclarativeOAuth Client Secret Key Override",
841
- )
842
- scope_key: Optional[str] = Field(
843
- None,
844
- description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.",
845
- examples=[{"scope_key": "my_custom_scope_key_key_name"}],
846
- title="(Optional) DeclarativeOAuth Scope Key Override",
847
- )
848
- state_key: Optional[str] = Field(
849
- None,
850
- description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider. ",
851
- examples=[{"state_key": "my_custom_state_key_key_name"}],
852
- title="(Optional) DeclarativeOAuth State Key Override",
853
- )
854
- auth_code_key: Optional[str] = Field(
855
- None,
856
- description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider. ",
857
- examples=[{"auth_code_key": "my_custom_auth_code_key_name"}],
858
- title="(Optional) DeclarativeOAuth Auth Code Key Override",
859
- )
860
- redirect_uri_key: Optional[str] = Field(
861
- None,
862
- description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.",
863
- examples=[{"redirect_uri_key": "my_custom_redirect_uri_key_name"}],
864
- title="(Optional) DeclarativeOAuth Redirect URI Key Override",
865
- )
866
-
867
-
868
751
  class OAuthConfigSpecification(BaseModel):
869
752
  class Config:
870
753
  extra = Extra.allow
@@ -883,11 +766,6 @@ class OAuthConfigSpecification(BaseModel):
883
766
  ],
884
767
  title="OAuth user input",
885
768
  )
886
- oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field(
887
- None,
888
- description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{my_var}`.\n- The nested resolution variables like `{{my_nested_var}}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}\n + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}\n + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}\n + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{auth_code_key}": "{{auth_code_key}}",\n "{client_id_key}": "{{client_id_key}}",\n "{client_secret_key}": "{{client_secret_key}}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }',
889
- title="DeclarativeOAuth Connector Specification",
890
- )
891
769
  complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
892
770
  None,
893
771
  description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }",
@@ -25,9 +25,6 @@ from typing import (
25
25
  get_type_hints,
26
26
  )
27
27
 
28
- from isodate import parse_duration
29
- from pydantic.v1 import BaseModel
30
-
31
28
  from airbyte_cdk.models import FailureType, Level
32
29
  from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
33
30
  from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncJobOrchestrator
@@ -370,6 +367,8 @@ from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_sta
370
367
  from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction
371
368
  from airbyte_cdk.sources.types import Config
372
369
  from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
370
+ from isodate import parse_duration
371
+ from pydantic.v1 import BaseModel
373
372
 
374
373
  ComponentDefinition = Mapping[str, Any]
375
374
 
@@ -384,7 +383,6 @@ class ModelToComponentFactory:
384
383
  emit_connector_builder_messages: bool = False,
385
384
  disable_retries: bool = False,
386
385
  disable_cache: bool = False,
387
- disable_resumable_full_refresh: bool = False,
388
386
  message_repository: Optional[MessageRepository] = None,
389
387
  ):
390
388
  self._init_mappings()
@@ -393,7 +391,6 @@ class ModelToComponentFactory:
393
391
  self._emit_connector_builder_messages = emit_connector_builder_messages
394
392
  self._disable_retries = disable_retries
395
393
  self._disable_cache = disable_cache
396
- self._disable_resumable_full_refresh = disable_resumable_full_refresh
397
394
  self._message_repository = message_repository or InMemoryMessageRepository( # type: ignore
398
395
  self._evaluate_log_level(emit_connector_builder_messages)
399
396
  )
@@ -1337,8 +1334,6 @@ class ModelToComponentFactory:
1337
1334
  if model.incremental_sync
1338
1335
  else None
1339
1336
  )
1340
- elif self._disable_resumable_full_refresh:
1341
- return stream_slicer
1342
1337
  elif stream_slicer:
1343
1338
  # For the Full-Refresh sub-streams, we use the nested `ChildPartitionResumableFullRefreshCursor`
1344
1339
  return PerPartitionCursor(
@@ -1785,7 +1780,6 @@ class ModelToComponentFactory:
1785
1780
  self,
1786
1781
  model: RecordSelectorModel,
1787
1782
  config: Config,
1788
- name: str,
1789
1783
  *,
1790
1784
  transformations: List[RecordTransformation],
1791
1785
  decoder: Optional[Decoder] = None,
@@ -1816,7 +1810,6 @@ class ModelToComponentFactory:
1816
1810
 
1817
1811
  return RecordSelector(
1818
1812
  extractor=extractor,
1819
- name=name,
1820
1813
  config=config,
1821
1814
  record_filter=record_filter,
1822
1815
  transformations=transformations,
@@ -1887,7 +1880,6 @@ class ModelToComponentFactory:
1887
1880
  )
1888
1881
  record_selector = self._create_component_from_model(
1889
1882
  model=model.record_selector,
1890
- name=name,
1891
1883
  config=config,
1892
1884
  decoder=decoder,
1893
1885
  transformations=transformations,
@@ -2042,7 +2034,6 @@ class ModelToComponentFactory:
2042
2034
  requester=download_requester,
2043
2035
  record_selector=RecordSelector(
2044
2036
  extractor=ResponseToFileExtractor(),
2045
- name=name,
2046
2037
  record_filter=None,
2047
2038
  transformations=[],
2048
2039
  schema_normalization=TypeTransformer(TransformConfig.NoTransform),
@@ -7,7 +7,6 @@ from dataclasses import InitVar, dataclass
7
7
  from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Union
8
8
 
9
9
  import dpath
10
-
11
10
  from airbyte_cdk.models import AirbyteMessage
12
11
  from airbyte_cdk.models import Type as MessageType
13
12
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
11
10
  from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
12
11
  from airbyte_cdk.sources.types import Config
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
11
10
  from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
12
11
  from airbyte_cdk.sources.types import Config
@@ -7,7 +7,6 @@ from dataclasses import InitVar, dataclass
7
7
  from typing import Any, Mapping, Optional, Union
8
8
 
9
9
  import requests
10
-
11
10
  from airbyte_cdk.models import FailureType
12
11
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
13
12
  from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
@@ -9,7 +9,6 @@ from dataclasses import InitVar, dataclass
9
9
  from typing import Any, Mapping, Optional, Union
10
10
 
11
11
  import requests
12
-
13
12
  from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
14
13
  from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
15
14
  get_numeric_value_from_header,
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, List, Mapping, Optional, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
11
10
  from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
12
11
  ErrorResolution,
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass, field
6
6
  from typing import Any, List, Mapping, MutableMapping, Optional, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.requesters.error_handlers.default_http_response_filter import (
11
10
  DefaultHttpResponseFilter,
12
11
  )
@@ -5,7 +5,6 @@
5
5
  from typing import Optional, Union
6
6
 
7
7
  import requests
8
-
9
8
  from airbyte_cdk.sources.declarative.requesters.error_handlers.http_response_filter import (
10
9
  HttpResponseFilter,
11
10
  )
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
6
6
  from typing import Any, Mapping, Optional, Set, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.models import FailureType
11
10
  from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
12
11
  from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
@@ -6,8 +6,6 @@ from datetime import timedelta
6
6
  from typing import Any, Dict, Iterable, Mapping, Optional
7
7
 
8
8
  import requests
9
- from requests import Response
10
-
11
9
  from airbyte_cdk import AirbyteMessage
12
10
  from airbyte_cdk.logger import lazy_log
13
11
  from airbyte_cdk.models import FailureType, Type
@@ -25,6 +23,7 @@ from airbyte_cdk.sources.declarative.requesters.requester import Requester
25
23
  from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
26
24
  from airbyte_cdk.sources.types import Record, StreamSlice
27
25
  from airbyte_cdk.utils import AirbyteTracedException
26
+ from requests import Response
28
27
 
29
28
  LOGGER = logging.getLogger("airbyte")
30
29
 
@@ -9,7 +9,6 @@ from typing import Any, Callable, Mapping, MutableMapping, Optional, Union
9
9
  from urllib.parse import urljoin
10
10
 
11
11
  import requests
12
-
13
12
  from airbyte_cdk.sources.declarative.auth.declarative_authenticator import (
14
13
  DeclarativeAuthenticator,
15
14
  NoAuth,
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass, field
6
6
  from typing import Any, Mapping, MutableMapping, Optional, Union
7
7
 
8
8
  import requests
9
-
10
9
  from airbyte_cdk.sources.declarative.decoders import (
11
10
  Decoder,
12
11
  JsonDecoder,