airbyte-cdk 6.7.0rc3__py3-none-any.whl → 6.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/cli/source_declarative_manifest/_run.py +2 -1
- airbyte_cdk/config_observation.py +2 -1
- airbyte_cdk/connector.py +1 -0
- airbyte_cdk/connector_builder/connector_builder_handler.py +1 -1
- airbyte_cdk/connector_builder/main.py +2 -1
- airbyte_cdk/destinations/destination.py +2 -1
- airbyte_cdk/destinations/vector_db_based/config.py +2 -1
- airbyte_cdk/destinations/vector_db_based/document_processor.py +4 -3
- airbyte_cdk/destinations/vector_db_based/embedder.py +5 -4
- airbyte_cdk/entrypoint.py +3 -2
- airbyte_cdk/logger.py +2 -1
- airbyte_cdk/models/__init__.py +2 -0
- airbyte_cdk/models/airbyte_protocol.py +2 -1
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +3 -3
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +1 -1
- airbyte_cdk/sources/config.py +2 -1
- airbyte_cdk/sources/declarative/auth/jwt.py +1 -0
- airbyte_cdk/sources/declarative/auth/oauth.py +1 -0
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +1 -0
- airbyte_cdk/sources/declarative/auth/token.py +2 -1
- airbyte_cdk/sources/declarative/auth/token_provider.py +3 -2
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +24 -54
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +167 -0
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +3 -2
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +1 -0
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +1 -0
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +1 -0
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +1 -0
- airbyte_cdk/sources/declarative/extractors/http_selector.py +1 -0
- airbyte_cdk/sources/declarative/extractors/record_filter.py +6 -48
- airbyte_cdk/sources/declarative/extractors/record_selector.py +32 -4
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +2 -1
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +2 -1
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +5 -2
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +5 -2
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +1 -3
- airbyte_cdk/sources/declarative/interpolation/jinja.py +5 -4
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +4 -3
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +1 -1
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +122 -0
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +26 -17
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +1 -0
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +2 -1
- airbyte_cdk/sources/declarative/requesters/http_requester.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +9 -3
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +2 -1
- airbyte_cdk/sources/declarative/requesters/requester.py +1 -0
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +2 -1
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +12 -4
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +7 -4
- airbyte_cdk/sources/declarative/transformations/add_fields.py +1 -0
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +1 -0
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +1 -0
- airbyte_cdk/sources/embedded/tools.py +1 -0
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +2 -1
- airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
- airbyte_cdk/sources/file_based/config/csv_format.py +2 -1
- airbyte_cdk/sources/file_based/config/excel_format.py +2 -1
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +2 -1
- airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
- airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
- airbyte_cdk/sources/file_based/config/unstructured_format.py +2 -1
- airbyte_cdk/sources/file_based/file_based_source.py +2 -1
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +2 -1
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +1 -0
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +2 -1
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +5 -5
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +2 -1
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +2 -1
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +9 -8
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +2 -1
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +5 -4
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +1 -1
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +1 -1
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +1 -1
- airbyte_cdk/sources/http_logger.py +1 -0
- airbyte_cdk/sources/streams/call_rate.py +1 -2
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +2 -1
- airbyte_cdk/sources/streams/concurrent/adapters.py +8 -4
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +2 -1
- airbyte_cdk/sources/streams/concurrent/cursor.py +30 -6
- airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -1
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +1 -1
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +1 -1
- airbyte_cdk/sources/streams/core.py +2 -1
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +2 -1
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +1 -0
- airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +1 -0
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +2 -1
- airbyte_cdk/sources/streams/http/http.py +3 -2
- airbyte_cdk/sources/streams/http/http_client.py +49 -2
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +2 -1
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +1 -0
- airbyte_cdk/sources/types.py +14 -1
- airbyte_cdk/sources/utils/schema_helpers.py +3 -2
- airbyte_cdk/sql/secrets.py +2 -1
- airbyte_cdk/sql/shared/sql_processor.py +8 -6
- airbyte_cdk/test/entrypoint_wrapper.py +4 -3
- airbyte_cdk/test/mock_http/mocker.py +1 -0
- airbyte_cdk/utils/schema_inferrer.py +2 -1
- airbyte_cdk/utils/slice_hasher.py +1 -1
- airbyte_cdk/utils/traced_exception.py +2 -1
- {airbyte_cdk-6.7.0rc3.dist-info → airbyte_cdk-6.7.1.dist-info}/METADATA +9 -2
- {airbyte_cdk-6.7.0rc3.dist-info → airbyte_cdk-6.7.1.dist-info}/RECORD +122 -123
- airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -35
- {airbyte_cdk-6.7.0rc3.dist-info → airbyte_cdk-6.7.1.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.7.0rc3.dist-info → airbyte_cdk-6.7.1.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.7.0rc3.dist-info → airbyte_cdk-6.7.1.dist-info}/entry_points.txt +0 -0
@@ -7,6 +7,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Union
|
|
7
7
|
|
8
8
|
import dpath
|
9
9
|
import requests
|
10
|
+
|
10
11
|
from airbyte_cdk.sources.declarative.decoders import Decoder, JsonDecoder
|
11
12
|
from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
|
12
13
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
@@ -1,7 +1,6 @@
|
|
1
1
|
#
|
2
2
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
3
|
#
|
4
|
-
import datetime
|
5
4
|
from dataclasses import InitVar, dataclass
|
6
5
|
from typing import Any, Iterable, Mapping, Optional, Union
|
7
6
|
|
@@ -11,7 +10,7 @@ from airbyte_cdk.sources.declarative.incremental import (
|
|
11
10
|
PerPartitionWithGlobalCursor,
|
12
11
|
)
|
13
12
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
|
14
|
-
from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
|
13
|
+
from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
|
15
14
|
|
16
15
|
|
17
16
|
@dataclass
|
@@ -68,20 +67,6 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
|
|
68
67
|
self._date_time_based_cursor = date_time_based_cursor
|
69
68
|
self._substream_cursor = substream_cursor
|
70
69
|
|
71
|
-
@property
|
72
|
-
def _cursor_field(self) -> str:
|
73
|
-
return self._date_time_based_cursor.cursor_field.eval(self._date_time_based_cursor.config) # type: ignore # eval returns a string in this context
|
74
|
-
|
75
|
-
@property
|
76
|
-
def _start_date_from_config(self) -> datetime.datetime:
|
77
|
-
return self._date_time_based_cursor._start_datetime.get_datetime(
|
78
|
-
self._date_time_based_cursor.config
|
79
|
-
)
|
80
|
-
|
81
|
-
@property
|
82
|
-
def _end_datetime(self) -> datetime.datetime:
|
83
|
-
return self._date_time_based_cursor.select_best_end_datetime()
|
84
|
-
|
85
70
|
def filter_records(
|
86
71
|
self,
|
87
72
|
records: Iterable[Mapping[str, Any]],
|
@@ -89,16 +74,14 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
|
|
89
74
|
stream_slice: Optional[StreamSlice] = None,
|
90
75
|
next_page_token: Optional[Mapping[str, Any]] = None,
|
91
76
|
) -> Iterable[Mapping[str, Any]]:
|
92
|
-
state_value = self._get_state_value(
|
93
|
-
stream_state, stream_slice or StreamSlice(partition={}, cursor_slice={})
|
94
|
-
)
|
95
|
-
filter_date: datetime.datetime = self._get_filter_date(state_value)
|
96
77
|
records = (
|
97
78
|
record
|
98
79
|
for record in records
|
99
|
-
if self.
|
100
|
-
|
101
|
-
|
80
|
+
if (self._substream_cursor or self._date_time_based_cursor).should_be_synced(
|
81
|
+
# Record is created on the fly to align with cursors interface; stream name is ignored as we don't need it here
|
82
|
+
# Record stream name is empty cause it is not used durig the filtering
|
83
|
+
Record(data=record, associated_slice=stream_slice, stream_name="")
|
84
|
+
)
|
102
85
|
)
|
103
86
|
if self.condition:
|
104
87
|
records = super().filter_records(
|
@@ -108,28 +91,3 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
|
|
108
91
|
next_page_token=next_page_token,
|
109
92
|
)
|
110
93
|
yield from records
|
111
|
-
|
112
|
-
def _get_state_value(
|
113
|
-
self, stream_state: StreamState, stream_slice: StreamSlice
|
114
|
-
) -> Optional[str]:
|
115
|
-
"""
|
116
|
-
Return cursor_value or None in case it was not found.
|
117
|
-
Cursor_value may be empty if:
|
118
|
-
1. It is an initial sync => no stream_state exist at all.
|
119
|
-
2. In Parent-child stream, and we already make initial sync, so stream_state is present.
|
120
|
-
During the second read, we receive one extra record from parent and therefore no stream_state for this record will be found.
|
121
|
-
|
122
|
-
:param StreamState stream_state: State
|
123
|
-
:param StreamSlice stream_slice: Current Stream slice
|
124
|
-
:return Optional[str]: cursor_value in case it was found, otherwise None.
|
125
|
-
"""
|
126
|
-
state = (self._substream_cursor or self._date_time_based_cursor).select_state(stream_slice)
|
127
|
-
|
128
|
-
return state.get(self._cursor_field) if state else None
|
129
|
-
|
130
|
-
def _get_filter_date(self, state_value: Optional[str]) -> datetime.datetime:
|
131
|
-
start_date_parsed = self._start_date_from_config
|
132
|
-
if state_value:
|
133
|
-
return max(start_date_parsed, self._date_time_based_cursor.parse_date(state_value))
|
134
|
-
else:
|
135
|
-
return start_date_parsed
|
@@ -3,12 +3,14 @@
|
|
3
3
|
#
|
4
4
|
|
5
5
|
from dataclasses import InitVar, dataclass, field
|
6
|
-
from typing import Any, Iterable, List, Mapping, Optional
|
6
|
+
from typing import Any, Iterable, List, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
|
10
11
|
from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
|
11
12
|
from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
|
13
|
+
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
12
14
|
from airbyte_cdk.sources.declarative.models import SchemaNormalization
|
13
15
|
from airbyte_cdk.sources.declarative.transformations import RecordTransformation
|
14
16
|
from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
|
@@ -37,11 +39,34 @@ class RecordSelector(HttpSelector):
|
|
37
39
|
config: Config
|
38
40
|
parameters: InitVar[Mapping[str, Any]]
|
39
41
|
schema_normalization: TypeTransformer
|
42
|
+
name: str
|
43
|
+
_name: Union[InterpolatedString, str] = field(init=False, repr=False, default="")
|
40
44
|
record_filter: Optional[RecordFilter] = None
|
41
45
|
transformations: List[RecordTransformation] = field(default_factory=lambda: [])
|
42
46
|
|
43
47
|
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
44
48
|
self._parameters = parameters
|
49
|
+
self._name = (
|
50
|
+
InterpolatedString(self._name, parameters=parameters)
|
51
|
+
if isinstance(self._name, str)
|
52
|
+
else self._name
|
53
|
+
)
|
54
|
+
|
55
|
+
@property # type: ignore
|
56
|
+
def name(self) -> str:
|
57
|
+
"""
|
58
|
+
:return: Stream name
|
59
|
+
"""
|
60
|
+
return (
|
61
|
+
str(self._name.eval(self.config))
|
62
|
+
if isinstance(self._name, InterpolatedString)
|
63
|
+
else self._name
|
64
|
+
)
|
65
|
+
|
66
|
+
@name.setter
|
67
|
+
def name(self, value: str) -> None:
|
68
|
+
if not isinstance(value, property):
|
69
|
+
self._name = value
|
45
70
|
|
46
71
|
def select_records(
|
47
72
|
self,
|
@@ -85,7 +110,7 @@ class RecordSelector(HttpSelector):
|
|
85
110
|
transformed_data = self._transform(filtered_data, stream_state, stream_slice)
|
86
111
|
normalized_data = self._normalize_by_schema(transformed_data, schema=records_schema)
|
87
112
|
for data in normalized_data:
|
88
|
-
yield Record(data, stream_slice)
|
113
|
+
yield Record(data=data, stream_name=self.name, associated_slice=stream_slice)
|
89
114
|
|
90
115
|
def _normalize_by_schema(
|
91
116
|
self, records: Iterable[Mapping[str, Any]], schema: Optional[Mapping[str, Any]]
|
@@ -125,6 +150,9 @@ class RecordSelector(HttpSelector):
|
|
125
150
|
for record in records:
|
126
151
|
for transformation in self.transformations:
|
127
152
|
transformation.transform(
|
128
|
-
record,
|
129
|
-
|
153
|
+
record, # type: ignore # record has type Mapping[str, Any], but Dict[str, Any] expected
|
154
|
+
config=self.config,
|
155
|
+
stream_state=stream_state,
|
156
|
+
stream_slice=stream_slice,
|
157
|
+
)
|
130
158
|
yield record
|
@@ -10,9 +10,10 @@ from typing import Any, Dict, Iterable, Mapping, Optional, Tuple
|
|
10
10
|
|
11
11
|
import pandas as pd
|
12
12
|
import requests
|
13
|
-
from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
|
14
13
|
from numpy import nan
|
15
14
|
|
15
|
+
from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
|
16
|
+
|
16
17
|
EMPTY_STR: str = ""
|
17
18
|
DEFAULT_ENCODING: str = "utf-8"
|
18
19
|
DOWNLOAD_CHUNK_SIZE: int = 1024 * 10
|
@@ -7,6 +7,8 @@ from dataclasses import InitVar, dataclass, field
|
|
7
7
|
from datetime import timedelta
|
8
8
|
from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Union
|
9
9
|
|
10
|
+
from isodate import Duration, duration_isoformat, parse_duration
|
11
|
+
|
10
12
|
from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type
|
11
13
|
from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
|
12
14
|
from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
|
@@ -19,7 +21,6 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
|
|
19
21
|
)
|
20
22
|
from airbyte_cdk.sources.message import MessageRepository
|
21
23
|
from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
|
22
|
-
from isodate import Duration, duration_isoformat, parse_duration
|
23
24
|
|
24
25
|
|
25
26
|
@dataclass
|
@@ -340,8 +340,11 @@ class GlobalSubstreamCursor(DeclarativeCursor):
|
|
340
340
|
@staticmethod
|
341
341
|
def _convert_record_to_cursor_record(record: Record) -> Record:
|
342
342
|
return Record(
|
343
|
-
record.data,
|
344
|
-
|
343
|
+
data=record.data,
|
344
|
+
stream_name=record.stream_name,
|
345
|
+
associated_slice=StreamSlice(
|
346
|
+
partition={}, cursor_slice=record.associated_slice.cursor_slice
|
347
|
+
)
|
345
348
|
if record.associated_slice
|
346
349
|
else None,
|
347
350
|
)
|
@@ -325,8 +325,11 @@ class PerPartitionCursor(DeclarativeCursor):
|
|
325
325
|
@staticmethod
|
326
326
|
def _convert_record_to_cursor_record(record: Record) -> Record:
|
327
327
|
return Record(
|
328
|
-
record.data,
|
329
|
-
|
328
|
+
data=record.data,
|
329
|
+
stream_name=record.stream_name,
|
330
|
+
associated_slice=StreamSlice(
|
331
|
+
partition={}, cursor_slice=record.associated_slice.cursor_slice
|
332
|
+
)
|
330
333
|
if record.associated_slice
|
331
334
|
else None,
|
332
335
|
)
|
@@ -194,9 +194,7 @@ class PerPartitionWithGlobalCursor(DeclarativeCursor):
|
|
194
194
|
)
|
195
195
|
|
196
196
|
def should_be_synced(self, record: Record) -> bool:
|
197
|
-
return self.
|
198
|
-
record
|
199
|
-
) or self._per_partition_cursor.should_be_synced(record)
|
197
|
+
return self._get_active_cursor().should_be_synced(record)
|
200
198
|
|
201
199
|
def is_greater_than_or_equal(self, first: Record, second: Record) -> bool:
|
202
200
|
return self._global_cursor.is_greater_than_or_equal(first, second)
|
@@ -6,15 +6,16 @@ import ast
|
|
6
6
|
from functools import cache
|
7
7
|
from typing import Any, Mapping, Optional, Tuple, Type
|
8
8
|
|
9
|
-
from airbyte_cdk.sources.declarative.interpolation.filters import filters
|
10
|
-
from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
|
11
|
-
from airbyte_cdk.sources.declarative.interpolation.macros import macros
|
12
|
-
from airbyte_cdk.sources.types import Config
|
13
9
|
from jinja2 import meta
|
14
10
|
from jinja2.environment import Template
|
15
11
|
from jinja2.exceptions import UndefinedError
|
16
12
|
from jinja2.sandbox import SandboxedEnvironment
|
17
13
|
|
14
|
+
from airbyte_cdk.sources.declarative.interpolation.filters import filters
|
15
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
|
16
|
+
from airbyte_cdk.sources.declarative.interpolation.macros import macros
|
17
|
+
from airbyte_cdk.sources.types import Config
|
18
|
+
|
18
19
|
|
19
20
|
class StreamPartitionAccessEnvironment(SandboxedEnvironment):
|
20
21
|
"""
|
@@ -8,9 +8,12 @@ import pkgutil
|
|
8
8
|
from copy import deepcopy
|
9
9
|
from importlib import metadata
|
10
10
|
from typing import Any, Dict, Iterator, List, Mapping, Optional
|
11
|
-
from packaging.version import Version, InvalidVersion
|
12
11
|
|
13
12
|
import yaml
|
13
|
+
from jsonschema.exceptions import ValidationError
|
14
|
+
from jsonschema.validators import validate
|
15
|
+
from packaging.version import InvalidVersion, Version
|
16
|
+
|
14
17
|
from airbyte_cdk.models import (
|
15
18
|
AirbyteConnectionStatus,
|
16
19
|
AirbyteMessage,
|
@@ -44,8 +47,6 @@ from airbyte_cdk.sources.utils.slice_logger import (
|
|
44
47
|
DebugSliceLogger,
|
45
48
|
SliceLogger,
|
46
49
|
)
|
47
|
-
from jsonschema.exceptions import ValidationError
|
48
|
-
from jsonschema.validators import validate
|
49
50
|
|
50
51
|
|
51
52
|
class ManifestDeclarativeSource(DeclarativeSource):
|
@@ -5,9 +5,9 @@ from typing import Any, Mapping
|
|
5
5
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
6
6
|
from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
|
7
7
|
from airbyte_cdk.sources.declarative.models import (
|
8
|
+
CustomIncrementalSync,
|
8
9
|
DatetimeBasedCursor,
|
9
10
|
SubstreamPartitionRouter,
|
10
|
-
CustomIncrementalSync,
|
11
11
|
)
|
12
12
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import ParentStreamConfig
|
13
13
|
|
@@ -748,6 +748,123 @@ class NoPagination(BaseModel):
|
|
748
748
|
type: Literal["NoPagination"]
|
749
749
|
|
750
750
|
|
751
|
+
class State(BaseModel):
|
752
|
+
class Config:
|
753
|
+
extra = Extra.allow
|
754
|
+
|
755
|
+
min: int
|
756
|
+
max: int
|
757
|
+
|
758
|
+
|
759
|
+
class OauthConnectorInputSpecification(BaseModel):
|
760
|
+
class Config:
|
761
|
+
extra = Extra.allow
|
762
|
+
|
763
|
+
consent_url: str = Field(
|
764
|
+
...,
|
765
|
+
description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.",
|
766
|
+
examples=[
|
767
|
+
{
|
768
|
+
"consent_url": "https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}"
|
769
|
+
},
|
770
|
+
{
|
771
|
+
"consent_url": "https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}"
|
772
|
+
},
|
773
|
+
],
|
774
|
+
title="DeclarativeOAuth Consent URL",
|
775
|
+
)
|
776
|
+
scope: Optional[str] = Field(
|
777
|
+
None,
|
778
|
+
description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.",
|
779
|
+
examples=[{"scope": "user:read user:read_orders workspaces:read"}],
|
780
|
+
title="(Optional) DeclarativeOAuth Scope",
|
781
|
+
)
|
782
|
+
access_token_url: str = Field(
|
783
|
+
...,
|
784
|
+
description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.",
|
785
|
+
examples=[
|
786
|
+
{
|
787
|
+
"access_token_url": "https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
|
788
|
+
}
|
789
|
+
],
|
790
|
+
title="DeclarativeOAuth Access Token URL",
|
791
|
+
)
|
792
|
+
access_token_headers: Optional[Dict[str, Any]] = Field(
|
793
|
+
None,
|
794
|
+
description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.",
|
795
|
+
examples=[
|
796
|
+
{
|
797
|
+
"access_token_headers": {
|
798
|
+
"Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"
|
799
|
+
}
|
800
|
+
}
|
801
|
+
],
|
802
|
+
title="(Optional) DeclarativeOAuth Access Token Headers",
|
803
|
+
)
|
804
|
+
access_token_params: Optional[Dict[str, Any]] = Field(
|
805
|
+
None,
|
806
|
+
description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.",
|
807
|
+
examples=[
|
808
|
+
{
|
809
|
+
"access_token_params": {
|
810
|
+
"{auth_code_key}": "{{auth_code_key}}",
|
811
|
+
"{client_id_key}": "{{client_id_key}}",
|
812
|
+
"{client_secret_key}": "{{client_secret_key}}",
|
813
|
+
}
|
814
|
+
}
|
815
|
+
],
|
816
|
+
title="(Optional) DeclarativeOAuth Access Token Query Params (Json Encoded)",
|
817
|
+
)
|
818
|
+
extract_output: List[str] = Field(
|
819
|
+
...,
|
820
|
+
description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config. ",
|
821
|
+
examples=[{"extract_output": ["access_token", "refresh_token", "other_field"]}],
|
822
|
+
title="DeclarativeOAuth Extract Output",
|
823
|
+
)
|
824
|
+
state: Optional[State] = Field(
|
825
|
+
None,
|
826
|
+
description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity. ",
|
827
|
+
examples=[{"state": {"min": 7, "max": 128}}],
|
828
|
+
title="(Optional) DeclarativeOAuth Configurable State Query Param",
|
829
|
+
)
|
830
|
+
client_id_key: Optional[str] = Field(
|
831
|
+
None,
|
832
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.",
|
833
|
+
examples=[{"client_id_key": "my_custom_client_id_key_name"}],
|
834
|
+
title="(Optional) DeclarativeOAuth Client ID Key Override",
|
835
|
+
)
|
836
|
+
client_secret_key: Optional[str] = Field(
|
837
|
+
None,
|
838
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.",
|
839
|
+
examples=[{"client_secret_key": "my_custom_client_secret_key_name"}],
|
840
|
+
title="(Optional) DeclarativeOAuth Client Secret Key Override",
|
841
|
+
)
|
842
|
+
scope_key: Optional[str] = Field(
|
843
|
+
None,
|
844
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.",
|
845
|
+
examples=[{"scope_key": "my_custom_scope_key_key_name"}],
|
846
|
+
title="(Optional) DeclarativeOAuth Scope Key Override",
|
847
|
+
)
|
848
|
+
state_key: Optional[str] = Field(
|
849
|
+
None,
|
850
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider. ",
|
851
|
+
examples=[{"state_key": "my_custom_state_key_key_name"}],
|
852
|
+
title="(Optional) DeclarativeOAuth State Key Override",
|
853
|
+
)
|
854
|
+
auth_code_key: Optional[str] = Field(
|
855
|
+
None,
|
856
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider. ",
|
857
|
+
examples=[{"auth_code_key": "my_custom_auth_code_key_name"}],
|
858
|
+
title="(Optional) DeclarativeOAuth Auth Code Key Override",
|
859
|
+
)
|
860
|
+
redirect_uri_key: Optional[str] = Field(
|
861
|
+
None,
|
862
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.",
|
863
|
+
examples=[{"redirect_uri_key": "my_custom_redirect_uri_key_name"}],
|
864
|
+
title="(Optional) DeclarativeOAuth Redirect URI Key Override",
|
865
|
+
)
|
866
|
+
|
867
|
+
|
751
868
|
class OAuthConfigSpecification(BaseModel):
|
752
869
|
class Config:
|
753
870
|
extra = Extra.allow
|
@@ -766,6 +883,11 @@ class OAuthConfigSpecification(BaseModel):
|
|
766
883
|
],
|
767
884
|
title="OAuth user input",
|
768
885
|
)
|
886
|
+
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field(
|
887
|
+
None,
|
888
|
+
description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{my_var}`.\n- The nested resolution variables like `{{my_nested_var}}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}\n + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}\n + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}\n + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{auth_code_key}": "{{auth_code_key}}",\n "{client_id_key}": "{{client_id_key}}",\n "{client_secret_key}": "{{client_secret_key}}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }',
|
889
|
+
title="DeclarativeOAuth Connector Specification",
|
890
|
+
)
|
769
891
|
complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
|
770
892
|
None,
|
771
893
|
description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }",
|
@@ -17,6 +17,7 @@ from typing import (
|
|
17
17
|
Mapping,
|
18
18
|
MutableMapping,
|
19
19
|
Optional,
|
20
|
+
Tuple,
|
20
21
|
Type,
|
21
22
|
Union,
|
22
23
|
get_args,
|
@@ -24,6 +25,9 @@ from typing import (
|
|
24
25
|
get_type_hints,
|
25
26
|
)
|
26
27
|
|
28
|
+
from isodate import parse_duration
|
29
|
+
from pydantic.v1 import BaseModel
|
30
|
+
|
27
31
|
from airbyte_cdk.models import FailureType, Level
|
28
32
|
from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
|
29
33
|
from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncJobOrchestrator
|
@@ -366,8 +370,6 @@ from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_sta
|
|
366
370
|
from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction
|
367
371
|
from airbyte_cdk.sources.types import Config
|
368
372
|
from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
|
369
|
-
from isodate import parse_duration
|
370
|
-
from pydantic.v1 import BaseModel
|
371
373
|
|
372
374
|
ComponentDefinition = Mapping[str, Any]
|
373
375
|
|
@@ -752,7 +754,7 @@ class ModelToComponentFactory:
|
|
752
754
|
config: Config,
|
753
755
|
stream_state: MutableMapping[str, Any],
|
754
756
|
**kwargs: Any,
|
755
|
-
) -> ConcurrentCursor:
|
757
|
+
) -> Tuple[ConcurrentCursor, DateTimeStreamStateConverter]:
|
756
758
|
component_type = component_definition.get("type")
|
757
759
|
if component_definition.get("type") != model_type.__name__:
|
758
760
|
raise ValueError(
|
@@ -883,20 +885,23 @@ class ModelToComponentFactory:
|
|
883
885
|
if evaluated_step:
|
884
886
|
step_length = parse_duration(evaluated_step)
|
885
887
|
|
886
|
-
return
|
887
|
-
|
888
|
-
|
889
|
-
|
890
|
-
|
891
|
-
|
892
|
-
|
893
|
-
|
894
|
-
|
895
|
-
|
896
|
-
|
897
|
-
|
898
|
-
|
899
|
-
|
888
|
+
return (
|
889
|
+
ConcurrentCursor(
|
890
|
+
stream_name=stream_name,
|
891
|
+
stream_namespace=stream_namespace,
|
892
|
+
stream_state=stream_state,
|
893
|
+
message_repository=self._message_repository, # type: ignore # message_repository is always instantiated with a value by factory
|
894
|
+
connector_state_manager=state_manager,
|
895
|
+
connector_state_converter=connector_state_converter,
|
896
|
+
cursor_field=cursor_field,
|
897
|
+
slice_boundary_fields=slice_boundary_fields,
|
898
|
+
start=start_date, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice
|
899
|
+
end_provider=end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice
|
900
|
+
lookback_window=lookback_window,
|
901
|
+
slice_range=step_length,
|
902
|
+
cursor_granularity=cursor_granularity,
|
903
|
+
),
|
904
|
+
connector_state_converter,
|
900
905
|
)
|
901
906
|
|
902
907
|
@staticmethod
|
@@ -1776,6 +1781,7 @@ class ModelToComponentFactory:
|
|
1776
1781
|
self,
|
1777
1782
|
model: RecordSelectorModel,
|
1778
1783
|
config: Config,
|
1784
|
+
name: str,
|
1779
1785
|
*,
|
1780
1786
|
transformations: List[RecordTransformation],
|
1781
1787
|
decoder: Optional[Decoder] = None,
|
@@ -1806,6 +1812,7 @@ class ModelToComponentFactory:
|
|
1806
1812
|
|
1807
1813
|
return RecordSelector(
|
1808
1814
|
extractor=extractor,
|
1815
|
+
name=name,
|
1809
1816
|
config=config,
|
1810
1817
|
record_filter=record_filter,
|
1811
1818
|
transformations=transformations,
|
@@ -1876,6 +1883,7 @@ class ModelToComponentFactory:
|
|
1876
1883
|
)
|
1877
1884
|
record_selector = self._create_component_from_model(
|
1878
1885
|
model=model.record_selector,
|
1886
|
+
name=name,
|
1879
1887
|
config=config,
|
1880
1888
|
decoder=decoder,
|
1881
1889
|
transformations=transformations,
|
@@ -2030,6 +2038,7 @@ class ModelToComponentFactory:
|
|
2030
2038
|
requester=download_requester,
|
2031
2039
|
record_selector=RecordSelector(
|
2032
2040
|
extractor=ResponseToFileExtractor(),
|
2041
|
+
name=name,
|
2033
2042
|
record_filter=None,
|
2034
2043
|
transformations=[],
|
2035
2044
|
schema_normalization=TypeTransformer(TransformConfig.NoTransform),
|
@@ -7,6 +7,7 @@ from dataclasses import InitVar, dataclass
|
|
7
7
|
from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Union
|
8
8
|
|
9
9
|
import dpath
|
10
|
+
|
10
11
|
from airbyte_cdk.models import AirbyteMessage
|
11
12
|
from airbyte_cdk.models import Type as MessageType
|
12
13
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
10
11
|
from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
|
11
12
|
from airbyte_cdk.sources.types import Config
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
10
11
|
from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
|
11
12
|
from airbyte_cdk.sources.types import Config
|
@@ -7,6 +7,7 @@ from dataclasses import InitVar, dataclass
|
|
7
7
|
from typing import Any, Mapping, Optional, Union
|
8
8
|
|
9
9
|
import requests
|
10
|
+
|
10
11
|
from airbyte_cdk.models import FailureType
|
11
12
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
12
13
|
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
|
@@ -9,6 +9,7 @@ from dataclasses import InitVar, dataclass
|
|
9
9
|
from typing import Any, Mapping, Optional, Union
|
10
10
|
|
11
11
|
import requests
|
12
|
+
|
12
13
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
13
14
|
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
|
14
15
|
get_numeric_value_from_header,
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, List, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
|
10
11
|
from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
|
11
12
|
ErrorResolution,
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass, field
|
|
6
6
|
from typing import Any, List, Mapping, MutableMapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.declarative.requesters.error_handlers.default_http_response_filter import (
|
10
11
|
DefaultHttpResponseFilter,
|
11
12
|
)
|