airbyte-cdk 6.7.1__py3-none-any.whl → 6.7.1rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/cli/source_declarative_manifest/_run.py +1 -2
- airbyte_cdk/config_observation.py +1 -2
- airbyte_cdk/connector.py +0 -1
- airbyte_cdk/connector_builder/connector_builder_handler.py +1 -1
- airbyte_cdk/connector_builder/main.py +1 -2
- airbyte_cdk/destinations/destination.py +1 -2
- airbyte_cdk/destinations/vector_db_based/config.py +1 -2
- airbyte_cdk/destinations/vector_db_based/document_processor.py +3 -4
- airbyte_cdk/destinations/vector_db_based/embedder.py +4 -5
- airbyte_cdk/entrypoint.py +2 -3
- airbyte_cdk/logger.py +1 -2
- airbyte_cdk/models/__init__.py +0 -2
- airbyte_cdk/models/airbyte_protocol.py +1 -2
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +3 -3
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +1 -1
- airbyte_cdk/sources/config.py +1 -2
- airbyte_cdk/sources/declarative/auth/jwt.py +0 -1
- airbyte_cdk/sources/declarative/auth/oauth.py +0 -1
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +0 -1
- airbyte_cdk/sources/declarative/auth/token.py +1 -2
- airbyte_cdk/sources/declarative/auth/token_provider.py +2 -3
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +4 -6
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +0 -167
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +2 -3
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +0 -1
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +0 -1
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +0 -1
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +0 -1
- airbyte_cdk/sources/declarative/extractors/http_selector.py +0 -1
- airbyte_cdk/sources/declarative/extractors/record_filter.py +48 -6
- airbyte_cdk/sources/declarative/extractors/record_selector.py +4 -32
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +1 -2
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +1 -2
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +2 -5
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +2 -5
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +3 -1
- airbyte_cdk/sources/declarative/interpolation/jinja.py +4 -5
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +3 -4
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +1 -1
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +0 -122
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +2 -7
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +0 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +0 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +0 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +0 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +0 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +0 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +0 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +0 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +0 -1
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +1 -2
- airbyte_cdk/sources/declarative/requesters/http_requester.py +0 -1
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +0 -1
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +0 -1
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +0 -1
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +0 -1
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +0 -1
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +0 -1
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +0 -1
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +3 -9
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +1 -2
- airbyte_cdk/sources/declarative/requesters/requester.py +0 -1
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +1 -2
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +7 -12
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +4 -7
- airbyte_cdk/sources/declarative/transformations/add_fields.py +0 -1
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +0 -1
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +0 -1
- airbyte_cdk/sources/embedded/tools.py +0 -1
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +1 -2
- airbyte_cdk/sources/file_based/config/avro_format.py +1 -2
- airbyte_cdk/sources/file_based/config/csv_format.py +1 -2
- airbyte_cdk/sources/file_based/config/excel_format.py +1 -2
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +1 -2
- airbyte_cdk/sources/file_based/config/jsonl_format.py +1 -2
- airbyte_cdk/sources/file_based/config/parquet_format.py +1 -2
- airbyte_cdk/sources/file_based/config/unstructured_format.py +1 -2
- airbyte_cdk/sources/file_based/file_based_source.py +1 -2
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +1 -2
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +0 -1
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +1 -2
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +5 -5
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +1 -2
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +1 -2
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +8 -9
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +1 -2
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +4 -5
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +1 -1
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +1 -1
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +1 -1
- airbyte_cdk/sources/http_logger.py +0 -1
- airbyte_cdk/sources/streams/call_rate.py +2 -1
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +1 -2
- airbyte_cdk/sources/streams/concurrent/adapters.py +4 -8
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +1 -2
- airbyte_cdk/sources/streams/concurrent/cursor.py +6 -30
- airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -1
- airbyte_cdk/sources/streams/concurrent/partitions/record.py +35 -0
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +1 -1
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +1 -1
- airbyte_cdk/sources/streams/core.py +1 -2
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +1 -2
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +0 -1
- airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +0 -1
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +1 -2
- airbyte_cdk/sources/streams/http/http.py +2 -3
- airbyte_cdk/sources/streams/http/http_client.py +8 -49
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +1 -2
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +0 -1
- airbyte_cdk/sources/types.py +1 -14
- airbyte_cdk/sources/utils/schema_helpers.py +2 -3
- airbyte_cdk/sql/secrets.py +1 -2
- airbyte_cdk/sql/shared/sql_processor.py +6 -8
- airbyte_cdk/test/entrypoint_wrapper.py +3 -4
- airbyte_cdk/test/mock_http/mocker.py +0 -1
- airbyte_cdk/utils/schema_inferrer.py +1 -2
- airbyte_cdk/utils/slice_hasher.py +1 -1
- airbyte_cdk/utils/traced_exception.py +1 -2
- {airbyte_cdk-6.7.1.dist-info → airbyte_cdk-6.7.1rc1.dist-info}/METADATA +2 -9
- {airbyte_cdk-6.7.1.dist-info → airbyte_cdk-6.7.1rc1.dist-info}/RECORD +123 -122
- {airbyte_cdk-6.7.1.dist-info → airbyte_cdk-6.7.1rc1.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.7.1.dist-info → airbyte_cdk-6.7.1rc1.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.7.1.dist-info → airbyte_cdk-6.7.1rc1.dist-info}/entry_points.txt +0 -0
@@ -1,6 +1,7 @@
|
|
1
1
|
#
|
2
2
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
3
|
#
|
4
|
+
import datetime
|
4
5
|
from dataclasses import InitVar, dataclass
|
5
6
|
from typing import Any, Iterable, Mapping, Optional, Union
|
6
7
|
|
@@ -10,7 +11,7 @@ from airbyte_cdk.sources.declarative.incremental import (
|
|
10
11
|
PerPartitionWithGlobalCursor,
|
11
12
|
)
|
12
13
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
|
13
|
-
from airbyte_cdk.sources.types import Config,
|
14
|
+
from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
|
14
15
|
|
15
16
|
|
16
17
|
@dataclass
|
@@ -67,6 +68,20 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
|
|
67
68
|
self._date_time_based_cursor = date_time_based_cursor
|
68
69
|
self._substream_cursor = substream_cursor
|
69
70
|
|
71
|
+
@property
|
72
|
+
def _cursor_field(self) -> str:
|
73
|
+
return self._date_time_based_cursor.cursor_field.eval(self._date_time_based_cursor.config) # type: ignore # eval returns a string in this context
|
74
|
+
|
75
|
+
@property
|
76
|
+
def _start_date_from_config(self) -> datetime.datetime:
|
77
|
+
return self._date_time_based_cursor._start_datetime.get_datetime(
|
78
|
+
self._date_time_based_cursor.config
|
79
|
+
)
|
80
|
+
|
81
|
+
@property
|
82
|
+
def _end_datetime(self) -> datetime.datetime:
|
83
|
+
return self._date_time_based_cursor.select_best_end_datetime()
|
84
|
+
|
70
85
|
def filter_records(
|
71
86
|
self,
|
72
87
|
records: Iterable[Mapping[str, Any]],
|
@@ -74,14 +89,16 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
|
|
74
89
|
stream_slice: Optional[StreamSlice] = None,
|
75
90
|
next_page_token: Optional[Mapping[str, Any]] = None,
|
76
91
|
) -> Iterable[Mapping[str, Any]]:
|
92
|
+
state_value = self._get_state_value(
|
93
|
+
stream_state, stream_slice or StreamSlice(partition={}, cursor_slice={})
|
94
|
+
)
|
95
|
+
filter_date: datetime.datetime = self._get_filter_date(state_value)
|
77
96
|
records = (
|
78
97
|
record
|
79
98
|
for record in records
|
80
|
-
if
|
81
|
-
|
82
|
-
|
83
|
-
Record(data=record, associated_slice=stream_slice, stream_name="")
|
84
|
-
)
|
99
|
+
if self._end_datetime
|
100
|
+
>= self._date_time_based_cursor.parse_date(record[self._cursor_field])
|
101
|
+
>= filter_date
|
85
102
|
)
|
86
103
|
if self.condition:
|
87
104
|
records = super().filter_records(
|
@@ -91,3 +108,28 @@ class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
|
|
91
108
|
next_page_token=next_page_token,
|
92
109
|
)
|
93
110
|
yield from records
|
111
|
+
|
112
|
+
def _get_state_value(
|
113
|
+
self, stream_state: StreamState, stream_slice: StreamSlice
|
114
|
+
) -> Optional[str]:
|
115
|
+
"""
|
116
|
+
Return cursor_value or None in case it was not found.
|
117
|
+
Cursor_value may be empty if:
|
118
|
+
1. It is an initial sync => no stream_state exist at all.
|
119
|
+
2. In Parent-child stream, and we already make initial sync, so stream_state is present.
|
120
|
+
During the second read, we receive one extra record from parent and therefore no stream_state for this record will be found.
|
121
|
+
|
122
|
+
:param StreamState stream_state: State
|
123
|
+
:param StreamSlice stream_slice: Current Stream slice
|
124
|
+
:return Optional[str]: cursor_value in case it was found, otherwise None.
|
125
|
+
"""
|
126
|
+
state = (self._substream_cursor or self._date_time_based_cursor).select_state(stream_slice)
|
127
|
+
|
128
|
+
return state.get(self._cursor_field) if state else None
|
129
|
+
|
130
|
+
def _get_filter_date(self, state_value: Optional[str]) -> datetime.datetime:
|
131
|
+
start_date_parsed = self._start_date_from_config
|
132
|
+
if state_value:
|
133
|
+
return max(start_date_parsed, self._date_time_based_cursor.parse_date(state_value))
|
134
|
+
else:
|
135
|
+
return start_date_parsed
|
@@ -3,14 +3,12 @@
|
|
3
3
|
#
|
4
4
|
|
5
5
|
from dataclasses import InitVar, dataclass, field
|
6
|
-
from typing import Any, Iterable, List, Mapping, Optional
|
6
|
+
from typing import Any, Iterable, List, Mapping, Optional
|
7
7
|
|
8
8
|
import requests
|
9
|
-
|
10
9
|
from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
|
11
10
|
from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
|
12
11
|
from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
|
13
|
-
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
14
12
|
from airbyte_cdk.sources.declarative.models import SchemaNormalization
|
15
13
|
from airbyte_cdk.sources.declarative.transformations import RecordTransformation
|
16
14
|
from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
|
@@ -39,34 +37,11 @@ class RecordSelector(HttpSelector):
|
|
39
37
|
config: Config
|
40
38
|
parameters: InitVar[Mapping[str, Any]]
|
41
39
|
schema_normalization: TypeTransformer
|
42
|
-
name: str
|
43
|
-
_name: Union[InterpolatedString, str] = field(init=False, repr=False, default="")
|
44
40
|
record_filter: Optional[RecordFilter] = None
|
45
41
|
transformations: List[RecordTransformation] = field(default_factory=lambda: [])
|
46
42
|
|
47
43
|
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
48
44
|
self._parameters = parameters
|
49
|
-
self._name = (
|
50
|
-
InterpolatedString(self._name, parameters=parameters)
|
51
|
-
if isinstance(self._name, str)
|
52
|
-
else self._name
|
53
|
-
)
|
54
|
-
|
55
|
-
@property # type: ignore
|
56
|
-
def name(self) -> str:
|
57
|
-
"""
|
58
|
-
:return: Stream name
|
59
|
-
"""
|
60
|
-
return (
|
61
|
-
str(self._name.eval(self.config))
|
62
|
-
if isinstance(self._name, InterpolatedString)
|
63
|
-
else self._name
|
64
|
-
)
|
65
|
-
|
66
|
-
@name.setter
|
67
|
-
def name(self, value: str) -> None:
|
68
|
-
if not isinstance(value, property):
|
69
|
-
self._name = value
|
70
45
|
|
71
46
|
def select_records(
|
72
47
|
self,
|
@@ -110,7 +85,7 @@ class RecordSelector(HttpSelector):
|
|
110
85
|
transformed_data = self._transform(filtered_data, stream_state, stream_slice)
|
111
86
|
normalized_data = self._normalize_by_schema(transformed_data, schema=records_schema)
|
112
87
|
for data in normalized_data:
|
113
|
-
yield Record(data
|
88
|
+
yield Record(data, stream_slice)
|
114
89
|
|
115
90
|
def _normalize_by_schema(
|
116
91
|
self, records: Iterable[Mapping[str, Any]], schema: Optional[Mapping[str, Any]]
|
@@ -150,9 +125,6 @@ class RecordSelector(HttpSelector):
|
|
150
125
|
for record in records:
|
151
126
|
for transformation in self.transformations:
|
152
127
|
transformation.transform(
|
153
|
-
record,
|
154
|
-
|
155
|
-
stream_state=stream_state,
|
156
|
-
stream_slice=stream_slice,
|
157
|
-
)
|
128
|
+
record, config=self.config, stream_state=stream_state, stream_slice=stream_slice
|
129
|
+
) # type: ignore # record has type Mapping[str, Any], but Dict[str, Any] expected
|
158
130
|
yield record
|
@@ -10,9 +10,8 @@ from typing import Any, Dict, Iterable, Mapping, Optional, Tuple
|
|
10
10
|
|
11
11
|
import pandas as pd
|
12
12
|
import requests
|
13
|
-
from numpy import nan
|
14
|
-
|
15
13
|
from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
|
14
|
+
from numpy import nan
|
16
15
|
|
17
16
|
EMPTY_STR: str = ""
|
18
17
|
DEFAULT_ENCODING: str = "utf-8"
|
@@ -7,8 +7,6 @@ from dataclasses import InitVar, dataclass, field
|
|
7
7
|
from datetime import timedelta
|
8
8
|
from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Union
|
9
9
|
|
10
|
-
from isodate import Duration, duration_isoformat, parse_duration
|
11
|
-
|
12
10
|
from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type
|
13
11
|
from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
|
14
12
|
from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
|
@@ -21,6 +19,7 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
|
|
21
19
|
)
|
22
20
|
from airbyte_cdk.sources.message import MessageRepository
|
23
21
|
from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
|
22
|
+
from isodate import Duration, duration_isoformat, parse_duration
|
24
23
|
|
25
24
|
|
26
25
|
@dataclass
|
@@ -340,11 +340,8 @@ class GlobalSubstreamCursor(DeclarativeCursor):
|
|
340
340
|
@staticmethod
|
341
341
|
def _convert_record_to_cursor_record(record: Record) -> Record:
|
342
342
|
return Record(
|
343
|
-
|
344
|
-
|
345
|
-
associated_slice=StreamSlice(
|
346
|
-
partition={}, cursor_slice=record.associated_slice.cursor_slice
|
347
|
-
)
|
343
|
+
record.data,
|
344
|
+
StreamSlice(partition={}, cursor_slice=record.associated_slice.cursor_slice)
|
348
345
|
if record.associated_slice
|
349
346
|
else None,
|
350
347
|
)
|
@@ -325,11 +325,8 @@ class PerPartitionCursor(DeclarativeCursor):
|
|
325
325
|
@staticmethod
|
326
326
|
def _convert_record_to_cursor_record(record: Record) -> Record:
|
327
327
|
return Record(
|
328
|
-
|
329
|
-
|
330
|
-
associated_slice=StreamSlice(
|
331
|
-
partition={}, cursor_slice=record.associated_slice.cursor_slice
|
332
|
-
)
|
328
|
+
record.data,
|
329
|
+
StreamSlice(partition={}, cursor_slice=record.associated_slice.cursor_slice)
|
333
330
|
if record.associated_slice
|
334
331
|
else None,
|
335
332
|
)
|
@@ -194,7 +194,9 @@ class PerPartitionWithGlobalCursor(DeclarativeCursor):
|
|
194
194
|
)
|
195
195
|
|
196
196
|
def should_be_synced(self, record: Record) -> bool:
|
197
|
-
return self.
|
197
|
+
return self._global_cursor.should_be_synced(
|
198
|
+
record
|
199
|
+
) or self._per_partition_cursor.should_be_synced(record)
|
198
200
|
|
199
201
|
def is_greater_than_or_equal(self, first: Record, second: Record) -> bool:
|
200
202
|
return self._global_cursor.is_greater_than_or_equal(first, second)
|
@@ -6,15 +6,14 @@ import ast
|
|
6
6
|
from functools import cache
|
7
7
|
from typing import Any, Mapping, Optional, Tuple, Type
|
8
8
|
|
9
|
-
from jinja2 import meta
|
10
|
-
from jinja2.environment import Template
|
11
|
-
from jinja2.exceptions import UndefinedError
|
12
|
-
from jinja2.sandbox import SandboxedEnvironment
|
13
|
-
|
14
9
|
from airbyte_cdk.sources.declarative.interpolation.filters import filters
|
15
10
|
from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
|
16
11
|
from airbyte_cdk.sources.declarative.interpolation.macros import macros
|
17
12
|
from airbyte_cdk.sources.types import Config
|
13
|
+
from jinja2 import meta
|
14
|
+
from jinja2.environment import Template
|
15
|
+
from jinja2.exceptions import UndefinedError
|
16
|
+
from jinja2.sandbox import SandboxedEnvironment
|
18
17
|
|
19
18
|
|
20
19
|
class StreamPartitionAccessEnvironment(SandboxedEnvironment):
|
@@ -8,12 +8,9 @@ import pkgutil
|
|
8
8
|
from copy import deepcopy
|
9
9
|
from importlib import metadata
|
10
10
|
from typing import Any, Dict, Iterator, List, Mapping, Optional
|
11
|
+
from packaging.version import Version, InvalidVersion
|
11
12
|
|
12
13
|
import yaml
|
13
|
-
from jsonschema.exceptions import ValidationError
|
14
|
-
from jsonschema.validators import validate
|
15
|
-
from packaging.version import InvalidVersion, Version
|
16
|
-
|
17
14
|
from airbyte_cdk.models import (
|
18
15
|
AirbyteConnectionStatus,
|
19
16
|
AirbyteMessage,
|
@@ -47,6 +44,8 @@ from airbyte_cdk.sources.utils.slice_logger import (
|
|
47
44
|
DebugSliceLogger,
|
48
45
|
SliceLogger,
|
49
46
|
)
|
47
|
+
from jsonschema.exceptions import ValidationError
|
48
|
+
from jsonschema.validators import validate
|
50
49
|
|
51
50
|
|
52
51
|
class ManifestDeclarativeSource(DeclarativeSource):
|
@@ -5,9 +5,9 @@ from typing import Any, Mapping
|
|
5
5
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
6
6
|
from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
|
7
7
|
from airbyte_cdk.sources.declarative.models import (
|
8
|
-
CustomIncrementalSync,
|
9
8
|
DatetimeBasedCursor,
|
10
9
|
SubstreamPartitionRouter,
|
10
|
+
CustomIncrementalSync,
|
11
11
|
)
|
12
12
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import ParentStreamConfig
|
13
13
|
|
@@ -748,123 +748,6 @@ class NoPagination(BaseModel):
|
|
748
748
|
type: Literal["NoPagination"]
|
749
749
|
|
750
750
|
|
751
|
-
class State(BaseModel):
|
752
|
-
class Config:
|
753
|
-
extra = Extra.allow
|
754
|
-
|
755
|
-
min: int
|
756
|
-
max: int
|
757
|
-
|
758
|
-
|
759
|
-
class OauthConnectorInputSpecification(BaseModel):
|
760
|
-
class Config:
|
761
|
-
extra = Extra.allow
|
762
|
-
|
763
|
-
consent_url: str = Field(
|
764
|
-
...,
|
765
|
-
description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.",
|
766
|
-
examples=[
|
767
|
-
{
|
768
|
-
"consent_url": "https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}"
|
769
|
-
},
|
770
|
-
{
|
771
|
-
"consent_url": "https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}"
|
772
|
-
},
|
773
|
-
],
|
774
|
-
title="DeclarativeOAuth Consent URL",
|
775
|
-
)
|
776
|
-
scope: Optional[str] = Field(
|
777
|
-
None,
|
778
|
-
description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.",
|
779
|
-
examples=[{"scope": "user:read user:read_orders workspaces:read"}],
|
780
|
-
title="(Optional) DeclarativeOAuth Scope",
|
781
|
-
)
|
782
|
-
access_token_url: str = Field(
|
783
|
-
...,
|
784
|
-
description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.",
|
785
|
-
examples=[
|
786
|
-
{
|
787
|
-
"access_token_url": "https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
|
788
|
-
}
|
789
|
-
],
|
790
|
-
title="DeclarativeOAuth Access Token URL",
|
791
|
-
)
|
792
|
-
access_token_headers: Optional[Dict[str, Any]] = Field(
|
793
|
-
None,
|
794
|
-
description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.",
|
795
|
-
examples=[
|
796
|
-
{
|
797
|
-
"access_token_headers": {
|
798
|
-
"Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"
|
799
|
-
}
|
800
|
-
}
|
801
|
-
],
|
802
|
-
title="(Optional) DeclarativeOAuth Access Token Headers",
|
803
|
-
)
|
804
|
-
access_token_params: Optional[Dict[str, Any]] = Field(
|
805
|
-
None,
|
806
|
-
description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.",
|
807
|
-
examples=[
|
808
|
-
{
|
809
|
-
"access_token_params": {
|
810
|
-
"{auth_code_key}": "{{auth_code_key}}",
|
811
|
-
"{client_id_key}": "{{client_id_key}}",
|
812
|
-
"{client_secret_key}": "{{client_secret_key}}",
|
813
|
-
}
|
814
|
-
}
|
815
|
-
],
|
816
|
-
title="(Optional) DeclarativeOAuth Access Token Query Params (Json Encoded)",
|
817
|
-
)
|
818
|
-
extract_output: List[str] = Field(
|
819
|
-
...,
|
820
|
-
description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config. ",
|
821
|
-
examples=[{"extract_output": ["access_token", "refresh_token", "other_field"]}],
|
822
|
-
title="DeclarativeOAuth Extract Output",
|
823
|
-
)
|
824
|
-
state: Optional[State] = Field(
|
825
|
-
None,
|
826
|
-
description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity. ",
|
827
|
-
examples=[{"state": {"min": 7, "max": 128}}],
|
828
|
-
title="(Optional) DeclarativeOAuth Configurable State Query Param",
|
829
|
-
)
|
830
|
-
client_id_key: Optional[str] = Field(
|
831
|
-
None,
|
832
|
-
description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.",
|
833
|
-
examples=[{"client_id_key": "my_custom_client_id_key_name"}],
|
834
|
-
title="(Optional) DeclarativeOAuth Client ID Key Override",
|
835
|
-
)
|
836
|
-
client_secret_key: Optional[str] = Field(
|
837
|
-
None,
|
838
|
-
description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.",
|
839
|
-
examples=[{"client_secret_key": "my_custom_client_secret_key_name"}],
|
840
|
-
title="(Optional) DeclarativeOAuth Client Secret Key Override",
|
841
|
-
)
|
842
|
-
scope_key: Optional[str] = Field(
|
843
|
-
None,
|
844
|
-
description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.",
|
845
|
-
examples=[{"scope_key": "my_custom_scope_key_key_name"}],
|
846
|
-
title="(Optional) DeclarativeOAuth Scope Key Override",
|
847
|
-
)
|
848
|
-
state_key: Optional[str] = Field(
|
849
|
-
None,
|
850
|
-
description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider. ",
|
851
|
-
examples=[{"state_key": "my_custom_state_key_key_name"}],
|
852
|
-
title="(Optional) DeclarativeOAuth State Key Override",
|
853
|
-
)
|
854
|
-
auth_code_key: Optional[str] = Field(
|
855
|
-
None,
|
856
|
-
description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider. ",
|
857
|
-
examples=[{"auth_code_key": "my_custom_auth_code_key_name"}],
|
858
|
-
title="(Optional) DeclarativeOAuth Auth Code Key Override",
|
859
|
-
)
|
860
|
-
redirect_uri_key: Optional[str] = Field(
|
861
|
-
None,
|
862
|
-
description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.",
|
863
|
-
examples=[{"redirect_uri_key": "my_custom_redirect_uri_key_name"}],
|
864
|
-
title="(Optional) DeclarativeOAuth Redirect URI Key Override",
|
865
|
-
)
|
866
|
-
|
867
|
-
|
868
751
|
class OAuthConfigSpecification(BaseModel):
|
869
752
|
class Config:
|
870
753
|
extra = Extra.allow
|
@@ -883,11 +766,6 @@ class OAuthConfigSpecification(BaseModel):
|
|
883
766
|
],
|
884
767
|
title="OAuth user input",
|
885
768
|
)
|
886
|
-
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field(
|
887
|
-
None,
|
888
|
-
description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{my_var}`.\n- The nested resolution variables like `{{my_nested_var}}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}\n + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}\n + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}\n + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{auth_code_key}": "{{auth_code_key}}",\n "{client_id_key}": "{{client_id_key}}",\n "{client_secret_key}": "{{client_secret_key}}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }',
|
889
|
-
title="DeclarativeOAuth Connector Specification",
|
890
|
-
)
|
891
769
|
complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
|
892
770
|
None,
|
893
771
|
description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }",
|
@@ -25,9 +25,6 @@ from typing import (
|
|
25
25
|
get_type_hints,
|
26
26
|
)
|
27
27
|
|
28
|
-
from isodate import parse_duration
|
29
|
-
from pydantic.v1 import BaseModel
|
30
|
-
|
31
28
|
from airbyte_cdk.models import FailureType, Level
|
32
29
|
from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
|
33
30
|
from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncJobOrchestrator
|
@@ -370,6 +367,8 @@ from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_sta
|
|
370
367
|
from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction
|
371
368
|
from airbyte_cdk.sources.types import Config
|
372
369
|
from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
|
370
|
+
from isodate import parse_duration
|
371
|
+
from pydantic.v1 import BaseModel
|
373
372
|
|
374
373
|
ComponentDefinition = Mapping[str, Any]
|
375
374
|
|
@@ -1781,7 +1780,6 @@ class ModelToComponentFactory:
|
|
1781
1780
|
self,
|
1782
1781
|
model: RecordSelectorModel,
|
1783
1782
|
config: Config,
|
1784
|
-
name: str,
|
1785
1783
|
*,
|
1786
1784
|
transformations: List[RecordTransformation],
|
1787
1785
|
decoder: Optional[Decoder] = None,
|
@@ -1812,7 +1810,6 @@ class ModelToComponentFactory:
|
|
1812
1810
|
|
1813
1811
|
return RecordSelector(
|
1814
1812
|
extractor=extractor,
|
1815
|
-
name=name,
|
1816
1813
|
config=config,
|
1817
1814
|
record_filter=record_filter,
|
1818
1815
|
transformations=transformations,
|
@@ -1883,7 +1880,6 @@ class ModelToComponentFactory:
|
|
1883
1880
|
)
|
1884
1881
|
record_selector = self._create_component_from_model(
|
1885
1882
|
model=model.record_selector,
|
1886
|
-
name=name,
|
1887
1883
|
config=config,
|
1888
1884
|
decoder=decoder,
|
1889
1885
|
transformations=transformations,
|
@@ -2038,7 +2034,6 @@ class ModelToComponentFactory:
|
|
2038
2034
|
requester=download_requester,
|
2039
2035
|
record_selector=RecordSelector(
|
2040
2036
|
extractor=ResponseToFileExtractor(),
|
2041
|
-
name=name,
|
2042
2037
|
record_filter=None,
|
2043
2038
|
transformations=[],
|
2044
2039
|
schema_normalization=TypeTransformer(TransformConfig.NoTransform),
|
@@ -7,7 +7,6 @@ from dataclasses import InitVar, dataclass
|
|
7
7
|
from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Union
|
8
8
|
|
9
9
|
import dpath
|
10
|
-
|
11
10
|
from airbyte_cdk.models import AirbyteMessage
|
12
11
|
from airbyte_cdk.models import Type as MessageType
|
13
12
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
-
|
10
9
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
11
10
|
from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
|
12
11
|
from airbyte_cdk.sources.types import Config
|
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
-
|
10
9
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
11
10
|
from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
|
12
11
|
from airbyte_cdk.sources.types import Config
|
@@ -7,7 +7,6 @@ from dataclasses import InitVar, dataclass
|
|
7
7
|
from typing import Any, Mapping, Optional, Union
|
8
8
|
|
9
9
|
import requests
|
10
|
-
|
11
10
|
from airbyte_cdk.models import FailureType
|
12
11
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
13
12
|
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
|
@@ -9,7 +9,6 @@ from dataclasses import InitVar, dataclass
|
|
9
9
|
from typing import Any, Mapping, Optional, Union
|
10
10
|
|
11
11
|
import requests
|
12
|
-
|
13
12
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
14
13
|
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
|
15
14
|
get_numeric_value_from_header,
|
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, List, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
-
|
10
9
|
from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
|
11
10
|
from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
|
12
11
|
ErrorResolution,
|
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass, field
|
|
6
6
|
from typing import Any, List, Mapping, MutableMapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
-
|
10
9
|
from airbyte_cdk.sources.declarative.requesters.error_handlers.default_http_response_filter import (
|
11
10
|
DefaultHttpResponseFilter,
|
12
11
|
)
|
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, Optional, Set, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
-
|
10
9
|
from airbyte_cdk.models import FailureType
|
11
10
|
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
12
11
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
|
@@ -6,8 +6,6 @@ from datetime import timedelta
|
|
6
6
|
from typing import Any, Dict, Iterable, Mapping, Optional
|
7
7
|
|
8
8
|
import requests
|
9
|
-
from requests import Response
|
10
|
-
|
11
9
|
from airbyte_cdk import AirbyteMessage
|
12
10
|
from airbyte_cdk.logger import lazy_log
|
13
11
|
from airbyte_cdk.models import FailureType, Type
|
@@ -25,6 +23,7 @@ from airbyte_cdk.sources.declarative.requesters.requester import Requester
|
|
25
23
|
from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
|
26
24
|
from airbyte_cdk.sources.types import Record, StreamSlice
|
27
25
|
from airbyte_cdk.utils import AirbyteTracedException
|
26
|
+
from requests import Response
|
28
27
|
|
29
28
|
LOGGER = logging.getLogger("airbyte")
|
30
29
|
|
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, MutableMapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
-
|
10
9
|
from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator
|
11
10
|
from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
|
12
11
|
|
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
-
|
10
9
|
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
11
10
|
from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
|
12
11
|
PaginationStrategy,
|