airbyte-cdk 6.7.0__py3-none-any.whl → 6.7.0.dev5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/cli/source_declarative_manifest/_run.py +1 -2
- airbyte_cdk/config_observation.py +1 -2
- airbyte_cdk/connector.py +0 -1
- airbyte_cdk/connector_builder/connector_builder_handler.py +1 -1
- airbyte_cdk/connector_builder/main.py +1 -2
- airbyte_cdk/destinations/destination.py +1 -2
- airbyte_cdk/destinations/vector_db_based/config.py +1 -2
- airbyte_cdk/destinations/vector_db_based/document_processor.py +3 -4
- airbyte_cdk/destinations/vector_db_based/embedder.py +4 -5
- airbyte_cdk/entrypoint.py +2 -3
- airbyte_cdk/logger.py +1 -2
- airbyte_cdk/models/airbyte_protocol.py +1 -2
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +3 -3
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +1 -1
- airbyte_cdk/sources/config.py +1 -2
- airbyte_cdk/sources/declarative/auth/jwt.py +0 -1
- airbyte_cdk/sources/declarative/auth/oauth.py +0 -1
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +0 -1
- airbyte_cdk/sources/declarative/auth/token.py +1 -2
- airbyte_cdk/sources/declarative/auth/token_provider.py +2 -3
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +4 -6
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +13 -13
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +2 -3
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +0 -1
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +0 -1
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +0 -1
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +0 -1
- airbyte_cdk/sources/declarative/extractors/http_selector.py +0 -1
- airbyte_cdk/sources/declarative/extractors/record_filter.py +48 -6
- airbyte_cdk/sources/declarative/extractors/record_selector.py +4 -32
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +1 -2
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +1 -2
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +2 -5
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +2 -5
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +3 -1
- airbyte_cdk/sources/declarative/interpolation/jinja.py +4 -5
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +3 -4
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +1 -1
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +2 -7
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +0 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +0 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +0 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +0 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +0 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +0 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +0 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +0 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +0 -1
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +1 -2
- airbyte_cdk/sources/declarative/requesters/http_requester.py +0 -1
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +0 -1
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +0 -1
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +0 -1
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +0 -1
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +0 -1
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +0 -1
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +0 -1
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +3 -9
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +1 -2
- airbyte_cdk/sources/declarative/requesters/requester.py +0 -1
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +1 -2
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +4 -12
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +4 -7
- airbyte_cdk/sources/declarative/transformations/add_fields.py +0 -1
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +0 -1
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +0 -1
- airbyte_cdk/sources/embedded/tools.py +0 -1
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +1 -2
- airbyte_cdk/sources/file_based/config/avro_format.py +1 -2
- airbyte_cdk/sources/file_based/config/csv_format.py +1 -2
- airbyte_cdk/sources/file_based/config/excel_format.py +1 -2
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +1 -2
- airbyte_cdk/sources/file_based/config/jsonl_format.py +1 -2
- airbyte_cdk/sources/file_based/config/parquet_format.py +1 -2
- airbyte_cdk/sources/file_based/config/unstructured_format.py +1 -2
- airbyte_cdk/sources/file_based/file_based_source.py +1 -2
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +1 -2
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +0 -1
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +1 -2
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +5 -5
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +1 -2
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +1 -2
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +8 -9
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +1 -2
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +4 -5
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +1 -1
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +1 -1
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +1 -1
- airbyte_cdk/sources/http_logger.py +0 -1
- airbyte_cdk/sources/streams/call_rate.py +2 -1
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +1 -2
- airbyte_cdk/sources/streams/concurrent/adapters.py +4 -8
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +1 -2
- airbyte_cdk/sources/streams/concurrent/cursor.py +6 -30
- airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -1
- airbyte_cdk/sources/streams/concurrent/partitions/record.py +35 -0
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +1 -1
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +1 -1
- airbyte_cdk/sources/streams/core.py +1 -2
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +1 -2
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +0 -1
- airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +0 -1
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +1 -2
- airbyte_cdk/sources/streams/http/http.py +2 -3
- airbyte_cdk/sources/streams/http/http_client.py +2 -49
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +1 -2
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +0 -1
- airbyte_cdk/sources/types.py +1 -14
- airbyte_cdk/sources/utils/schema_helpers.py +2 -3
- airbyte_cdk/sql/secrets.py +1 -2
- airbyte_cdk/sql/shared/sql_processor.py +6 -8
- airbyte_cdk/test/entrypoint_wrapper.py +3 -4
- airbyte_cdk/test/mock_http/mocker.py +0 -1
- airbyte_cdk/utils/schema_inferrer.py +1 -2
- airbyte_cdk/utils/slice_hasher.py +1 -1
- airbyte_cdk/utils/traced_exception.py +1 -2
- {airbyte_cdk-6.7.0.dist-info → airbyte_cdk-6.7.0.dev5.dist-info}/METADATA +1 -8
- {airbyte_cdk-6.7.0.dist-info → airbyte_cdk-6.7.0.dev5.dist-info}/RECORD +121 -120
- {airbyte_cdk-6.7.0.dist-info → airbyte_cdk-6.7.0.dev5.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.7.0.dist-info → airbyte_cdk-6.7.0.dev5.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.7.0.dist-info → airbyte_cdk-6.7.0.dev5.dist-info}/entry_points.txt +0 -0
@@ -3,14 +3,12 @@
|
|
3
3
|
#
|
4
4
|
|
5
5
|
from dataclasses import InitVar, dataclass, field
|
6
|
-
from typing import Any, Iterable, List, Mapping, Optional
|
6
|
+
from typing import Any, Iterable, List, Mapping, Optional
|
7
7
|
|
8
8
|
import requests
|
9
|
-
|
10
9
|
from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
|
11
10
|
from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
|
12
11
|
from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
|
13
|
-
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
14
12
|
from airbyte_cdk.sources.declarative.models import SchemaNormalization
|
15
13
|
from airbyte_cdk.sources.declarative.transformations import RecordTransformation
|
16
14
|
from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
|
@@ -39,34 +37,11 @@ class RecordSelector(HttpSelector):
|
|
39
37
|
config: Config
|
40
38
|
parameters: InitVar[Mapping[str, Any]]
|
41
39
|
schema_normalization: TypeTransformer
|
42
|
-
name: str
|
43
|
-
_name: Union[InterpolatedString, str] = field(init=False, repr=False, default="")
|
44
40
|
record_filter: Optional[RecordFilter] = None
|
45
41
|
transformations: List[RecordTransformation] = field(default_factory=lambda: [])
|
46
42
|
|
47
43
|
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
48
44
|
self._parameters = parameters
|
49
|
-
self._name = (
|
50
|
-
InterpolatedString(self._name, parameters=parameters)
|
51
|
-
if isinstance(self._name, str)
|
52
|
-
else self._name
|
53
|
-
)
|
54
|
-
|
55
|
-
@property # type: ignore
|
56
|
-
def name(self) -> str:
|
57
|
-
"""
|
58
|
-
:return: Stream name
|
59
|
-
"""
|
60
|
-
return (
|
61
|
-
str(self._name.eval(self.config))
|
62
|
-
if isinstance(self._name, InterpolatedString)
|
63
|
-
else self._name
|
64
|
-
)
|
65
|
-
|
66
|
-
@name.setter
|
67
|
-
def name(self, value: str) -> None:
|
68
|
-
if not isinstance(value, property):
|
69
|
-
self._name = value
|
70
45
|
|
71
46
|
def select_records(
|
72
47
|
self,
|
@@ -110,7 +85,7 @@ class RecordSelector(HttpSelector):
|
|
110
85
|
transformed_data = self._transform(filtered_data, stream_state, stream_slice)
|
111
86
|
normalized_data = self._normalize_by_schema(transformed_data, schema=records_schema)
|
112
87
|
for data in normalized_data:
|
113
|
-
yield Record(data
|
88
|
+
yield Record(data, stream_slice)
|
114
89
|
|
115
90
|
def _normalize_by_schema(
|
116
91
|
self, records: Iterable[Mapping[str, Any]], schema: Optional[Mapping[str, Any]]
|
@@ -150,9 +125,6 @@ class RecordSelector(HttpSelector):
|
|
150
125
|
for record in records:
|
151
126
|
for transformation in self.transformations:
|
152
127
|
transformation.transform(
|
153
|
-
record,
|
154
|
-
|
155
|
-
stream_state=stream_state,
|
156
|
-
stream_slice=stream_slice,
|
157
|
-
)
|
128
|
+
record, config=self.config, stream_state=stream_state, stream_slice=stream_slice
|
129
|
+
) # type: ignore # record has type Mapping[str, Any], but Dict[str, Any] expected
|
158
130
|
yield record
|
@@ -10,9 +10,8 @@ from typing import Any, Dict, Iterable, Mapping, Optional, Tuple
|
|
10
10
|
|
11
11
|
import pandas as pd
|
12
12
|
import requests
|
13
|
-
from numpy import nan
|
14
|
-
|
15
13
|
from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
|
14
|
+
from numpy import nan
|
16
15
|
|
17
16
|
EMPTY_STR: str = ""
|
18
17
|
DEFAULT_ENCODING: str = "utf-8"
|
@@ -7,8 +7,6 @@ from dataclasses import InitVar, dataclass, field
|
|
7
7
|
from datetime import timedelta
|
8
8
|
from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Union
|
9
9
|
|
10
|
-
from isodate import Duration, duration_isoformat, parse_duration
|
11
|
-
|
12
10
|
from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type
|
13
11
|
from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
|
14
12
|
from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
|
@@ -21,6 +19,7 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
|
|
21
19
|
)
|
22
20
|
from airbyte_cdk.sources.message import MessageRepository
|
23
21
|
from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
|
22
|
+
from isodate import Duration, duration_isoformat, parse_duration
|
24
23
|
|
25
24
|
|
26
25
|
@dataclass
|
@@ -340,11 +340,8 @@ class GlobalSubstreamCursor(DeclarativeCursor):
|
|
340
340
|
@staticmethod
|
341
341
|
def _convert_record_to_cursor_record(record: Record) -> Record:
|
342
342
|
return Record(
|
343
|
-
|
344
|
-
|
345
|
-
associated_slice=StreamSlice(
|
346
|
-
partition={}, cursor_slice=record.associated_slice.cursor_slice
|
347
|
-
)
|
343
|
+
record.data,
|
344
|
+
StreamSlice(partition={}, cursor_slice=record.associated_slice.cursor_slice)
|
348
345
|
if record.associated_slice
|
349
346
|
else None,
|
350
347
|
)
|
@@ -325,11 +325,8 @@ class PerPartitionCursor(DeclarativeCursor):
|
|
325
325
|
@staticmethod
|
326
326
|
def _convert_record_to_cursor_record(record: Record) -> Record:
|
327
327
|
return Record(
|
328
|
-
|
329
|
-
|
330
|
-
associated_slice=StreamSlice(
|
331
|
-
partition={}, cursor_slice=record.associated_slice.cursor_slice
|
332
|
-
)
|
328
|
+
record.data,
|
329
|
+
StreamSlice(partition={}, cursor_slice=record.associated_slice.cursor_slice)
|
333
330
|
if record.associated_slice
|
334
331
|
else None,
|
335
332
|
)
|
@@ -194,7 +194,9 @@ class PerPartitionWithGlobalCursor(DeclarativeCursor):
|
|
194
194
|
)
|
195
195
|
|
196
196
|
def should_be_synced(self, record: Record) -> bool:
|
197
|
-
return self.
|
197
|
+
return self._global_cursor.should_be_synced(
|
198
|
+
record
|
199
|
+
) or self._per_partition_cursor.should_be_synced(record)
|
198
200
|
|
199
201
|
def is_greater_than_or_equal(self, first: Record, second: Record) -> bool:
|
200
202
|
return self._global_cursor.is_greater_than_or_equal(first, second)
|
@@ -6,15 +6,14 @@ import ast
|
|
6
6
|
from functools import cache
|
7
7
|
from typing import Any, Mapping, Optional, Tuple, Type
|
8
8
|
|
9
|
-
from jinja2 import meta
|
10
|
-
from jinja2.environment import Template
|
11
|
-
from jinja2.exceptions import UndefinedError
|
12
|
-
from jinja2.sandbox import SandboxedEnvironment
|
13
|
-
|
14
9
|
from airbyte_cdk.sources.declarative.interpolation.filters import filters
|
15
10
|
from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
|
16
11
|
from airbyte_cdk.sources.declarative.interpolation.macros import macros
|
17
12
|
from airbyte_cdk.sources.types import Config
|
13
|
+
from jinja2 import meta
|
14
|
+
from jinja2.environment import Template
|
15
|
+
from jinja2.exceptions import UndefinedError
|
16
|
+
from jinja2.sandbox import SandboxedEnvironment
|
18
17
|
|
19
18
|
|
20
19
|
class StreamPartitionAccessEnvironment(SandboxedEnvironment):
|
@@ -8,12 +8,9 @@ import pkgutil
|
|
8
8
|
from copy import deepcopy
|
9
9
|
from importlib import metadata
|
10
10
|
from typing import Any, Dict, Iterator, List, Mapping, Optional
|
11
|
+
from packaging.version import Version, InvalidVersion
|
11
12
|
|
12
13
|
import yaml
|
13
|
-
from jsonschema.exceptions import ValidationError
|
14
|
-
from jsonschema.validators import validate
|
15
|
-
from packaging.version import InvalidVersion, Version
|
16
|
-
|
17
14
|
from airbyte_cdk.models import (
|
18
15
|
AirbyteConnectionStatus,
|
19
16
|
AirbyteMessage,
|
@@ -47,6 +44,8 @@ from airbyte_cdk.sources.utils.slice_logger import (
|
|
47
44
|
DebugSliceLogger,
|
48
45
|
SliceLogger,
|
49
46
|
)
|
47
|
+
from jsonschema.exceptions import ValidationError
|
48
|
+
from jsonschema.validators import validate
|
50
49
|
|
51
50
|
|
52
51
|
class ManifestDeclarativeSource(DeclarativeSource):
|
@@ -5,9 +5,9 @@ from typing import Any, Mapping
|
|
5
5
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
6
6
|
from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
|
7
7
|
from airbyte_cdk.sources.declarative.models import (
|
8
|
-
CustomIncrementalSync,
|
9
8
|
DatetimeBasedCursor,
|
10
9
|
SubstreamPartitionRouter,
|
10
|
+
CustomIncrementalSync,
|
11
11
|
)
|
12
12
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import ParentStreamConfig
|
13
13
|
|
@@ -25,9 +25,6 @@ from typing import (
|
|
25
25
|
get_type_hints,
|
26
26
|
)
|
27
27
|
|
28
|
-
from isodate import parse_duration
|
29
|
-
from pydantic.v1 import BaseModel
|
30
|
-
|
31
28
|
from airbyte_cdk.models import FailureType, Level
|
32
29
|
from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
|
33
30
|
from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncJobOrchestrator
|
@@ -370,6 +367,8 @@ from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_sta
|
|
370
367
|
from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction
|
371
368
|
from airbyte_cdk.sources.types import Config
|
372
369
|
from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
|
370
|
+
from isodate import parse_duration
|
371
|
+
from pydantic.v1 import BaseModel
|
373
372
|
|
374
373
|
ComponentDefinition = Mapping[str, Any]
|
375
374
|
|
@@ -1781,7 +1780,6 @@ class ModelToComponentFactory:
|
|
1781
1780
|
self,
|
1782
1781
|
model: RecordSelectorModel,
|
1783
1782
|
config: Config,
|
1784
|
-
name: str,
|
1785
1783
|
*,
|
1786
1784
|
transformations: List[RecordTransformation],
|
1787
1785
|
decoder: Optional[Decoder] = None,
|
@@ -1812,7 +1810,6 @@ class ModelToComponentFactory:
|
|
1812
1810
|
|
1813
1811
|
return RecordSelector(
|
1814
1812
|
extractor=extractor,
|
1815
|
-
name=name,
|
1816
1813
|
config=config,
|
1817
1814
|
record_filter=record_filter,
|
1818
1815
|
transformations=transformations,
|
@@ -1883,7 +1880,6 @@ class ModelToComponentFactory:
|
|
1883
1880
|
)
|
1884
1881
|
record_selector = self._create_component_from_model(
|
1885
1882
|
model=model.record_selector,
|
1886
|
-
name=name,
|
1887
1883
|
config=config,
|
1888
1884
|
decoder=decoder,
|
1889
1885
|
transformations=transformations,
|
@@ -2038,7 +2034,6 @@ class ModelToComponentFactory:
|
|
2038
2034
|
requester=download_requester,
|
2039
2035
|
record_selector=RecordSelector(
|
2040
2036
|
extractor=ResponseToFileExtractor(),
|
2041
|
-
name=name,
|
2042
2037
|
record_filter=None,
|
2043
2038
|
transformations=[],
|
2044
2039
|
schema_normalization=TypeTransformer(TransformConfig.NoTransform),
|
@@ -7,7 +7,6 @@ from dataclasses import InitVar, dataclass
|
|
7
7
|
from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Union
|
8
8
|
|
9
9
|
import dpath
|
10
|
-
|
11
10
|
from airbyte_cdk.models import AirbyteMessage
|
12
11
|
from airbyte_cdk.models import Type as MessageType
|
13
12
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
-
|
10
9
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
11
10
|
from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
|
12
11
|
from airbyte_cdk.sources.types import Config
|
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
-
|
10
9
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
11
10
|
from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
|
12
11
|
from airbyte_cdk.sources.types import Config
|
@@ -7,7 +7,6 @@ from dataclasses import InitVar, dataclass
|
|
7
7
|
from typing import Any, Mapping, Optional, Union
|
8
8
|
|
9
9
|
import requests
|
10
|
-
|
11
10
|
from airbyte_cdk.models import FailureType
|
12
11
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
13
12
|
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
|
@@ -9,7 +9,6 @@ from dataclasses import InitVar, dataclass
|
|
9
9
|
from typing import Any, Mapping, Optional, Union
|
10
10
|
|
11
11
|
import requests
|
12
|
-
|
13
12
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
14
13
|
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
|
15
14
|
get_numeric_value_from_header,
|
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, List, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
-
|
10
9
|
from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
|
11
10
|
from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
|
12
11
|
ErrorResolution,
|
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass, field
|
|
6
6
|
from typing import Any, List, Mapping, MutableMapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
-
|
10
9
|
from airbyte_cdk.sources.declarative.requesters.error_handlers.default_http_response_filter import (
|
11
10
|
DefaultHttpResponseFilter,
|
12
11
|
)
|
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, Optional, Set, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
-
|
10
9
|
from airbyte_cdk.models import FailureType
|
11
10
|
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
12
11
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
|
@@ -6,8 +6,6 @@ from datetime import timedelta
|
|
6
6
|
from typing import Any, Dict, Iterable, Mapping, Optional
|
7
7
|
|
8
8
|
import requests
|
9
|
-
from requests import Response
|
10
|
-
|
11
9
|
from airbyte_cdk import AirbyteMessage
|
12
10
|
from airbyte_cdk.logger import lazy_log
|
13
11
|
from airbyte_cdk.models import FailureType, Type
|
@@ -25,6 +23,7 @@ from airbyte_cdk.sources.declarative.requesters.requester import Requester
|
|
25
23
|
from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
|
26
24
|
from airbyte_cdk.sources.types import Record, StreamSlice
|
27
25
|
from airbyte_cdk.utils import AirbyteTracedException
|
26
|
+
from requests import Response
|
28
27
|
|
29
28
|
LOGGER = logging.getLogger("airbyte")
|
30
29
|
|
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, MutableMapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
-
|
10
9
|
from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator
|
11
10
|
from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
|
12
11
|
|
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
-
|
10
9
|
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
11
10
|
from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
|
12
11
|
PaginationStrategy,
|
@@ -6,12 +6,10 @@ from abc import ABC, abstractmethod
|
|
6
6
|
from typing import Any, Optional
|
7
7
|
|
8
8
|
import requests
|
9
|
-
|
10
9
|
from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
|
11
10
|
from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
|
12
11
|
PaginationStrategy,
|
13
12
|
)
|
14
|
-
from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor
|
15
13
|
from airbyte_cdk.sources.types import Record
|
16
14
|
|
17
15
|
|
@@ -27,11 +25,7 @@ class PaginationStopCondition(ABC):
|
|
27
25
|
|
28
26
|
|
29
27
|
class CursorStopCondition(PaginationStopCondition):
|
30
|
-
def __init__(
|
31
|
-
self,
|
32
|
-
cursor: DeclarativeCursor
|
33
|
-
| ConcurrentCursor, # migrate to use both old and concurrent versions
|
34
|
-
):
|
28
|
+
def __init__(self, cursor: DeclarativeCursor):
|
35
29
|
self._cursor = cursor
|
36
30
|
|
37
31
|
def is_met(self, record: Record) -> bool:
|
@@ -52,8 +46,8 @@ class StopConditionPaginationStrategyDecorator(PaginationStrategy):
|
|
52
46
|
return None
|
53
47
|
return self._delegate.next_page_token(response, last_page_size, last_record)
|
54
48
|
|
55
|
-
def reset(self
|
56
|
-
self._delegate.reset(
|
49
|
+
def reset(self) -> None:
|
50
|
+
self._delegate.reset()
|
57
51
|
|
58
52
|
def get_page_size(self) -> Optional[int]:
|
59
53
|
return self._delegate.get_page_size()
|
airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py
CHANGED
@@ -5,8 +5,6 @@
|
|
5
5
|
from dataclasses import InitVar, dataclass, field
|
6
6
|
from typing import Any, Mapping, MutableMapping, Optional, Union
|
7
7
|
|
8
|
-
from deprecated import deprecated
|
9
|
-
|
10
8
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_nested_mapping import NestedMapping
|
11
9
|
from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_nested_request_input_provider import (
|
12
10
|
InterpolatedNestedRequestInputProvider,
|
@@ -19,6 +17,7 @@ from airbyte_cdk.sources.declarative.requesters.request_options.request_options_
|
|
19
17
|
)
|
20
18
|
from airbyte_cdk.sources.source import ExperimentalClassWarning
|
21
19
|
from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
|
20
|
+
from deprecated import deprecated
|
22
21
|
|
23
22
|
RequestInput = Union[str, Mapping[str, str]]
|
24
23
|
ValidRequestTypes = (str, list)
|
@@ -7,7 +7,6 @@ from enum import Enum
|
|
7
7
|
from typing import Any, Callable, Mapping, MutableMapping, Optional, Union
|
8
8
|
|
9
9
|
import requests
|
10
|
-
|
11
10
|
from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
|
12
11
|
from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
|
13
12
|
RequestOptionsProvider,
|
@@ -4,8 +4,6 @@
|
|
4
4
|
from dataclasses import InitVar, dataclass, field
|
5
5
|
from typing import Any, Callable, Iterable, Mapping, Optional
|
6
6
|
|
7
|
-
from deprecated.classic import deprecated
|
8
|
-
|
9
7
|
from airbyte_cdk.models import FailureType
|
10
8
|
from airbyte_cdk.sources.declarative.async_job.job_orchestrator import (
|
11
9
|
AsyncJobOrchestrator,
|
@@ -19,6 +17,7 @@ from airbyte_cdk.sources.source import ExperimentalClassWarning
|
|
19
17
|
from airbyte_cdk.sources.streams.core import StreamData
|
20
18
|
from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
|
21
19
|
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
20
|
+
from deprecated.classic import deprecated
|
22
21
|
|
23
22
|
|
24
23
|
@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
|
@@ -20,7 +20,6 @@ from typing import (
|
|
20
20
|
)
|
21
21
|
|
22
22
|
import requests
|
23
|
-
|
24
23
|
from airbyte_cdk.models import AirbyteMessage
|
25
24
|
from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
|
26
25
|
from airbyte_cdk.sources.declarative.incremental import ResumableFullRefreshCursor
|
@@ -468,9 +467,8 @@ class SimpleRetriever(Retriever):
|
|
468
467
|
else:
|
469
468
|
return None
|
470
469
|
|
471
|
-
|
472
|
-
|
473
|
-
) -> Optional[Record]:
|
470
|
+
@staticmethod
|
471
|
+
def _extract_record(stream_data: StreamData, stream_slice: StreamSlice) -> Optional[Record]:
|
474
472
|
"""
|
475
473
|
As we allow the output of _read_pages to be StreamData, it can be multiple things. Therefore, we need to filter out and normalize
|
476
474
|
to data to streamline the rest of the process.
|
@@ -479,15 +477,9 @@ class SimpleRetriever(Retriever):
|
|
479
477
|
# Record is not part of `StreamData` but is the most common implementation of `Mapping[str, Any]` which is part of `StreamData`
|
480
478
|
return stream_data
|
481
479
|
elif isinstance(stream_data, (dict, Mapping)):
|
482
|
-
return Record(
|
483
|
-
data=dict(stream_data), associated_slice=stream_slice, stream_name=self.name
|
484
|
-
)
|
480
|
+
return Record(dict(stream_data), stream_slice)
|
485
481
|
elif isinstance(stream_data, AirbyteMessage) and stream_data.record:
|
486
|
-
return Record(
|
487
|
-
data=stream_data.record.data, # type:ignore # AirbyteMessage always has record.data
|
488
|
-
associated_slice=stream_slice,
|
489
|
-
stream_name=self.name,
|
490
|
-
)
|
482
|
+
return Record(stream_data.record.data, stream_slice)
|
491
483
|
return None
|
492
484
|
|
493
485
|
# stream_slices is defined with arguments on http stream and fixing this has a long tail of dependencies. Will be resolved by the decoupling of http stream and simple retriever
|
@@ -1,13 +1,14 @@
|
|
1
1
|
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
2
|
|
3
|
-
from typing import
|
3
|
+
from typing import Iterable, Optional, Mapping, Any, Callable
|
4
4
|
|
5
5
|
from airbyte_cdk.sources.declarative.retrievers import Retriever
|
6
6
|
from airbyte_cdk.sources.message import MessageRepository
|
7
7
|
from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
|
8
8
|
from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator
|
9
|
+
from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
|
9
10
|
from airbyte_cdk.sources.streams.concurrent.partitions.stream_slicer import StreamSlicer
|
10
|
-
from airbyte_cdk.sources.types import
|
11
|
+
from airbyte_cdk.sources.types import StreamSlice
|
11
12
|
from airbyte_cdk.utils.slice_hasher import SliceHasher
|
12
13
|
|
13
14
|
|
@@ -58,11 +59,7 @@ class DeclarativePartition(Partition):
|
|
58
59
|
def read(self) -> Iterable[Record]:
|
59
60
|
for stream_data in self._retriever.read_records(self._json_schema, self._stream_slice):
|
60
61
|
if isinstance(stream_data, Mapping):
|
61
|
-
yield Record(
|
62
|
-
data=stream_data,
|
63
|
-
stream_name=self.stream_name(),
|
64
|
-
associated_slice=self._stream_slice,
|
65
|
-
)
|
62
|
+
yield Record(stream_data, self)
|
66
63
|
else:
|
67
64
|
self._message_repository.emit_message(stream_data)
|
68
65
|
|
@@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass, field
|
|
6
6
|
from typing import Any, Dict, List, Mapping, Optional, Type, Union
|
7
7
|
|
8
8
|
import dpath
|
9
|
-
|
10
9
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
11
10
|
from airbyte_cdk.sources.declarative.transformations import RecordTransformation
|
12
11
|
from airbyte_cdk.sources.types import Config, FieldPointer, StreamSlice, StreamState
|
@@ -7,7 +7,6 @@ from typing import Any, Dict, List, Mapping, Optional
|
|
7
7
|
|
8
8
|
import dpath
|
9
9
|
import dpath.exceptions
|
10
|
-
|
11
10
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
|
12
11
|
from airbyte_cdk.sources.declarative.transformations import RecordTransformation
|
13
12
|
from airbyte_cdk.sources.types import Config, FieldPointer, StreamSlice, StreamState
|
@@ -6,7 +6,6 @@ import pkgutil
|
|
6
6
|
from typing import Any, List, Mapping, Optional
|
7
7
|
|
8
8
|
import yaml
|
9
|
-
|
10
9
|
from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog
|
11
10
|
from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
|
12
11
|
ConcurrentDeclarativeSource,
|
@@ -7,11 +7,10 @@ from abc import abstractmethod
|
|
7
7
|
from typing import Any, Dict, List, Literal, Optional, Union
|
8
8
|
|
9
9
|
import dpath
|
10
|
-
from pydantic.v1 import AnyUrl, BaseModel, Field
|
11
|
-
|
12
10
|
from airbyte_cdk import OneOfOptionConfig
|
13
11
|
from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
|
14
12
|
from airbyte_cdk.sources.utils import schema_helpers
|
13
|
+
from pydantic.v1 import AnyUrl, BaseModel, Field
|
15
14
|
|
16
15
|
|
17
16
|
class DeliverRecords(BaseModel):
|
@@ -6,11 +6,10 @@ import codecs
|
|
6
6
|
from enum import Enum
|
7
7
|
from typing import Any, Dict, List, Optional, Set, Union
|
8
8
|
|
9
|
+
from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
|
9
10
|
from pydantic.v1 import BaseModel, Field, root_validator, validator
|
10
11
|
from pydantic.v1.error_wrappers import ValidationError
|
11
12
|
|
12
|
-
from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig
|
13
|
-
|
14
13
|
|
15
14
|
class InferenceType(Enum):
|
16
15
|
NONE = "None"
|