airbyte-cdk 6.7.0rc2__py3-none-any.whl → 6.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/cli/__init__.py +1 -0
- airbyte_cdk/cli/source_declarative_manifest/__init__.py +6 -0
- airbyte_cdk/cli/source_declarative_manifest/_run.py +224 -0
- airbyte_cdk/cli/source_declarative_manifest/spec.json +17 -0
- airbyte_cdk/config_observation.py +2 -1
- airbyte_cdk/connector.py +1 -0
- airbyte_cdk/connector_builder/connector_builder_handler.py +1 -1
- airbyte_cdk/connector_builder/main.py +2 -1
- airbyte_cdk/destinations/destination.py +2 -1
- airbyte_cdk/destinations/vector_db_based/config.py +2 -1
- airbyte_cdk/destinations/vector_db_based/document_processor.py +4 -3
- airbyte_cdk/destinations/vector_db_based/embedder.py +5 -4
- airbyte_cdk/entrypoint.py +3 -2
- airbyte_cdk/logger.py +2 -1
- airbyte_cdk/models/__init__.py +2 -0
- airbyte_cdk/models/airbyte_protocol.py +2 -1
- airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +3 -3
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +1 -1
- airbyte_cdk/sources/config.py +2 -1
- airbyte_cdk/sources/declarative/auth/jwt.py +1 -0
- airbyte_cdk/sources/declarative/auth/oauth.py +1 -0
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +1 -0
- airbyte_cdk/sources/declarative/auth/token.py +2 -1
- airbyte_cdk/sources/declarative/auth/token_provider.py +3 -2
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +24 -54
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +180 -13
- airbyte_cdk/sources/declarative/decoders/json_decoder.py +3 -2
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +1 -0
- airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py +1 -0
- airbyte_cdk/sources/declarative/decoders/xml_decoder.py +1 -0
- airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +1 -0
- airbyte_cdk/sources/declarative/extractors/http_selector.py +1 -0
- airbyte_cdk/sources/declarative/extractors/record_filter.py +6 -48
- airbyte_cdk/sources/declarative/extractors/record_selector.py +32 -4
- airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py +2 -1
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +2 -1
- airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py +5 -2
- airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +5 -2
- airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py +1 -3
- airbyte_cdk/sources/declarative/interpolation/jinja.py +5 -4
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +4 -3
- airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py +1 -1
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +122 -0
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +26 -17
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/constant_backoff_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/exponential_backoff_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_until_time_from_header_backoff_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py +1 -0
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +1 -0
- airbyte_cdk/sources/declarative/requesters/http_job_repository.py +2 -1
- airbyte_cdk/sources/declarative/requesters/http_requester.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +1 -0
- airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +9 -3
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +2 -1
- airbyte_cdk/sources/declarative/requesters/requester.py +1 -0
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +2 -1
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +12 -4
- airbyte_cdk/sources/declarative/stream_slicers/declarative_partition_generator.py +7 -4
- airbyte_cdk/sources/declarative/transformations/add_fields.py +1 -0
- airbyte_cdk/sources/declarative/transformations/remove_fields.py +1 -0
- airbyte_cdk/sources/declarative/yaml_declarative_source.py +1 -0
- airbyte_cdk/sources/embedded/tools.py +1 -0
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +2 -1
- airbyte_cdk/sources/file_based/config/avro_format.py +2 -1
- airbyte_cdk/sources/file_based/config/csv_format.py +2 -1
- airbyte_cdk/sources/file_based/config/excel_format.py +2 -1
- airbyte_cdk/sources/file_based/config/file_based_stream_config.py +2 -1
- airbyte_cdk/sources/file_based/config/jsonl_format.py +2 -1
- airbyte_cdk/sources/file_based/config/parquet_format.py +2 -1
- airbyte_cdk/sources/file_based/config/unstructured_format.py +2 -1
- airbyte_cdk/sources/file_based/file_based_source.py +2 -1
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +2 -1
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +1 -0
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +2 -1
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +5 -5
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +2 -1
- airbyte_cdk/sources/file_based/file_types/parquet_parser.py +2 -1
- airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +9 -8
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +2 -1
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +5 -4
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py +1 -1
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py +1 -1
- airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py +1 -1
- airbyte_cdk/sources/http_logger.py +1 -0
- airbyte_cdk/sources/streams/call_rate.py +1 -2
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +2 -1
- airbyte_cdk/sources/streams/concurrent/adapters.py +8 -4
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +2 -1
- airbyte_cdk/sources/streams/concurrent/cursor.py +30 -6
- airbyte_cdk/sources/streams/concurrent/partitions/partition.py +1 -1
- airbyte_cdk/sources/streams/concurrent/partitions/types.py +1 -1
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +1 -1
- airbyte_cdk/sources/streams/core.py +2 -1
- airbyte_cdk/sources/streams/http/error_handlers/default_error_mapping.py +2 -1
- airbyte_cdk/sources/streams/http/error_handlers/http_status_error_handler.py +1 -0
- airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py +1 -0
- airbyte_cdk/sources/streams/http/error_handlers/response_models.py +2 -1
- airbyte_cdk/sources/streams/http/http.py +3 -2
- airbyte_cdk/sources/streams/http/http_client.py +49 -2
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +2 -1
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +1 -0
- airbyte_cdk/sources/types.py +14 -1
- airbyte_cdk/sources/utils/schema_helpers.py +3 -2
- airbyte_cdk/sql/secrets.py +2 -1
- airbyte_cdk/sql/shared/sql_processor.py +8 -6
- airbyte_cdk/test/entrypoint_wrapper.py +4 -3
- airbyte_cdk/test/mock_http/mocker.py +1 -0
- airbyte_cdk/utils/schema_inferrer.py +2 -1
- airbyte_cdk/utils/slice_hasher.py +1 -1
- airbyte_cdk/utils/traced_exception.py +2 -1
- {airbyte_cdk-6.7.0rc2.dist-info → airbyte_cdk-6.7.1.dist-info}/METADATA +11 -3
- {airbyte_cdk-6.7.0rc2.dist-info → airbyte_cdk-6.7.1.dist-info}/RECORD +125 -121
- airbyte_cdk-6.7.1.dist-info/entry_points.txt +3 -0
- airbyte_cdk/sources/streams/concurrent/partitions/record.py +0 -35
- {airbyte_cdk-6.7.0rc2.dist-info → airbyte_cdk-6.7.1.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.7.0rc2.dist-info → airbyte_cdk-6.7.1.dist-info}/WHEEL +0 -0
@@ -7,6 +7,8 @@ from dataclasses import InitVar, dataclass, field
|
|
7
7
|
from datetime import timedelta
|
8
8
|
from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Union
|
9
9
|
|
10
|
+
from isodate import Duration, duration_isoformat, parse_duration
|
11
|
+
|
10
12
|
from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type
|
11
13
|
from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
|
12
14
|
from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime
|
@@ -19,7 +21,6 @@ from airbyte_cdk.sources.declarative.requesters.request_option import (
|
|
19
21
|
)
|
20
22
|
from airbyte_cdk.sources.message import MessageRepository
|
21
23
|
from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
|
22
|
-
from isodate import Duration, duration_isoformat, parse_duration
|
23
24
|
|
24
25
|
|
25
26
|
@dataclass
|
@@ -340,8 +340,11 @@ class GlobalSubstreamCursor(DeclarativeCursor):
|
|
340
340
|
@staticmethod
|
341
341
|
def _convert_record_to_cursor_record(record: Record) -> Record:
|
342
342
|
return Record(
|
343
|
-
record.data,
|
344
|
-
|
343
|
+
data=record.data,
|
344
|
+
stream_name=record.stream_name,
|
345
|
+
associated_slice=StreamSlice(
|
346
|
+
partition={}, cursor_slice=record.associated_slice.cursor_slice
|
347
|
+
)
|
345
348
|
if record.associated_slice
|
346
349
|
else None,
|
347
350
|
)
|
@@ -325,8 +325,11 @@ class PerPartitionCursor(DeclarativeCursor):
|
|
325
325
|
@staticmethod
|
326
326
|
def _convert_record_to_cursor_record(record: Record) -> Record:
|
327
327
|
return Record(
|
328
|
-
record.data,
|
329
|
-
|
328
|
+
data=record.data,
|
329
|
+
stream_name=record.stream_name,
|
330
|
+
associated_slice=StreamSlice(
|
331
|
+
partition={}, cursor_slice=record.associated_slice.cursor_slice
|
332
|
+
)
|
330
333
|
if record.associated_slice
|
331
334
|
else None,
|
332
335
|
)
|
@@ -194,9 +194,7 @@ class PerPartitionWithGlobalCursor(DeclarativeCursor):
|
|
194
194
|
)
|
195
195
|
|
196
196
|
def should_be_synced(self, record: Record) -> bool:
|
197
|
-
return self.
|
198
|
-
record
|
199
|
-
) or self._per_partition_cursor.should_be_synced(record)
|
197
|
+
return self._get_active_cursor().should_be_synced(record)
|
200
198
|
|
201
199
|
def is_greater_than_or_equal(self, first: Record, second: Record) -> bool:
|
202
200
|
return self._global_cursor.is_greater_than_or_equal(first, second)
|
@@ -6,15 +6,16 @@ import ast
|
|
6
6
|
from functools import cache
|
7
7
|
from typing import Any, Mapping, Optional, Tuple, Type
|
8
8
|
|
9
|
-
from airbyte_cdk.sources.declarative.interpolation.filters import filters
|
10
|
-
from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
|
11
|
-
from airbyte_cdk.sources.declarative.interpolation.macros import macros
|
12
|
-
from airbyte_cdk.sources.types import Config
|
13
9
|
from jinja2 import meta
|
14
10
|
from jinja2.environment import Template
|
15
11
|
from jinja2.exceptions import UndefinedError
|
16
12
|
from jinja2.sandbox import SandboxedEnvironment
|
17
13
|
|
14
|
+
from airbyte_cdk.sources.declarative.interpolation.filters import filters
|
15
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
|
16
|
+
from airbyte_cdk.sources.declarative.interpolation.macros import macros
|
17
|
+
from airbyte_cdk.sources.types import Config
|
18
|
+
|
18
19
|
|
19
20
|
class StreamPartitionAccessEnvironment(SandboxedEnvironment):
|
20
21
|
"""
|
@@ -8,9 +8,12 @@ import pkgutil
|
|
8
8
|
from copy import deepcopy
|
9
9
|
from importlib import metadata
|
10
10
|
from typing import Any, Dict, Iterator, List, Mapping, Optional
|
11
|
-
from packaging.version import Version, InvalidVersion
|
12
11
|
|
13
12
|
import yaml
|
13
|
+
from jsonschema.exceptions import ValidationError
|
14
|
+
from jsonschema.validators import validate
|
15
|
+
from packaging.version import InvalidVersion, Version
|
16
|
+
|
14
17
|
from airbyte_cdk.models import (
|
15
18
|
AirbyteConnectionStatus,
|
16
19
|
AirbyteMessage,
|
@@ -44,8 +47,6 @@ from airbyte_cdk.sources.utils.slice_logger import (
|
|
44
47
|
DebugSliceLogger,
|
45
48
|
SliceLogger,
|
46
49
|
)
|
47
|
-
from jsonschema.exceptions import ValidationError
|
48
|
-
from jsonschema.validators import validate
|
49
50
|
|
50
51
|
|
51
52
|
class ManifestDeclarativeSource(DeclarativeSource):
|
@@ -5,9 +5,9 @@ from typing import Any, Mapping
|
|
5
5
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
6
6
|
from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
|
7
7
|
from airbyte_cdk.sources.declarative.models import (
|
8
|
+
CustomIncrementalSync,
|
8
9
|
DatetimeBasedCursor,
|
9
10
|
SubstreamPartitionRouter,
|
10
|
-
CustomIncrementalSync,
|
11
11
|
)
|
12
12
|
from airbyte_cdk.sources.declarative.models.declarative_component_schema import ParentStreamConfig
|
13
13
|
|
@@ -748,6 +748,123 @@ class NoPagination(BaseModel):
|
|
748
748
|
type: Literal["NoPagination"]
|
749
749
|
|
750
750
|
|
751
|
+
class State(BaseModel):
|
752
|
+
class Config:
|
753
|
+
extra = Extra.allow
|
754
|
+
|
755
|
+
min: int
|
756
|
+
max: int
|
757
|
+
|
758
|
+
|
759
|
+
class OauthConnectorInputSpecification(BaseModel):
|
760
|
+
class Config:
|
761
|
+
extra = Extra.allow
|
762
|
+
|
763
|
+
consent_url: str = Field(
|
764
|
+
...,
|
765
|
+
description="The DeclarativeOAuth Specific string URL string template to initiate the authentication.\nThe placeholders are replaced during the processing to provide neccessary values.",
|
766
|
+
examples=[
|
767
|
+
{
|
768
|
+
"consent_url": "https://domain.host.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}"
|
769
|
+
},
|
770
|
+
{
|
771
|
+
"consent_url": "https://endpoint.host.com/oauth2/authorize?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{scope_key}={urlEncoder:{{scope_key}}}&{state_key}={{state_key}}&subdomain={subdomain}"
|
772
|
+
},
|
773
|
+
],
|
774
|
+
title="DeclarativeOAuth Consent URL",
|
775
|
+
)
|
776
|
+
scope: Optional[str] = Field(
|
777
|
+
None,
|
778
|
+
description="The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user.",
|
779
|
+
examples=[{"scope": "user:read user:read_orders workspaces:read"}],
|
780
|
+
title="(Optional) DeclarativeOAuth Scope",
|
781
|
+
)
|
782
|
+
access_token_url: str = Field(
|
783
|
+
...,
|
784
|
+
description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.",
|
785
|
+
examples=[
|
786
|
+
{
|
787
|
+
"access_token_url": "https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}"
|
788
|
+
}
|
789
|
+
],
|
790
|
+
title="DeclarativeOAuth Access Token URL",
|
791
|
+
)
|
792
|
+
access_token_headers: Optional[Dict[str, Any]] = Field(
|
793
|
+
None,
|
794
|
+
description="The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.",
|
795
|
+
examples=[
|
796
|
+
{
|
797
|
+
"access_token_headers": {
|
798
|
+
"Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"
|
799
|
+
}
|
800
|
+
}
|
801
|
+
],
|
802
|
+
title="(Optional) DeclarativeOAuth Access Token Headers",
|
803
|
+
)
|
804
|
+
access_token_params: Optional[Dict[str, Any]] = Field(
|
805
|
+
None,
|
806
|
+
description="The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.\nWhen this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.",
|
807
|
+
examples=[
|
808
|
+
{
|
809
|
+
"access_token_params": {
|
810
|
+
"{auth_code_key}": "{{auth_code_key}}",
|
811
|
+
"{client_id_key}": "{{client_id_key}}",
|
812
|
+
"{client_secret_key}": "{{client_secret_key}}",
|
813
|
+
}
|
814
|
+
}
|
815
|
+
],
|
816
|
+
title="(Optional) DeclarativeOAuth Access Token Query Params (Json Encoded)",
|
817
|
+
)
|
818
|
+
extract_output: List[str] = Field(
|
819
|
+
...,
|
820
|
+
description="The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config. ",
|
821
|
+
examples=[{"extract_output": ["access_token", "refresh_token", "other_field"]}],
|
822
|
+
title="DeclarativeOAuth Extract Output",
|
823
|
+
)
|
824
|
+
state: Optional[State] = Field(
|
825
|
+
None,
|
826
|
+
description="The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,\nincluding length and complexity. ",
|
827
|
+
examples=[{"state": {"min": 7, "max": 128}}],
|
828
|
+
title="(Optional) DeclarativeOAuth Configurable State Query Param",
|
829
|
+
)
|
830
|
+
client_id_key: Optional[str] = Field(
|
831
|
+
None,
|
832
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `client_id` key name, if required by data-provider.",
|
833
|
+
examples=[{"client_id_key": "my_custom_client_id_key_name"}],
|
834
|
+
title="(Optional) DeclarativeOAuth Client ID Key Override",
|
835
|
+
)
|
836
|
+
client_secret_key: Optional[str] = Field(
|
837
|
+
None,
|
838
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `client_secret` key name, if required by data-provider.",
|
839
|
+
examples=[{"client_secret_key": "my_custom_client_secret_key_name"}],
|
840
|
+
title="(Optional) DeclarativeOAuth Client Secret Key Override",
|
841
|
+
)
|
842
|
+
scope_key: Optional[str] = Field(
|
843
|
+
None,
|
844
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `scope` key name, if required by data-provider.",
|
845
|
+
examples=[{"scope_key": "my_custom_scope_key_key_name"}],
|
846
|
+
title="(Optional) DeclarativeOAuth Scope Key Override",
|
847
|
+
)
|
848
|
+
state_key: Optional[str] = Field(
|
849
|
+
None,
|
850
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider. ",
|
851
|
+
examples=[{"state_key": "my_custom_state_key_key_name"}],
|
852
|
+
title="(Optional) DeclarativeOAuth State Key Override",
|
853
|
+
)
|
854
|
+
auth_code_key: Optional[str] = Field(
|
855
|
+
None,
|
856
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider. ",
|
857
|
+
examples=[{"auth_code_key": "my_custom_auth_code_key_name"}],
|
858
|
+
title="(Optional) DeclarativeOAuth Auth Code Key Override",
|
859
|
+
)
|
860
|
+
redirect_uri_key: Optional[str] = Field(
|
861
|
+
None,
|
862
|
+
description="The DeclarativeOAuth Specific optional override to provide the custom `redirect_uri` key name to something like `callback_uri`, if required by data-provider.",
|
863
|
+
examples=[{"redirect_uri_key": "my_custom_redirect_uri_key_name"}],
|
864
|
+
title="(Optional) DeclarativeOAuth Redirect URI Key Override",
|
865
|
+
)
|
866
|
+
|
867
|
+
|
751
868
|
class OAuthConfigSpecification(BaseModel):
|
752
869
|
class Config:
|
753
870
|
extra = Extra.allow
|
@@ -766,6 +883,11 @@ class OAuthConfigSpecification(BaseModel):
|
|
766
883
|
],
|
767
884
|
title="OAuth user input",
|
768
885
|
)
|
886
|
+
oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field(
|
887
|
+
None,
|
888
|
+
description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{my_var}`.\n- The nested resolution variables like `{{my_nested_var}}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {base64Encoder:{my_var_a}:{my_var_b}}\n + base64Decorer - decode from `base64` encoded string, {base64Decoder:{my_string_variable_or_string_value}}\n + urlEncoder - encode the input string to URL-like format, {urlEncoder:https://test.host.com/endpoint}\n + urlDecorer - decode the input url-encoded string into text format, {urlDecoder:https%3A%2F%2Fairbyte.io}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {codeChallengeS256:{state_value}}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{auth_code_key}": "{{auth_code_key}}",\n "{client_id_key}": "{{client_id_key}}",\n "{client_secret_key}": "{{client_secret_key}}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }',
|
889
|
+
title="DeclarativeOAuth Connector Specification",
|
890
|
+
)
|
769
891
|
complete_oauth_output_specification: Optional[Dict[str, Any]] = Field(
|
770
892
|
None,
|
771
893
|
description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }",
|
@@ -17,6 +17,7 @@ from typing import (
|
|
17
17
|
Mapping,
|
18
18
|
MutableMapping,
|
19
19
|
Optional,
|
20
|
+
Tuple,
|
20
21
|
Type,
|
21
22
|
Union,
|
22
23
|
get_args,
|
@@ -24,6 +25,9 @@ from typing import (
|
|
24
25
|
get_type_hints,
|
25
26
|
)
|
26
27
|
|
28
|
+
from isodate import parse_duration
|
29
|
+
from pydantic.v1 import BaseModel
|
30
|
+
|
27
31
|
from airbyte_cdk.models import FailureType, Level
|
28
32
|
from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
|
29
33
|
from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncJobOrchestrator
|
@@ -366,8 +370,6 @@ from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_sta
|
|
366
370
|
from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction
|
367
371
|
from airbyte_cdk.sources.types import Config
|
368
372
|
from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
|
369
|
-
from isodate import parse_duration
|
370
|
-
from pydantic.v1 import BaseModel
|
371
373
|
|
372
374
|
ComponentDefinition = Mapping[str, Any]
|
373
375
|
|
@@ -752,7 +754,7 @@ class ModelToComponentFactory:
|
|
752
754
|
config: Config,
|
753
755
|
stream_state: MutableMapping[str, Any],
|
754
756
|
**kwargs: Any,
|
755
|
-
) -> ConcurrentCursor:
|
757
|
+
) -> Tuple[ConcurrentCursor, DateTimeStreamStateConverter]:
|
756
758
|
component_type = component_definition.get("type")
|
757
759
|
if component_definition.get("type") != model_type.__name__:
|
758
760
|
raise ValueError(
|
@@ -883,20 +885,23 @@ class ModelToComponentFactory:
|
|
883
885
|
if evaluated_step:
|
884
886
|
step_length = parse_duration(evaluated_step)
|
885
887
|
|
886
|
-
return
|
887
|
-
|
888
|
-
|
889
|
-
|
890
|
-
|
891
|
-
|
892
|
-
|
893
|
-
|
894
|
-
|
895
|
-
|
896
|
-
|
897
|
-
|
898
|
-
|
899
|
-
|
888
|
+
return (
|
889
|
+
ConcurrentCursor(
|
890
|
+
stream_name=stream_name,
|
891
|
+
stream_namespace=stream_namespace,
|
892
|
+
stream_state=stream_state,
|
893
|
+
message_repository=self._message_repository, # type: ignore # message_repository is always instantiated with a value by factory
|
894
|
+
connector_state_manager=state_manager,
|
895
|
+
connector_state_converter=connector_state_converter,
|
896
|
+
cursor_field=cursor_field,
|
897
|
+
slice_boundary_fields=slice_boundary_fields,
|
898
|
+
start=start_date, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice
|
899
|
+
end_provider=end_date_provider, # type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice
|
900
|
+
lookback_window=lookback_window,
|
901
|
+
slice_range=step_length,
|
902
|
+
cursor_granularity=cursor_granularity,
|
903
|
+
),
|
904
|
+
connector_state_converter,
|
900
905
|
)
|
901
906
|
|
902
907
|
@staticmethod
|
@@ -1776,6 +1781,7 @@ class ModelToComponentFactory:
|
|
1776
1781
|
self,
|
1777
1782
|
model: RecordSelectorModel,
|
1778
1783
|
config: Config,
|
1784
|
+
name: str,
|
1779
1785
|
*,
|
1780
1786
|
transformations: List[RecordTransformation],
|
1781
1787
|
decoder: Optional[Decoder] = None,
|
@@ -1806,6 +1812,7 @@ class ModelToComponentFactory:
|
|
1806
1812
|
|
1807
1813
|
return RecordSelector(
|
1808
1814
|
extractor=extractor,
|
1815
|
+
name=name,
|
1809
1816
|
config=config,
|
1810
1817
|
record_filter=record_filter,
|
1811
1818
|
transformations=transformations,
|
@@ -1876,6 +1883,7 @@ class ModelToComponentFactory:
|
|
1876
1883
|
)
|
1877
1884
|
record_selector = self._create_component_from_model(
|
1878
1885
|
model=model.record_selector,
|
1886
|
+
name=name,
|
1879
1887
|
config=config,
|
1880
1888
|
decoder=decoder,
|
1881
1889
|
transformations=transformations,
|
@@ -2030,6 +2038,7 @@ class ModelToComponentFactory:
|
|
2030
2038
|
requester=download_requester,
|
2031
2039
|
record_selector=RecordSelector(
|
2032
2040
|
extractor=ResponseToFileExtractor(),
|
2041
|
+
name=name,
|
2033
2042
|
record_filter=None,
|
2034
2043
|
transformations=[],
|
2035
2044
|
schema_normalization=TypeTransformer(TransformConfig.NoTransform),
|
@@ -7,6 +7,7 @@ from dataclasses import InitVar, dataclass
|
|
7
7
|
from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, Optional, Union
|
8
8
|
|
9
9
|
import dpath
|
10
|
+
|
10
11
|
from airbyte_cdk.models import AirbyteMessage
|
11
12
|
from airbyte_cdk.models import Type as MessageType
|
12
13
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
10
11
|
from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
|
11
12
|
from airbyte_cdk.sources.types import Config
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
10
11
|
from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy
|
11
12
|
from airbyte_cdk.sources.types import Config
|
@@ -7,6 +7,7 @@ from dataclasses import InitVar, dataclass
|
|
7
7
|
from typing import Any, Mapping, Optional, Union
|
8
8
|
|
9
9
|
import requests
|
10
|
+
|
10
11
|
from airbyte_cdk.models import FailureType
|
11
12
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
12
13
|
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
|
@@ -9,6 +9,7 @@ from dataclasses import InitVar, dataclass
|
|
9
9
|
from typing import Any, Mapping, Optional, Union
|
10
10
|
|
11
11
|
import requests
|
12
|
+
|
12
13
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
13
14
|
from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import (
|
14
15
|
get_numeric_value_from_header,
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, List, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
|
10
11
|
from airbyte_cdk.sources.streams.http.error_handlers.response_models import (
|
11
12
|
ErrorResolution,
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass, field
|
|
6
6
|
from typing import Any, List, Mapping, MutableMapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.declarative.requesters.error_handlers.default_http_response_filter import (
|
10
11
|
DefaultHttpResponseFilter,
|
11
12
|
)
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, Optional, Set, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.models import FailureType
|
10
11
|
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
11
12
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
|
@@ -6,6 +6,8 @@ from datetime import timedelta
|
|
6
6
|
from typing import Any, Dict, Iterable, Mapping, Optional
|
7
7
|
|
8
8
|
import requests
|
9
|
+
from requests import Response
|
10
|
+
|
9
11
|
from airbyte_cdk import AirbyteMessage
|
10
12
|
from airbyte_cdk.logger import lazy_log
|
11
13
|
from airbyte_cdk.models import FailureType, Type
|
@@ -23,7 +25,6 @@ from airbyte_cdk.sources.declarative.requesters.requester import Requester
|
|
23
25
|
from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
|
24
26
|
from airbyte_cdk.sources.types import Record, StreamSlice
|
25
27
|
from airbyte_cdk.utils import AirbyteTracedException
|
26
|
-
from requests import Response
|
27
28
|
|
28
29
|
LOGGER = logging.getLogger("airbyte")
|
29
30
|
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, MutableMapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator
|
10
11
|
from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
|
11
12
|
|
@@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass
|
|
6
6
|
from typing import Any, Mapping, Optional, Union
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
10
11
|
from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
|
11
12
|
PaginationStrategy,
|
@@ -6,10 +6,12 @@ from abc import ABC, abstractmethod
|
|
6
6
|
from typing import Any, Optional
|
7
7
|
|
8
8
|
import requests
|
9
|
+
|
9
10
|
from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
|
10
11
|
from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
|
11
12
|
PaginationStrategy,
|
12
13
|
)
|
14
|
+
from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor
|
13
15
|
from airbyte_cdk.sources.types import Record
|
14
16
|
|
15
17
|
|
@@ -25,7 +27,11 @@ class PaginationStopCondition(ABC):
|
|
25
27
|
|
26
28
|
|
27
29
|
class CursorStopCondition(PaginationStopCondition):
|
28
|
-
def __init__(
|
30
|
+
def __init__(
|
31
|
+
self,
|
32
|
+
cursor: DeclarativeCursor
|
33
|
+
| ConcurrentCursor, # migrate to use both old and concurrent versions
|
34
|
+
):
|
29
35
|
self._cursor = cursor
|
30
36
|
|
31
37
|
def is_met(self, record: Record) -> bool:
|
@@ -46,8 +52,8 @@ class StopConditionPaginationStrategyDecorator(PaginationStrategy):
|
|
46
52
|
return None
|
47
53
|
return self._delegate.next_page_token(response, last_page_size, last_record)
|
48
54
|
|
49
|
-
def reset(self) -> None:
|
50
|
-
self._delegate.reset()
|
55
|
+
def reset(self, reset_value: Optional[Any] = None) -> None:
|
56
|
+
self._delegate.reset(reset_value)
|
51
57
|
|
52
58
|
def get_page_size(self) -> Optional[int]:
|
53
59
|
return self._delegate.get_page_size()
|
airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py
CHANGED
@@ -5,6 +5,8 @@
|
|
5
5
|
from dataclasses import InitVar, dataclass, field
|
6
6
|
from typing import Any, Mapping, MutableMapping, Optional, Union
|
7
7
|
|
8
|
+
from deprecated import deprecated
|
9
|
+
|
8
10
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_nested_mapping import NestedMapping
|
9
11
|
from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_nested_request_input_provider import (
|
10
12
|
InterpolatedNestedRequestInputProvider,
|
@@ -17,7 +19,6 @@ from airbyte_cdk.sources.declarative.requesters.request_options.request_options_
|
|
17
19
|
)
|
18
20
|
from airbyte_cdk.sources.source import ExperimentalClassWarning
|
19
21
|
from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
|
20
|
-
from deprecated import deprecated
|
21
22
|
|
22
23
|
RequestInput = Union[str, Mapping[str, str]]
|
23
24
|
ValidRequestTypes = (str, list)
|
@@ -7,6 +7,7 @@ from enum import Enum
|
|
7
7
|
from typing import Any, Callable, Mapping, MutableMapping, Optional, Union
|
8
8
|
|
9
9
|
import requests
|
10
|
+
|
10
11
|
from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
|
11
12
|
from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import (
|
12
13
|
RequestOptionsProvider,
|
@@ -4,6 +4,8 @@
|
|
4
4
|
from dataclasses import InitVar, dataclass, field
|
5
5
|
from typing import Any, Callable, Iterable, Mapping, Optional
|
6
6
|
|
7
|
+
from deprecated.classic import deprecated
|
8
|
+
|
7
9
|
from airbyte_cdk.models import FailureType
|
8
10
|
from airbyte_cdk.sources.declarative.async_job.job_orchestrator import (
|
9
11
|
AsyncJobOrchestrator,
|
@@ -17,7 +19,6 @@ from airbyte_cdk.sources.source import ExperimentalClassWarning
|
|
17
19
|
from airbyte_cdk.sources.streams.core import StreamData
|
18
20
|
from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
|
19
21
|
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
20
|
-
from deprecated.classic import deprecated
|
21
22
|
|
22
23
|
|
23
24
|
@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
|