airbyte-cdk 6.9.1__py3-none-any.whl → 6.9.1.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +35 -30
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +3 -101
- airbyte_cdk/sources/declarative/interpolation/jinja.py +35 -36
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +2 -53
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +2 -95
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +0 -6
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +21 -95
- airbyte_cdk/sources/declarative/partition_routers/__init__.py +1 -2
- airbyte_cdk/sources/streams/http/http_client.py +5 -15
- airbyte_cdk/test/utils/manifest_only_fixtures.py +80 -0
- airbyte_cdk-6.9.1.dev1.dist-info/METADATA +306 -0
- {airbyte_cdk-6.9.1.dist-info → airbyte_cdk-6.9.1.dev1.dist-info}/RECORD +15 -17
- airbyte_cdk/sources/declarative/resolvers/__init__.py +0 -13
- airbyte_cdk/sources/declarative/resolvers/components_resolver.py +0 -55
- airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py +0 -106
- airbyte_cdk-6.9.1.dist-info/METADATA +0 -108
- {airbyte_cdk-6.9.1.dist-info → airbyte_cdk-6.9.1.dev1.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.9.1.dist-info → airbyte_cdk-6.9.1.dev1.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.9.1.dist-info → airbyte_cdk-6.9.1.dev1.dist-info}/entry_points.txt +0 -0
@@ -62,11 +62,11 @@ airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=dAA-UhmMj0WLXCkRQr
|
|
62
62
|
airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
|
63
63
|
airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
|
64
64
|
airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
|
65
|
-
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=
|
65
|
+
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=F2X2ZS9eDfrohNbxG2TgPW-f4YP8IAkMjO1XHtD6NIg,23464
|
66
66
|
airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=l9LG7Qm6e5r_qgqfVKnx3mXYtg1I9MmMjomVIPfU4XA,177
|
67
67
|
airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=SX9JjdesN1edN2WVUVMzU_ptqp2QB1OnsnjZ4mwcX7w,2579
|
68
68
|
airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
|
69
|
-
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=
|
69
|
+
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=Q6zOEKvgAjMUsu89VmNteMhqdP0XwO9WWfA6-rMVn6A,120545
|
70
70
|
airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
|
71
71
|
airbyte_cdk/sources/declarative/declarative_stream.py,sha256=JRyNeOIpsFu4ztVZsN6sncqUEIqIE-bUkD2TPgbMgk0,10375
|
72
72
|
airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=hNlhaB5FjNC6IfJyglj5ZJWkYD2nEAukMDmzRz5PC6o,671
|
@@ -97,20 +97,20 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py,sha256=UrF
|
|
97
97
|
airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py,sha256=i2L0gREX8nHA-pKokdVqwBf4aJgWP71KOxIABj_DHcY,1857
|
98
98
|
airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=LYEZnZ_hB7rvBSZxG9s0RSrzsOkDWbBY0_P6qu5lEfc,3212
|
99
99
|
airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=-V5UddGm69UKEB6o_O1EIES9kfY8FV_X4Ji8w1yOuSA,981
|
100
|
-
airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=
|
100
|
+
airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=gcihTEnfD_6sUivxOomoY5r7VMAGqVVnK_HEsid9Y5k,6605
|
101
101
|
airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=jf24RK-1fBhTYDpcGEakZtGNNJfG5NS8CCF5bEgNmRo,3977
|
102
|
-
airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256
|
102
|
+
airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=FEOmFo2mwdpmO8pH9jnw-sUAnijjuigZWYqH_0Gq9oQ,12919
|
103
103
|
airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
104
104
|
airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
|
105
105
|
airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
|
106
106
|
airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
|
107
|
-
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=
|
107
|
+
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=tZcVSMRJ5-5MoX4ejpXxt5E2suqiQ5AwprQWQiWHhho,85060
|
108
108
|
airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
|
109
109
|
airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=Rir9_z3Kcd5Es0-LChrzk-0qubAsiK_RSEnLmK2OXm8,553
|
110
|
-
airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=
|
110
|
+
airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=jVZ3ZV5YZrmDNIX5cM2mugXmnbH27zHRcD22_3oatpo,8454
|
111
111
|
airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=IWUOdF03o-aQn0Occo1BJCxU0Pz-QILk5L67nzw2thw,6803
|
112
|
-
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=
|
113
|
-
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=
|
112
|
+
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=tO7xkv4y5iH6wGkj5As1T5ItUQxlw6cLflHAH48PKwc,96355
|
113
|
+
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=8uGos2u7TFTx_EJBdcjdUGn3Eyx6jUuEa1_VB8UP_dI,631
|
114
114
|
airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
|
115
115
|
airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py,sha256=t7pRdFWfFWJtQQG19c9PVeMODyO2BknRTakpM5U9N-8,4844
|
116
116
|
airbyte_cdk/sources/declarative/partition_routers/partition_router.py,sha256=YyEIzdmLd1FjbVP3QbQ2VFCLW_P-OGbVh6VpZShp54k,2218
|
@@ -152,9 +152,6 @@ airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_
|
|
152
152
|
airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py,sha256=8YRiDzjYvqJ-aMmKFcjqzv_-e8OZ5QG_TbpZ-nuCu6s,2590
|
153
153
|
airbyte_cdk/sources/declarative/requesters/request_path.py,sha256=S3MeFvcaQrMbOkSY2W2VbXLNomqt_3eXqVd9ZhgNwUs,299
|
154
154
|
airbyte_cdk/sources/declarative/requesters/requester.py,sha256=iVVpXQ4KEd9OyZNwmOofMvx7_06i8ZRxGo3aNTrEQLM,4946
|
155
|
-
airbyte_cdk/sources/declarative/resolvers/__init__.py,sha256=8cL--3WCJvtKVIIAqKWoIEhmmf3FemB7PlsnO0Mx_cY,688
|
156
|
-
airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=KPjKc0yb9artL4ZkeqN8RmEykHH6FJgqXD7fCEnh1X0,1936
|
157
|
-
airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=ZA2vrHQKfXNMcH3x1iuyFOTGNzYDhUFT2qcaiOzSK0A,4271
|
158
155
|
airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=FVQpUGVwp2Gibk4gp07VmLKX5AafUlsZWFSrDpUDuJM,443
|
159
156
|
airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=WDFnjrXLz3-YEjFhmlMkWAn9AJvnZ0mk9FyC8DAhEYk,4976
|
160
157
|
airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
|
@@ -276,7 +273,7 @@ airbyte_cdk/sources/streams/http/error_handlers/json_error_message_parser.py,sha
|
|
276
273
|
airbyte_cdk/sources/streams/http/error_handlers/response_models.py,sha256=xGIVELBFY0TmH9aUq1ikoqJz8oHLr6di2JLvKWVEO-s,2236
|
277
274
|
airbyte_cdk/sources/streams/http/exceptions.py,sha256=njC7MlMJoFYcSGz4mIp6-bqLFTr6vC8ej25X0oSeyjE,1824
|
278
275
|
airbyte_cdk/sources/streams/http/http.py,sha256=JAMpiTdS9HFNOlwayWNvQdxoqs2rpW9wdYlhFHv_1Q4,28496
|
279
|
-
airbyte_cdk/sources/streams/http/http_client.py,sha256=
|
276
|
+
airbyte_cdk/sources/streams/http/http_client.py,sha256=Jqmbd3jL8jjnOfA1325-cpG3nE80YDMDwyxPZ08D7wo,21341
|
280
277
|
airbyte_cdk/sources/streams/http/rate_limiting.py,sha256=IwdjrHKUnU97XO4qONgYRv4YYW51xQ8SJm4WLafXDB8,6351
|
281
278
|
airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py,sha256=RN0D3nOX1xLgwEwKWu6pkGy3XqBFzKSNZ8Lf6umU2eY,413
|
282
279
|
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=nxI94yJ3bGfpDO8RR3QvOJ-PSW0n9CElSAkgl5ae80Y,10321
|
@@ -316,6 +313,7 @@ airbyte_cdk/test/state_builder.py,sha256=kLPql9lNzUJaBg5YYRLJlY_Hy5JLHJDVyKPMZMo
|
|
316
313
|
airbyte_cdk/test/utils/__init__.py,sha256=Hu-1XT2KDoYjDF7-_ziDwv5bY3PueGjANOCbzeOegDg,57
|
317
314
|
airbyte_cdk/test/utils/data.py,sha256=CkCR1_-rujWNmPXFR1IXTMwx1rAl06wAyIKWpDcN02w,820
|
318
315
|
airbyte_cdk/test/utils/http_mocking.py,sha256=F2hpm2q4ijojQN5u2XtgTAp8aNgHgJ64eZNkZ9BW0ig,550
|
316
|
+
airbyte_cdk/test/utils/manifest_only_fixtures.py,sha256=R5smgqlheUI8b3wUcnOO_TkWw3so1ujUv9J31dvtsvE,2760
|
319
317
|
airbyte_cdk/test/utils/reading.py,sha256=SOTDYlps6Te9KumfTJ3vVDSm9EUXhvKtE8aD7gvdPlg,965
|
320
318
|
airbyte_cdk/utils/__init__.py,sha256=gHjOCoUkolS_nKtgFSudXUY-ObK2vUo6aNQLvW7o8q8,347
|
321
319
|
airbyte_cdk/utils/airbyte_secrets_utils.py,sha256=wEtRnl5KRhN6eLJwrDrC4FJjyqt_4vkA1F65mdl8c24,3142
|
@@ -333,8 +331,8 @@ airbyte_cdk/utils/slice_hasher.py,sha256=-pHexlNYoWYPnXNH-M7HEbjmeJe9Zk7SJijdQ7d
|
|
333
331
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
334
332
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
335
333
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
336
|
-
airbyte_cdk-6.9.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
337
|
-
airbyte_cdk-6.9.1.dist-info/METADATA,sha256=
|
338
|
-
airbyte_cdk-6.9.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
339
|
-
airbyte_cdk-6.9.1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
|
340
|
-
airbyte_cdk-6.9.1.dist-info/RECORD,,
|
334
|
+
airbyte_cdk-6.9.1.dev1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
335
|
+
airbyte_cdk-6.9.1.dev1.dist-info/METADATA,sha256=Is6NHUF38LCSaMH4rAlCxW3J1WR9vTNSHIEKUzLGHlM,13485
|
336
|
+
airbyte_cdk-6.9.1.dev1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
337
|
+
airbyte_cdk-6.9.1.dev1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
|
338
|
+
airbyte_cdk-6.9.1.dev1.dist-info/RECORD,,
|
@@ -1,13 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
from airbyte_cdk.sources.declarative.resolvers.components_resolver import ComponentsResolver, ComponentMappingDefinition, ResolvedComponentMappingDefinition
|
6
|
-
from airbyte_cdk.sources.declarative.resolvers.http_components_resolver import HttpComponentsResolver
|
7
|
-
from airbyte_cdk.sources.declarative.models import HttpComponentsResolver as HttpComponentsResolverModel
|
8
|
-
|
9
|
-
COMPONENTS_RESOLVER_TYPE_MAPPING = {
|
10
|
-
"HttpComponentsResolver": HttpComponentsResolverModel
|
11
|
-
}
|
12
|
-
|
13
|
-
__all__ = ["ComponentsResolver", "HttpComponentsResolver", "ComponentMappingDefinition", "ResolvedComponentMappingDefinition", "COMPONENTS_RESOLVER_TYPE_MAPPING"]
|
@@ -1,55 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
from abc import ABC, abstractmethod
|
6
|
-
from dataclasses import InitVar, dataclass
|
7
|
-
from typing import Any, Dict, Iterable, List, Mapping, Optional, Type, Union
|
8
|
-
|
9
|
-
from typing_extensions import deprecated
|
10
|
-
|
11
|
-
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
12
|
-
from airbyte_cdk.sources.source import ExperimentalClassWarning
|
13
|
-
|
14
|
-
|
15
|
-
@dataclass(frozen=True)
|
16
|
-
class ComponentMappingDefinition:
|
17
|
-
"""Defines the configuration for mapping a component in a stream. This class specifies
|
18
|
-
what field in the stream template should be updated with value, supporting dynamic interpolation
|
19
|
-
and type enforcement."""
|
20
|
-
|
21
|
-
field_path: List["InterpolatedString"]
|
22
|
-
value: Union["InterpolatedString", str]
|
23
|
-
value_type: Optional[Type[Any]]
|
24
|
-
parameters: InitVar[Mapping[str, Any]]
|
25
|
-
|
26
|
-
|
27
|
-
@dataclass(frozen=True)
|
28
|
-
class ResolvedComponentMappingDefinition:
|
29
|
-
"""Defines resolved configuration for mapping a component in a stream. This class specifies
|
30
|
-
what field in the stream template should be updated with value, supporting dynamic interpolation
|
31
|
-
and type enforcement."""
|
32
|
-
|
33
|
-
field_path: List["InterpolatedString"]
|
34
|
-
value: "InterpolatedString"
|
35
|
-
value_type: Optional[Type[Any]]
|
36
|
-
parameters: InitVar[Mapping[str, Any]]
|
37
|
-
|
38
|
-
|
39
|
-
@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
|
40
|
-
@dataclass
|
41
|
-
class ComponentsResolver(ABC):
|
42
|
-
"""
|
43
|
-
Abstract base class for resolving components in a stream template.
|
44
|
-
"""
|
45
|
-
|
46
|
-
@abstractmethod
|
47
|
-
def resolve_components(
|
48
|
-
self, stream_template_config: Dict[str, Any]
|
49
|
-
) -> Iterable[Dict[str, Any]]:
|
50
|
-
"""
|
51
|
-
Maps and populates values into a stream template configuration.
|
52
|
-
:param stream_template_config: The stream template with placeholders for components.
|
53
|
-
:yields: The resolved stream config with populated values.
|
54
|
-
"""
|
55
|
-
pass
|
@@ -1,106 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
3
|
-
#
|
4
|
-
|
5
|
-
from copy import deepcopy
|
6
|
-
from dataclasses import InitVar, dataclass, field
|
7
|
-
from typing import Any, Dict, Iterable, List, Mapping
|
8
|
-
|
9
|
-
import dpath
|
10
|
-
from typing_extensions import deprecated
|
11
|
-
|
12
|
-
from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
|
13
|
-
from airbyte_cdk.sources.declarative.resolvers.components_resolver import (
|
14
|
-
ComponentMappingDefinition,
|
15
|
-
ComponentsResolver,
|
16
|
-
ResolvedComponentMappingDefinition,
|
17
|
-
)
|
18
|
-
from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
|
19
|
-
from airbyte_cdk.sources.source import ExperimentalClassWarning
|
20
|
-
from airbyte_cdk.sources.types import Config
|
21
|
-
|
22
|
-
|
23
|
-
@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
|
24
|
-
@dataclass
|
25
|
-
class HttpComponentsResolver(ComponentsResolver):
|
26
|
-
"""
|
27
|
-
Resolves and populates stream templates with components fetched via an HTTP retriever.
|
28
|
-
|
29
|
-
Attributes:
|
30
|
-
retriever (Retriever): The retriever used to fetch data from an API.
|
31
|
-
config (Config): Configuration object for the resolver.
|
32
|
-
components_mapping (List[ComponentMappingDefinition]): List of mappings to resolve.
|
33
|
-
parameters (InitVar[Mapping[str, Any]]): Additional parameters for interpolation.
|
34
|
-
"""
|
35
|
-
|
36
|
-
retriever: Retriever
|
37
|
-
config: Config
|
38
|
-
components_mapping: List[ComponentMappingDefinition]
|
39
|
-
parameters: InitVar[Mapping[str, Any]]
|
40
|
-
_resolved_components: List[ResolvedComponentMappingDefinition] = field(
|
41
|
-
init=False, repr=False, default_factory=list
|
42
|
-
)
|
43
|
-
|
44
|
-
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
45
|
-
"""
|
46
|
-
Initializes and parses component mappings, converting them to resolved definitions.
|
47
|
-
|
48
|
-
Args:
|
49
|
-
parameters (Mapping[str, Any]): Parameters for interpolation.
|
50
|
-
"""
|
51
|
-
for component_mapping in self.components_mapping:
|
52
|
-
if isinstance(component_mapping.value, (str, InterpolatedString)):
|
53
|
-
interpolated_value = (
|
54
|
-
InterpolatedString.create(component_mapping.value, parameters=parameters)
|
55
|
-
if isinstance(component_mapping.value, str)
|
56
|
-
else component_mapping.value
|
57
|
-
)
|
58
|
-
|
59
|
-
field_path = [
|
60
|
-
InterpolatedString.create(path, parameters=parameters)
|
61
|
-
for path in component_mapping.field_path
|
62
|
-
]
|
63
|
-
|
64
|
-
self._resolved_components.append(
|
65
|
-
ResolvedComponentMappingDefinition(
|
66
|
-
field_path=field_path,
|
67
|
-
value=interpolated_value,
|
68
|
-
value_type=component_mapping.value_type,
|
69
|
-
parameters=parameters,
|
70
|
-
)
|
71
|
-
)
|
72
|
-
else:
|
73
|
-
raise ValueError(
|
74
|
-
f"Expected a string or InterpolatedString for value in mapping: {component_mapping}"
|
75
|
-
)
|
76
|
-
|
77
|
-
def resolve_components(
|
78
|
-
self, stream_template_config: Dict[str, Any]
|
79
|
-
) -> Iterable[Dict[str, Any]]:
|
80
|
-
"""
|
81
|
-
Resolves components in the stream template configuration by populating values.
|
82
|
-
|
83
|
-
Args:
|
84
|
-
stream_template_config (Dict[str, Any]): Stream template to populate.
|
85
|
-
|
86
|
-
Yields:
|
87
|
-
Dict[str, Any]: Updated configurations with resolved components.
|
88
|
-
"""
|
89
|
-
kwargs = {"stream_template_config": stream_template_config}
|
90
|
-
|
91
|
-
for components_values in self.retriever.read_records({}):
|
92
|
-
updated_config = deepcopy(stream_template_config)
|
93
|
-
kwargs["components_values"] = components_values # type: ignore[assignment] # component_values will always be of type Mapping[str, Any]
|
94
|
-
|
95
|
-
for resolved_component in self._resolved_components:
|
96
|
-
valid_types = (
|
97
|
-
(resolved_component.value_type,) if resolved_component.value_type else None
|
98
|
-
)
|
99
|
-
value = resolved_component.value.eval(
|
100
|
-
self.config, valid_types=valid_types, **kwargs
|
101
|
-
)
|
102
|
-
|
103
|
-
path = [path.eval(self.config, **kwargs) for path in resolved_component.field_path]
|
104
|
-
dpath.set(updated_config, path, value)
|
105
|
-
|
106
|
-
yield updated_config
|
@@ -1,108 +0,0 @@
|
|
1
|
-
Metadata-Version: 2.1
|
2
|
-
Name: airbyte-cdk
|
3
|
-
Version: 6.9.1
|
4
|
-
Summary: A framework for writing Airbyte Connectors.
|
5
|
-
Home-page: https://airbyte.com
|
6
|
-
License: MIT
|
7
|
-
Keywords: airbyte,connector-development-kit,cdk
|
8
|
-
Author: Airbyte
|
9
|
-
Author-email: contact@airbyte.io
|
10
|
-
Requires-Python: >=3.10,<3.13
|
11
|
-
Classifier: Development Status :: 3 - Alpha
|
12
|
-
Classifier: Intended Audience :: Developers
|
13
|
-
Classifier: License :: OSI Approved :: MIT License
|
14
|
-
Classifier: Programming Language :: Python :: 3
|
15
|
-
Classifier: Programming Language :: Python :: 3.10
|
16
|
-
Classifier: Programming Language :: Python :: 3.11
|
17
|
-
Classifier: Programming Language :: Python :: 3.12
|
18
|
-
Classifier: Topic :: Scientific/Engineering
|
19
|
-
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
20
|
-
Provides-Extra: file-based
|
21
|
-
Provides-Extra: sql
|
22
|
-
Provides-Extra: vector-db-based
|
23
|
-
Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
|
24
|
-
Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
|
25
|
-
Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
|
26
|
-
Requires-Dist: avro (>=1.11.2,<1.12.0) ; extra == "file-based"
|
27
|
-
Requires-Dist: backoff
|
28
|
-
Requires-Dist: cachetools
|
29
|
-
Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
|
30
|
-
Requires-Dist: cryptography (>=42.0.5,<44.0.0)
|
31
|
-
Requires-Dist: dpath (>=2.1.6,<3.0.0)
|
32
|
-
Requires-Dist: dunamai (>=1.22.0,<2.0.0)
|
33
|
-
Requires-Dist: fastavro (>=1.8.0,<1.9.0) ; extra == "file-based"
|
34
|
-
Requires-Dist: genson (==1.3.0)
|
35
|
-
Requires-Dist: isodate (>=0.6.1,<0.7.0)
|
36
|
-
Requires-Dist: jsonref (>=0.2,<0.3)
|
37
|
-
Requires-Dist: jsonschema (>=4.17.3,<4.18.0)
|
38
|
-
Requires-Dist: langchain (==0.1.16) ; extra == "vector-db-based"
|
39
|
-
Requires-Dist: langchain_core (==0.1.42)
|
40
|
-
Requires-Dist: markdown ; extra == "file-based"
|
41
|
-
Requires-Dist: nltk (==3.9.1)
|
42
|
-
Requires-Dist: numpy (<2)
|
43
|
-
Requires-Dist: openai[embeddings] (==0.27.9) ; extra == "vector-db-based"
|
44
|
-
Requires-Dist: orjson (>=3.10.7,<4.0.0)
|
45
|
-
Requires-Dist: pandas (==2.2.2)
|
46
|
-
Requires-Dist: pdf2image (==1.16.3) ; extra == "file-based"
|
47
|
-
Requires-Dist: pdfminer.six (==20221105) ; extra == "file-based"
|
48
|
-
Requires-Dist: pendulum (<3.0.0)
|
49
|
-
Requires-Dist: psutil (==6.1.0)
|
50
|
-
Requires-Dist: pyarrow (>=15.0.0,<15.1.0) ; extra == "file-based"
|
51
|
-
Requires-Dist: pydantic (>=2.7,<3.0)
|
52
|
-
Requires-Dist: pyjwt (>=2.8.0,<3.0.0)
|
53
|
-
Requires-Dist: pyrate-limiter (>=3.1.0,<3.2.0)
|
54
|
-
Requires-Dist: pytesseract (==0.3.10) ; extra == "file-based"
|
55
|
-
Requires-Dist: python-calamine (==0.2.3) ; extra == "file-based"
|
56
|
-
Requires-Dist: python-dateutil
|
57
|
-
Requires-Dist: python-snappy (==0.7.3) ; extra == "file-based"
|
58
|
-
Requires-Dist: python-ulid (>=3.0.0,<4.0.0)
|
59
|
-
Requires-Dist: pytz (==2024.1)
|
60
|
-
Requires-Dist: rapidfuzz (>=3.10.1,<4.0.0)
|
61
|
-
Requires-Dist: requests
|
62
|
-
Requires-Dist: requests_cache
|
63
|
-
Requires-Dist: serpyco-rs (>=1.10.2,<2.0.0)
|
64
|
-
Requires-Dist: sqlalchemy (>=2.0,<3.0,!=2.0.36) ; extra == "sql"
|
65
|
-
Requires-Dist: tiktoken (==0.8.0) ; extra == "vector-db-based"
|
66
|
-
Requires-Dist: unstructured.pytesseract (>=0.3.12) ; extra == "file-based"
|
67
|
-
Requires-Dist: unstructured[docx,pptx] (==0.10.27) ; extra == "file-based"
|
68
|
-
Requires-Dist: wcmatch (==10.0)
|
69
|
-
Requires-Dist: xmltodict (>=0.13.0,<0.14.0)
|
70
|
-
Project-URL: Documentation, https://docs.airbyte.io/
|
71
|
-
Project-URL: Repository, https://github.com/airbytehq/airbyte-python-cdk
|
72
|
-
Description-Content-Type: text/markdown
|
73
|
-
|
74
|
-
# Airbyte Python CDK and Low-Code CDK
|
75
|
-
|
76
|
-
Airbyte Python CDK is a framework for building Airbyte API Source Connectors. It provides a set of
|
77
|
-
classes and helpers that make it easy to build a connector against an HTTP API (REST, GraphQL, etc),
|
78
|
-
or a generic Python source connector.
|
79
|
-
|
80
|
-
## Building Connectors with the CDK
|
81
|
-
|
82
|
-
If you're looking to build a connector, we highly recommend that you first
|
83
|
-
[start with the Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview).
|
84
|
-
It should be enough for 90% connectors out there. For more flexible and complex connectors, use the
|
85
|
-
[low-code CDK and `SourceDeclarativeManifest`](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview).
|
86
|
-
|
87
|
-
For more information on building connectors, please see the [Connector Development](https://docs.airbyte.com/connector-development/) guide on [docs.airbyte.com](https://docs.airbyte.com).
|
88
|
-
|
89
|
-
## Python CDK Overview
|
90
|
-
|
91
|
-
Airbyte CDK code is within `airbyte_cdk` directory. Here's a high level overview of what's inside:
|
92
|
-
|
93
|
-
- `airbyte_cdk/connector_builder`. Internal wrapper that helps the Connector Builder platform run a declarative manifest (low-code connector). You should not use this code directly. If you need to run a `SourceDeclarativeManifest`, take a look at [`source-declarative-manifest`](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-declarative-manifest) connector implementation instead.
|
94
|
-
- `airbyte_cdk/cli/source_declarative_manifest`. This module defines the `source-declarative-manifest` (aka "SDM") connector execution logic and associated CLI.
|
95
|
-
- `airbyte_cdk/destinations`. Basic Destination connector support! If you're building a Destination connector in Python, try that. Some of our vector DB destinations like `destination-pinecone` are using that code.
|
96
|
-
- `airbyte_cdk/models` expose `airbyte_protocol.models` as a part of `airbyte_cdk` package.
|
97
|
-
- `airbyte_cdk/sources/concurrent_source` is the Concurrent CDK implementation. It supports reading data from streams concurrently per slice / partition, useful for connectors with high throughput and high number of records.
|
98
|
-
- `airbyte_cdk/sources/declarative` is the low-code CDK. It works on top of Airbyte Python CDK, but provides a declarative manifest language to define streams, operations, etc. This makes it easier to build connectors without writing Python code.
|
99
|
-
- `airbyte_cdk/sources/file_based` is the CDK for file-based sources. Examples include S3, Azure, GCS, etc.
|
100
|
-
|
101
|
-
## Contributing
|
102
|
-
|
103
|
-
For instructions on how to contribute, please see our [Contributing Guide](docs/CONTRIBUTING.md).
|
104
|
-
|
105
|
-
## Release Management
|
106
|
-
|
107
|
-
Please see the [Release Management](docs/RELEASES.md) guide for information on how to perform releases and pre-releases.
|
108
|
-
|
File without changes
|
File without changes
|
File without changes
|