airbyte-cdk 6.43.0.dev0__py3-none-any.whl → 6.44.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/connector_builder/connector_builder_handler.py +22 -8
- airbyte_cdk/connector_builder/main.py +3 -3
- airbyte_cdk/sources/declarative/checks/__init__.py +5 -2
- airbyte_cdk/sources/declarative/checks/check_stream.py +113 -11
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +23 -129
- airbyte_cdk/sources/declarative/manifest_declarative_source.py +6 -0
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +21 -87
- airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py +16 -4
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +44 -192
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +2 -25
- airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +31 -100
- airbyte_cdk/sources/declarative/transformations/add_fields.py +3 -1
- {airbyte_cdk-6.43.0.dev0.dist-info → airbyte_cdk-6.44.0.dist-info}/METADATA +1 -1
- {airbyte_cdk-6.43.0.dev0.dist-info → airbyte_cdk-6.44.0.dist-info}/RECORD +18 -23
- airbyte_cdk/sources/declarative/requesters/query_properties/__init__.py +0 -14
- airbyte_cdk/sources/declarative/requesters/query_properties/group_by_key.py +0 -24
- airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py +0 -40
- airbyte_cdk/sources/declarative/requesters/query_properties/property_chunking.py +0 -65
- airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py +0 -48
- {airbyte_cdk-6.43.0.dev0.dist-info → airbyte_cdk-6.44.0.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.43.0.dev0.dist-info → airbyte_cdk-6.44.0.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.43.0.dev0.dist-info → airbyte_cdk-6.44.0.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.43.0.dev0.dist-info → airbyte_cdk-6.44.0.dist-info}/entry_points.txt +0 -0
@@ -4,7 +4,7 @@
|
|
4
4
|
|
5
5
|
|
6
6
|
from dataclasses import asdict, dataclass, field
|
7
|
-
from typing import Any, List, Mapping
|
7
|
+
from typing import Any, Dict, List, Mapping
|
8
8
|
|
9
9
|
from airbyte_cdk.connector_builder.test_reader import TestReader
|
10
10
|
from airbyte_cdk.models import (
|
@@ -27,30 +27,34 @@ from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
|
27
27
|
DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE = 5
|
28
28
|
DEFAULT_MAXIMUM_NUMBER_OF_SLICES = 5
|
29
29
|
DEFAULT_MAXIMUM_RECORDS = 100
|
30
|
+
DEFAULT_MAXIMUM_STREAMS = 100
|
30
31
|
|
31
32
|
MAX_PAGES_PER_SLICE_KEY = "max_pages_per_slice"
|
32
33
|
MAX_SLICES_KEY = "max_slices"
|
33
34
|
MAX_RECORDS_KEY = "max_records"
|
35
|
+
MAX_STREAMS_KEY = "max_streams"
|
34
36
|
|
35
37
|
|
36
38
|
@dataclass
|
37
|
-
class
|
39
|
+
class TestLimits:
|
38
40
|
max_records: int = field(default=DEFAULT_MAXIMUM_RECORDS)
|
39
41
|
max_pages_per_slice: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE)
|
40
42
|
max_slices: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_SLICES)
|
43
|
+
max_streams: int = field(default=DEFAULT_MAXIMUM_STREAMS)
|
41
44
|
|
42
45
|
|
43
|
-
def get_limits(config: Mapping[str, Any]) ->
|
46
|
+
def get_limits(config: Mapping[str, Any]) -> TestLimits:
|
44
47
|
command_config = config.get("__test_read_config", {})
|
45
48
|
max_pages_per_slice = (
|
46
49
|
command_config.get(MAX_PAGES_PER_SLICE_KEY) or DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE
|
47
50
|
)
|
48
51
|
max_slices = command_config.get(MAX_SLICES_KEY) or DEFAULT_MAXIMUM_NUMBER_OF_SLICES
|
49
52
|
max_records = command_config.get(MAX_RECORDS_KEY) or DEFAULT_MAXIMUM_RECORDS
|
50
|
-
|
53
|
+
max_streams = command_config.get(MAX_STREAMS_KEY) or DEFAULT_MAXIMUM_STREAMS
|
54
|
+
return TestLimits(max_records, max_pages_per_slice, max_slices, max_streams)
|
51
55
|
|
52
56
|
|
53
|
-
def create_source(config: Mapping[str, Any], limits:
|
57
|
+
def create_source(config: Mapping[str, Any], limits: TestLimits) -> ManifestDeclarativeSource:
|
54
58
|
manifest = config["__injected_declarative_manifest"]
|
55
59
|
return ManifestDeclarativeSource(
|
56
60
|
config=config,
|
@@ -71,7 +75,7 @@ def read_stream(
|
|
71
75
|
config: Mapping[str, Any],
|
72
76
|
configured_catalog: ConfiguredAirbyteCatalog,
|
73
77
|
state: List[AirbyteStateMessage],
|
74
|
-
limits:
|
78
|
+
limits: TestLimits,
|
75
79
|
) -> AirbyteMessage:
|
76
80
|
try:
|
77
81
|
test_read_handler = TestReader(
|
@@ -117,13 +121,23 @@ def resolve_manifest(source: ManifestDeclarativeSource) -> AirbyteMessage:
|
|
117
121
|
return error.as_airbyte_message()
|
118
122
|
|
119
123
|
|
120
|
-
def full_resolve_manifest(source: ManifestDeclarativeSource) -> AirbyteMessage:
|
124
|
+
def full_resolve_manifest(source: ManifestDeclarativeSource, limits: TestLimits) -> AirbyteMessage:
|
121
125
|
try:
|
122
126
|
manifest = {**source.resolved_manifest}
|
123
127
|
streams = manifest.get("streams", [])
|
124
128
|
for stream in streams:
|
125
129
|
stream["dynamic_stream_name"] = None
|
126
|
-
|
130
|
+
|
131
|
+
mapped_streams: Dict[str, List[Dict[str, Any]]] = {}
|
132
|
+
for stream in source.dynamic_streams:
|
133
|
+
generated_streams = mapped_streams.setdefault(stream["dynamic_stream_name"], [])
|
134
|
+
|
135
|
+
if len(generated_streams) < limits.max_streams:
|
136
|
+
generated_streams += [stream]
|
137
|
+
|
138
|
+
for generated_streams_list in mapped_streams.values():
|
139
|
+
streams.extend(generated_streams_list)
|
140
|
+
|
127
141
|
manifest["streams"] = streams
|
128
142
|
return AirbyteMessage(
|
129
143
|
type=Type.RECORD,
|
@@ -10,7 +10,7 @@ import orjson
|
|
10
10
|
|
11
11
|
from airbyte_cdk.connector import BaseConnector
|
12
12
|
from airbyte_cdk.connector_builder.connector_builder_handler import (
|
13
|
-
|
13
|
+
TestLimits,
|
14
14
|
create_source,
|
15
15
|
full_resolve_manifest,
|
16
16
|
get_limits,
|
@@ -73,7 +73,7 @@ def handle_connector_builder_request(
|
|
73
73
|
config: Mapping[str, Any],
|
74
74
|
catalog: Optional[ConfiguredAirbyteCatalog],
|
75
75
|
state: List[AirbyteStateMessage],
|
76
|
-
limits:
|
76
|
+
limits: TestLimits,
|
77
77
|
) -> AirbyteMessage:
|
78
78
|
if command == "resolve_manifest":
|
79
79
|
return resolve_manifest(source)
|
@@ -83,7 +83,7 @@ def handle_connector_builder_request(
|
|
83
83
|
), "`test_read` requires a valid `ConfiguredAirbyteCatalog`, got None."
|
84
84
|
return read_stream(source, config, catalog, state, limits)
|
85
85
|
elif command == "full_resolve_manifest":
|
86
|
-
return full_resolve_manifest(source)
|
86
|
+
return full_resolve_manifest(source, limits)
|
87
87
|
else:
|
88
88
|
raise ValueError(f"Unrecognized command {command}.")
|
89
89
|
|
@@ -7,7 +7,10 @@ from typing import Mapping
|
|
7
7
|
from pydantic.v1 import BaseModel
|
8
8
|
|
9
9
|
from airbyte_cdk.sources.declarative.checks.check_dynamic_stream import CheckDynamicStream
|
10
|
-
from airbyte_cdk.sources.declarative.checks.check_stream import
|
10
|
+
from airbyte_cdk.sources.declarative.checks.check_stream import (
|
11
|
+
CheckStream,
|
12
|
+
DynamicStreamCheckConfig,
|
13
|
+
)
|
11
14
|
from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
|
12
15
|
from airbyte_cdk.sources.declarative.models import (
|
13
16
|
CheckDynamicStream as CheckDynamicStreamModel,
|
@@ -21,4 +24,4 @@ COMPONENTS_CHECKER_TYPE_MAPPING: Mapping[str, type[BaseModel]] = {
|
|
21
24
|
"CheckDynamicStream": CheckDynamicStreamModel,
|
22
25
|
}
|
23
26
|
|
24
|
-
__all__ = ["CheckStream", "CheckDynamicStream", "ConnectionChecker"]
|
27
|
+
__all__ = ["CheckStream", "CheckDynamicStream", "ConnectionChecker", "DynamicStreamCheckConfig"]
|
@@ -5,13 +5,23 @@
|
|
5
5
|
import logging
|
6
6
|
import traceback
|
7
7
|
from dataclasses import InitVar, dataclass
|
8
|
-
from typing import Any, List, Mapping, Tuple
|
8
|
+
from typing import Any, Dict, List, Mapping, Optional, Tuple
|
9
9
|
|
10
10
|
from airbyte_cdk import AbstractSource
|
11
11
|
from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
|
12
12
|
from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy
|
13
13
|
|
14
14
|
|
15
|
+
@dataclass(frozen=True)
|
16
|
+
class DynamicStreamCheckConfig:
|
17
|
+
"""Defines the configuration for dynamic stream during connection checking. This class specifies
|
18
|
+
what dynamic streams in the stream template should be updated with value, supporting dynamic interpolation
|
19
|
+
and type enforcement."""
|
20
|
+
|
21
|
+
dynamic_stream_name: str
|
22
|
+
stream_count: int = 0
|
23
|
+
|
24
|
+
|
15
25
|
@dataclass
|
16
26
|
class CheckStream(ConnectionChecker):
|
17
27
|
"""
|
@@ -23,34 +33,126 @@ class CheckStream(ConnectionChecker):
|
|
23
33
|
|
24
34
|
stream_names: List[str]
|
25
35
|
parameters: InitVar[Mapping[str, Any]]
|
36
|
+
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
|
26
37
|
|
27
38
|
def __post_init__(self, parameters: Mapping[str, Any]) -> None:
|
28
39
|
self._parameters = parameters
|
40
|
+
if self.dynamic_streams_check_configs is None:
|
41
|
+
self.dynamic_streams_check_configs = []
|
42
|
+
|
43
|
+
def _log_error(self, logger: logging.Logger, action: str, error: Exception) -> Tuple[bool, str]:
|
44
|
+
"""Logs an error and returns a formatted error message."""
|
45
|
+
error_message = f"Encountered an error while {action}. Error: {error}"
|
46
|
+
logger.error(error_message + f"Error traceback: \n {traceback.format_exc()}", exc_info=True)
|
47
|
+
return False, error_message
|
29
48
|
|
30
49
|
def check_connection(
|
31
50
|
self, source: AbstractSource, logger: logging.Logger, config: Mapping[str, Any]
|
32
51
|
) -> Tuple[bool, Any]:
|
33
|
-
|
52
|
+
"""Checks the connection to the source and its streams."""
|
53
|
+
try:
|
54
|
+
streams = source.streams(config=config)
|
55
|
+
if not streams:
|
56
|
+
return False, f"No streams to connect to from source {source}"
|
57
|
+
except Exception as error:
|
58
|
+
return self._log_error(logger, "discovering streams", error)
|
59
|
+
|
34
60
|
stream_name_to_stream = {s.name: s for s in streams}
|
35
|
-
if len(streams) == 0:
|
36
|
-
return False, f"No streams to connect to from source {source}"
|
37
61
|
for stream_name in self.stream_names:
|
38
|
-
if stream_name not in stream_name_to_stream
|
62
|
+
if stream_name not in stream_name_to_stream:
|
39
63
|
raise ValueError(
|
40
|
-
f"{stream_name} is not part of the catalog. Expected one of {stream_name_to_stream.keys()}."
|
64
|
+
f"{stream_name} is not part of the catalog. Expected one of {list(stream_name_to_stream.keys())}."
|
41
65
|
)
|
42
66
|
|
67
|
+
stream_availability, message = self._check_stream_availability(
|
68
|
+
stream_name_to_stream, stream_name, logger
|
69
|
+
)
|
70
|
+
if not stream_availability:
|
71
|
+
return stream_availability, message
|
72
|
+
|
73
|
+
should_check_dynamic_streams = (
|
74
|
+
hasattr(source, "resolved_manifest")
|
75
|
+
and hasattr(source, "dynamic_streams")
|
76
|
+
and self.dynamic_streams_check_configs
|
77
|
+
)
|
78
|
+
|
79
|
+
if should_check_dynamic_streams:
|
80
|
+
return self._check_dynamic_streams_availability(source, stream_name_to_stream, logger)
|
81
|
+
|
82
|
+
return True, None
|
83
|
+
|
84
|
+
def _check_stream_availability(
|
85
|
+
self, stream_name_to_stream: Dict[str, Any], stream_name: str, logger: logging.Logger
|
86
|
+
) -> Tuple[bool, Any]:
|
87
|
+
"""Checks if streams are available."""
|
88
|
+
availability_strategy = HttpAvailabilityStrategy()
|
89
|
+
try:
|
43
90
|
stream = stream_name_to_stream[stream_name]
|
44
|
-
|
91
|
+
stream_is_available, reason = availability_strategy.check_availability(stream, logger)
|
92
|
+
if not stream_is_available:
|
93
|
+
message = f"Stream {stream_name} is not available: {reason}"
|
94
|
+
logger.warning(message)
|
95
|
+
return stream_is_available, message
|
96
|
+
except Exception as error:
|
97
|
+
return self._log_error(logger, f"checking availability of stream {stream_name}", error)
|
98
|
+
return True, None
|
99
|
+
|
100
|
+
def _check_dynamic_streams_availability(
|
101
|
+
self, source: AbstractSource, stream_name_to_stream: Dict[str, Any], logger: logging.Logger
|
102
|
+
) -> Tuple[bool, Any]:
|
103
|
+
"""Checks the availability of dynamic streams."""
|
104
|
+
dynamic_streams = source.resolved_manifest.get("dynamic_streams", []) # type: ignore[attr-defined] # The source's resolved_manifest manifest is checked before calling this method
|
105
|
+
dynamic_stream_name_to_dynamic_stream = {
|
106
|
+
ds.get("name", f"dynamic_stream_{i}"): ds for i, ds in enumerate(dynamic_streams)
|
107
|
+
}
|
108
|
+
generated_streams = self._map_generated_streams(source.dynamic_streams) # type: ignore[attr-defined] # The source's dynamic_streams manifest is checked before calling this method
|
109
|
+
|
110
|
+
for check_config in self.dynamic_streams_check_configs: # type: ignore[union-attr] # None value for self.dynamic_streams_check_configs handled in __post_init__
|
111
|
+
if check_config.dynamic_stream_name not in dynamic_stream_name_to_dynamic_stream:
|
112
|
+
return (
|
113
|
+
False,
|
114
|
+
f"Dynamic stream {check_config.dynamic_stream_name} is not found in manifest.",
|
115
|
+
)
|
116
|
+
|
117
|
+
generated = generated_streams.get(check_config.dynamic_stream_name, [])
|
118
|
+
stream_availability, message = self._check_generated_streams_availability(
|
119
|
+
generated, stream_name_to_stream, logger, check_config.stream_count
|
120
|
+
)
|
121
|
+
if not stream_availability:
|
122
|
+
return stream_availability, message
|
123
|
+
|
124
|
+
return True, None
|
125
|
+
|
126
|
+
def _map_generated_streams(
|
127
|
+
self, dynamic_streams: List[Dict[str, Any]]
|
128
|
+
) -> Dict[str, List[Dict[str, Any]]]:
|
129
|
+
"""Maps dynamic stream names to their corresponding generated streams."""
|
130
|
+
mapped_streams: Dict[str, List[Dict[str, Any]]] = {}
|
131
|
+
for stream in dynamic_streams:
|
132
|
+
mapped_streams.setdefault(stream["dynamic_stream_name"], []).append(stream)
|
133
|
+
return mapped_streams
|
134
|
+
|
135
|
+
def _check_generated_streams_availability(
|
136
|
+
self,
|
137
|
+
generated_streams: List[Dict[str, Any]],
|
138
|
+
stream_name_to_stream: Dict[str, Any],
|
139
|
+
logger: logging.Logger,
|
140
|
+
max_count: int,
|
141
|
+
) -> Tuple[bool, Any]:
|
142
|
+
"""Checks availability of generated dynamic streams."""
|
143
|
+
availability_strategy = HttpAvailabilityStrategy()
|
144
|
+
for declarative_stream in generated_streams[: min(max_count, len(generated_streams))]:
|
145
|
+
stream = stream_name_to_stream[declarative_stream["name"]]
|
45
146
|
try:
|
46
147
|
stream_is_available, reason = availability_strategy.check_availability(
|
47
148
|
stream, logger
|
48
149
|
)
|
49
150
|
if not stream_is_available:
|
50
|
-
|
151
|
+
message = f"Dynamic Stream {stream.name} is not available: {reason}"
|
152
|
+
logger.warning(message)
|
153
|
+
return False, message
|
51
154
|
except Exception as error:
|
52
|
-
|
53
|
-
f"
|
155
|
+
return self._log_error(
|
156
|
+
logger, f"checking availability of dynamic stream {stream.name}", error
|
54
157
|
)
|
55
|
-
return False, f"Unable to connect to stream {stream_name} - {error}"
|
56
158
|
return True, None
|
@@ -316,7 +316,6 @@ definitions:
|
|
316
316
|
type: object
|
317
317
|
required:
|
318
318
|
- type
|
319
|
-
- stream_names
|
320
319
|
properties:
|
321
320
|
type:
|
322
321
|
type: string
|
@@ -330,6 +329,28 @@ definitions:
|
|
330
329
|
examples:
|
331
330
|
- ["users"]
|
332
331
|
- ["users", "contacts"]
|
332
|
+
dynamic_streams_check_configs:
|
333
|
+
type: array
|
334
|
+
items:
|
335
|
+
"$ref": "#/definitions/DynamicStreamCheckConfig"
|
336
|
+
DynamicStreamCheckConfig:
|
337
|
+
type: object
|
338
|
+
required:
|
339
|
+
- type
|
340
|
+
- dynamic_stream_name
|
341
|
+
properties:
|
342
|
+
type:
|
343
|
+
type: string
|
344
|
+
enum: [ DynamicStreamCheckConfig ]
|
345
|
+
dynamic_stream_name:
|
346
|
+
title: Dynamic Stream Name
|
347
|
+
description: The dynamic stream name.
|
348
|
+
type: string
|
349
|
+
stream_count:
|
350
|
+
title: Stream Count
|
351
|
+
description: The number of streams to attempt reading from during a check operation. If `stream_count` exceeds the total number of available streams, the minimum of the two values will be used.
|
352
|
+
type: integer
|
353
|
+
default: 0
|
333
354
|
CheckDynamicStream:
|
334
355
|
title: Dynamic Streams to Check
|
335
356
|
description: (This component is experimental. Use at your own risk.) Defines the dynamic streams to try reading when running a check operation.
|
@@ -1023,15 +1044,6 @@ definitions:
|
|
1023
1044
|
$parameters:
|
1024
1045
|
type: object
|
1025
1046
|
additionalProperties: true
|
1026
|
-
EmitPartialRecordMergeStrategy:
|
1027
|
-
title: Emit Partial Record
|
1028
|
-
description: Record merge strategy where in the case where multiple requests are needed to retrieve all properties, properties are not consolidated back into a single record and are instead emitted as separate groups of properties. This strategy should only be used when records do not have a unique identifier like a primary key.
|
1029
|
-
required:
|
1030
|
-
- type
|
1031
|
-
properties:
|
1032
|
-
type:
|
1033
|
-
type: string
|
1034
|
-
enum: [EmitPartialRecordMergeStrategy]
|
1035
1047
|
JwtAuthenticator:
|
1036
1048
|
title: JWT Authenticator
|
1037
1049
|
description: Authenticator for requests using JWT authentication flow.
|
@@ -1740,30 +1752,6 @@ definitions:
|
|
1740
1752
|
$parameters:
|
1741
1753
|
type: object
|
1742
1754
|
additionalProperties: true
|
1743
|
-
GroupByKeyMergeStrategy:
|
1744
|
-
title: Group by Key
|
1745
|
-
description: Record merge strategy that combines records according to fields on the record.
|
1746
|
-
required:
|
1747
|
-
- type
|
1748
|
-
- key
|
1749
|
-
properties:
|
1750
|
-
type:
|
1751
|
-
type: string
|
1752
|
-
enum: [GroupByKeyMergeStrategy]
|
1753
|
-
key:
|
1754
|
-
title: Key
|
1755
|
-
description: The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.
|
1756
|
-
anyOf:
|
1757
|
-
- type: string
|
1758
|
-
- type: array
|
1759
|
-
items:
|
1760
|
-
type: string
|
1761
|
-
examples:
|
1762
|
-
- "id"
|
1763
|
-
- ["parent_id", "end_date"]
|
1764
|
-
$parameters:
|
1765
|
-
type: object
|
1766
|
-
additionalProperties: true
|
1767
1755
|
SessionTokenAuthenticator:
|
1768
1756
|
type: object
|
1769
1757
|
required:
|
@@ -1983,9 +1971,7 @@ definitions:
|
|
1983
1971
|
- type: string
|
1984
1972
|
- type: object
|
1985
1973
|
additionalProperties:
|
1986
|
-
|
1987
|
-
- type: string
|
1988
|
-
- $ref": "#/definitions/QueryProperties"
|
1974
|
+
type: string
|
1989
1975
|
interpolation_context:
|
1990
1976
|
- next_page_token
|
1991
1977
|
- stream_interval
|
@@ -3003,98 +2989,6 @@ definitions:
|
|
3003
2989
|
examples:
|
3004
2990
|
- id
|
3005
2991
|
- ["code", "type"]
|
3006
|
-
PropertiesFromEndpoint:
|
3007
|
-
title: Properties from Endpoint
|
3008
|
-
description: Defines the behavior for fetching the list of properties from an API that will be loaded into the requests to extract records.
|
3009
|
-
type: object
|
3010
|
-
required:
|
3011
|
-
- type
|
3012
|
-
- property_field_path
|
3013
|
-
- retriever
|
3014
|
-
properties:
|
3015
|
-
type:
|
3016
|
-
type: string
|
3017
|
-
enum: [PropertiesFromEndpoint]
|
3018
|
-
property_field_path:
|
3019
|
-
description: Describes the path to the field that should be extracted
|
3020
|
-
type: array
|
3021
|
-
items:
|
3022
|
-
type: string
|
3023
|
-
examples:
|
3024
|
-
- ["name"]
|
3025
|
-
interpolation_context:
|
3026
|
-
- config
|
3027
|
-
- parameters
|
3028
|
-
retriever:
|
3029
|
-
description: Requester component that describes how to fetch the properties to query from a remote API endpoint.
|
3030
|
-
anyOf:
|
3031
|
-
- "$ref": "#/definitions/CustomRetriever"
|
3032
|
-
- "$ref": "#/definitions/SimpleRetriever"
|
3033
|
-
$parameters:
|
3034
|
-
type: object
|
3035
|
-
additionalProperties: true
|
3036
|
-
PropertyChunking:
|
3037
|
-
title: Property Chunking
|
3038
|
-
description: For APIs with restrictions on the amount of properties that can be requester per request, property chunking can be applied to make multiple requests with a subset of the properties.
|
3039
|
-
type: object
|
3040
|
-
required:
|
3041
|
-
- type
|
3042
|
-
- property_limit_type
|
3043
|
-
properties:
|
3044
|
-
type:
|
3045
|
-
type: string
|
3046
|
-
enum: [PropertyChunking]
|
3047
|
-
property_limit_type:
|
3048
|
-
title: Property Limit Type
|
3049
|
-
description: The type used to determine the maximum number of properties per chunk
|
3050
|
-
enum:
|
3051
|
-
- characters
|
3052
|
-
- property_count
|
3053
|
-
property_limit:
|
3054
|
-
title: Property Limit
|
3055
|
-
description: The maximum amount of properties that can be retrieved per request according to the limit type.
|
3056
|
-
type: integer
|
3057
|
-
record_merge_strategy:
|
3058
|
-
title: Record Merge Strategy
|
3059
|
-
description: Dictates how to records that require multiple requests to get all properties should be emitted to the destination
|
3060
|
-
anyOf:
|
3061
|
-
- "$ref": "#/definitions/EmitPartialRecordMergeStrategy"
|
3062
|
-
- "$ref": "#/definitions/GroupByKeyMergeStrategy"
|
3063
|
-
$parameters:
|
3064
|
-
type: object
|
3065
|
-
additionalProperties: true
|
3066
|
-
QueryProperties:
|
3067
|
-
title: Query Properties
|
3068
|
-
description: For APIs that require explicit specification of the properties to query for, this component specifies which property fields and how they are supplied to outbound requests.
|
3069
|
-
type: object
|
3070
|
-
required:
|
3071
|
-
- type
|
3072
|
-
- property_list
|
3073
|
-
properties:
|
3074
|
-
type:
|
3075
|
-
type: string
|
3076
|
-
enum: [QueryProperties]
|
3077
|
-
property_list:
|
3078
|
-
title: Property List
|
3079
|
-
description: The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint
|
3080
|
-
anyOf:
|
3081
|
-
- type: array
|
3082
|
-
items:
|
3083
|
-
type: string
|
3084
|
-
- "$ref": "#/definitions/PropertiesFromEndpoint"
|
3085
|
-
always_include_properties:
|
3086
|
-
title: Always Include Properties
|
3087
|
-
description: The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.
|
3088
|
-
type: array
|
3089
|
-
items:
|
3090
|
-
type: string
|
3091
|
-
property_chunking:
|
3092
|
-
title: Property Chunking
|
3093
|
-
description: Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.
|
3094
|
-
"$ref": "#/definitions/PropertyChunking"
|
3095
|
-
$parameters:
|
3096
|
-
type: object
|
3097
|
-
additionalProperties: true
|
3098
2992
|
RecordFilter:
|
3099
2993
|
title: Record Filter
|
3100
2994
|
description: Filter applied on a list of records.
|
@@ -397,6 +397,12 @@ class ManifestDeclarativeSource(DeclarativeSource):
|
|
397
397
|
for dynamic_stream in components_resolver.resolve_components(
|
398
398
|
stream_template_config=stream_template_config
|
399
399
|
):
|
400
|
+
dynamic_stream = {
|
401
|
+
**ManifestComponentTransformer().propagate_types_and_parameters(
|
402
|
+
"", dynamic_stream, {}, use_parent_parameters=True
|
403
|
+
)
|
404
|
+
}
|
405
|
+
|
400
406
|
if "type" not in dynamic_stream:
|
401
407
|
dynamic_stream["type"] = "DeclarativeStream"
|
402
408
|
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
2
|
-
|
3
1
|
# generated by datamodel-codegen:
|
4
2
|
# filename: declarative_component_schema.yaml
|
5
3
|
|
@@ -44,13 +42,15 @@ class BearerAuthenticator(BaseModel):
|
|
44
42
|
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
45
43
|
|
46
44
|
|
47
|
-
class
|
48
|
-
type: Literal["
|
49
|
-
|
50
|
-
...,
|
51
|
-
|
52
|
-
|
53
|
-
|
45
|
+
class DynamicStreamCheckConfig(BaseModel):
|
46
|
+
type: Literal["DynamicStreamCheckConfig"]
|
47
|
+
dynamic_stream_name: str = Field(
|
48
|
+
..., description="The dynamic stream name.", title="Dynamic Stream Name"
|
49
|
+
)
|
50
|
+
stream_count: Optional[int] = Field(
|
51
|
+
0,
|
52
|
+
description="Numbers of the streams to try reading from when running a check operation.",
|
53
|
+
title="Stream Count",
|
54
54
|
)
|
55
55
|
|
56
56
|
|
@@ -345,10 +345,6 @@ class Clamping(BaseModel):
|
|
345
345
|
target_details: Optional[Dict[str, Any]] = None
|
346
346
|
|
347
347
|
|
348
|
-
class EmitPartialRecordMergeStrategy(BaseModel):
|
349
|
-
type: Literal["EmitPartialRecordMergeStrategy"]
|
350
|
-
|
351
|
-
|
352
348
|
class Algorithm(Enum):
|
353
349
|
HS256 = "HS256"
|
354
350
|
HS384 = "HS384"
|
@@ -722,17 +718,6 @@ class ExponentialBackoffStrategy(BaseModel):
|
|
722
718
|
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
723
719
|
|
724
720
|
|
725
|
-
class GroupByKeyMergeStrategy(BaseModel):
|
726
|
-
type: Literal["GroupByKeyMergeStrategy"]
|
727
|
-
key: Union[str, List[str]] = Field(
|
728
|
-
...,
|
729
|
-
description="The name of the field on the record whose value will be used to group properties that were retrieved through multiple API requests.",
|
730
|
-
examples=["id", ["parent_id", "end_date"]],
|
731
|
-
title="Key",
|
732
|
-
)
|
733
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
734
|
-
|
735
|
-
|
736
721
|
class SessionTokenRequestBearerAuthenticator(BaseModel):
|
737
722
|
type: Literal["Bearer"]
|
738
723
|
|
@@ -1204,33 +1189,6 @@ class PrimaryKey(BaseModel):
|
|
1204
1189
|
)
|
1205
1190
|
|
1206
1191
|
|
1207
|
-
class PropertyLimitType(Enum):
|
1208
|
-
characters = "characters"
|
1209
|
-
property_count = "property_count"
|
1210
|
-
|
1211
|
-
|
1212
|
-
class PropertyChunking(BaseModel):
|
1213
|
-
type: Literal["PropertyChunking"]
|
1214
|
-
property_limit_type: PropertyLimitType = Field(
|
1215
|
-
...,
|
1216
|
-
description="The type used to determine the maximum number of properties per chunk",
|
1217
|
-
title="Property Limit Type",
|
1218
|
-
)
|
1219
|
-
property_limit: Optional[int] = Field(
|
1220
|
-
None,
|
1221
|
-
description="The maximum amount of properties that can be retrieved per request according to the limit type.",
|
1222
|
-
title="Property Limit",
|
1223
|
-
)
|
1224
|
-
record_merge_strategy: Optional[
|
1225
|
-
Union[EmitPartialRecordMergeStrategy, GroupByKeyMergeStrategy]
|
1226
|
-
] = Field(
|
1227
|
-
None,
|
1228
|
-
description="Dictates how to records that require multiple requests to get all properties should be emitted to the destination",
|
1229
|
-
title="Record Merge Strategy",
|
1230
|
-
)
|
1231
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
1232
|
-
|
1233
|
-
|
1234
1192
|
class RecordFilter(BaseModel):
|
1235
1193
|
type: Literal["RecordFilter"]
|
1236
1194
|
condition: Optional[str] = Field(
|
@@ -1567,6 +1525,17 @@ class AuthFlow(BaseModel):
|
|
1567
1525
|
oauth_config_specification: Optional[OAuthConfigSpecification] = None
|
1568
1526
|
|
1569
1527
|
|
1528
|
+
class CheckStream(BaseModel):
|
1529
|
+
type: Literal["CheckStream"]
|
1530
|
+
stream_names: Optional[List[str]] = Field(
|
1531
|
+
None,
|
1532
|
+
description="Names of the streams to try reading from when running a check operation.",
|
1533
|
+
examples=[["users"], ["users", "contacts"]],
|
1534
|
+
title="Stream Names",
|
1535
|
+
)
|
1536
|
+
dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
|
1537
|
+
|
1538
|
+
|
1570
1539
|
class IncrementingCountCursor(BaseModel):
|
1571
1540
|
type: Literal["IncrementingCountCursor"]
|
1572
1541
|
cursor_field: str = Field(
|
@@ -2218,7 +2187,7 @@ class HttpRequester(BaseModel):
|
|
2218
2187
|
examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}],
|
2219
2188
|
title="Request Headers",
|
2220
2189
|
)
|
2221
|
-
request_parameters: Optional[Union[str, Dict[str,
|
2190
|
+
request_parameters: Optional[Union[str, Dict[str, str]]] = Field(
|
2222
2191
|
None,
|
2223
2192
|
description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.",
|
2224
2193
|
examples=[
|
@@ -2308,40 +2277,6 @@ class ParentStreamConfig(BaseModel):
|
|
2308
2277
|
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
2309
2278
|
|
2310
2279
|
|
2311
|
-
class PropertiesFromEndpoint(BaseModel):
|
2312
|
-
type: Literal["PropertiesFromEndpoint"]
|
2313
|
-
property_field_path: List[str] = Field(
|
2314
|
-
...,
|
2315
|
-
description="Describes the path to the field that should be extracted",
|
2316
|
-
examples=[["name"]],
|
2317
|
-
)
|
2318
|
-
retriever: Union[CustomRetriever, SimpleRetriever] = Field(
|
2319
|
-
...,
|
2320
|
-
description="Requester component that describes how to fetch the properties to query from a remote API endpoint.",
|
2321
|
-
)
|
2322
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
2323
|
-
|
2324
|
-
|
2325
|
-
class QueryProperties(BaseModel):
|
2326
|
-
type: Literal["QueryProperties"]
|
2327
|
-
property_list: Union[List[str], PropertiesFromEndpoint] = Field(
|
2328
|
-
...,
|
2329
|
-
description="The set of properties that will be queried for in the outbound request. This can either be statically defined or dynamic based on an API endpoint",
|
2330
|
-
title="Property List",
|
2331
|
-
)
|
2332
|
-
always_include_properties: Optional[List[str]] = Field(
|
2333
|
-
None,
|
2334
|
-
description="The list of properties that should be included in every set of properties when multiple chunks of properties are being requested.",
|
2335
|
-
title="Always Include Properties",
|
2336
|
-
)
|
2337
|
-
property_chunking: Optional[PropertyChunking] = Field(
|
2338
|
-
None,
|
2339
|
-
description="Defines how query properties will be grouped into smaller sets for APIs with limitations on the number of properties fetched per API request.",
|
2340
|
-
title="Property Chunking",
|
2341
|
-
)
|
2342
|
-
parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
|
2343
|
-
|
2344
|
-
|
2345
2280
|
class StateDelegatingStream(BaseModel):
|
2346
2281
|
type: Literal["StateDelegatingStream"]
|
2347
2282
|
name: str = Field(..., description="The stream name.", example=["Users"], title="Name")
|
@@ -2590,6 +2525,5 @@ DeclarativeStream.update_forward_refs()
|
|
2590
2525
|
SessionTokenAuthenticator.update_forward_refs()
|
2591
2526
|
DynamicSchemaLoader.update_forward_refs()
|
2592
2527
|
ParentStreamConfig.update_forward_refs()
|
2593
|
-
PropertiesFromEndpoint.update_forward_refs()
|
2594
2528
|
SimpleRetriever.update_forward_refs()
|
2595
2529
|
AsyncRetriever.update_forward_refs()
|