airbyte-cdk 6.48.7.dev1__py3-none-any.whl → 6.48.7.post2.dev14899985028__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/cli/airbyte_cdk/_connector.py +3 -3
- airbyte_cdk/destinations/destination.py +50 -78
- airbyte_cdk/models/__init__.py +0 -4
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +12 -2
- airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py +6 -1
- airbyte_cdk/sources/declarative/models/declarative_component_schema.py +10 -0
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +19 -2
- airbyte_cdk/sources/declarative/schema/composite_schema_loader.py +31 -0
- airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +12 -49
- airbyte_cdk/test/catalog_builder.py +1 -9
- airbyte_cdk/test/mock_http/request.py +1 -5
- {airbyte_cdk-6.48.7.dev1.dist-info → airbyte_cdk-6.48.7.post2.dev14899985028.dist-info}/METADATA +2 -2
- {airbyte_cdk-6.48.7.dev1.dist-info → airbyte_cdk-6.48.7.post2.dev14899985028.dist-info}/RECORD +17 -16
- {airbyte_cdk-6.48.7.dev1.dist-info → airbyte_cdk-6.48.7.post2.dev14899985028.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.48.7.dev1.dist-info → airbyte_cdk-6.48.7.post2.dev14899985028.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.48.7.dev1.dist-info → airbyte_cdk-6.48.7.post2.dev14899985028.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.48.7.dev1.dist-info → airbyte_cdk-6.48.7.post2.dev14899985028.dist-info}/entry_points.txt +0 -0
@@ -63,7 +63,7 @@ except ImportError:
|
|
63
63
|
|
64
64
|
TEST_FILE_TEMPLATE = '''
|
65
65
|
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
66
|
-
"""FAST Airbyte Standard Tests for the
|
66
|
+
"""FAST Airbyte Standard Tests for the {connector_name} source."""
|
67
67
|
|
68
68
|
#from airbyte_cdk.test.standard_tests import {base_class_name}
|
69
69
|
from airbyte_cdk.test.standard_tests.util import create_connector_test_suite
|
@@ -78,7 +78,7 @@ TestSuite = create_connector_test_suite(
|
|
78
78
|
)
|
79
79
|
|
80
80
|
# class TestSuite({base_class_name}):
|
81
|
-
# """Test suite for the
|
81
|
+
# """Test suite for the {connector_name} source.
|
82
82
|
|
83
83
|
# This class inherits from SourceTestSuiteBase and implements all of the tests in the suite.
|
84
84
|
|
@@ -152,7 +152,7 @@ def test(
|
|
152
152
|
|
153
153
|
file_text = TEST_FILE_TEMPLATE.format(
|
154
154
|
base_class_name=connector_test_suite.__bases__[0].__name__,
|
155
|
-
|
155
|
+
connector_name=connector_name,
|
156
156
|
)
|
157
157
|
test_file_path = Path() / ".tmp" / "integration_tests/test_airbyte_standards.py"
|
158
158
|
test_file_path = test_file_path.resolve().absolute()
|
@@ -10,18 +10,15 @@ from abc import ABC, abstractmethod
|
|
10
10
|
from typing import Any, Iterable, List, Mapping
|
11
11
|
|
12
12
|
import orjson
|
13
|
-
from airbyte_protocol_dataclasses.models import (
|
14
|
-
AirbyteMessage,
|
15
|
-
ConfiguredAirbyteCatalog,
|
16
|
-
DestinationCatalog,
|
17
|
-
Type,
|
18
|
-
)
|
19
13
|
|
20
14
|
from airbyte_cdk.connector import Connector
|
21
15
|
from airbyte_cdk.exception_handler import init_uncaught_exception_handler
|
22
16
|
from airbyte_cdk.models import (
|
17
|
+
AirbyteMessage,
|
23
18
|
AirbyteMessageSerializer,
|
19
|
+
ConfiguredAirbyteCatalog,
|
24
20
|
ConfiguredAirbyteCatalogSerializer,
|
21
|
+
Type,
|
25
22
|
)
|
26
23
|
from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit
|
27
24
|
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
@@ -29,74 +26,8 @@ from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
|
29
26
|
logger = logging.getLogger("airbyte")
|
30
27
|
|
31
28
|
|
32
|
-
def parse_args(args: List[str]) -> argparse.Namespace:
|
33
|
-
"""
|
34
|
-
:param args: commandline arguments
|
35
|
-
:return:
|
36
|
-
"""
|
37
|
-
|
38
|
-
parent_parser = argparse.ArgumentParser(add_help=False)
|
39
|
-
parent_parser.add_argument(
|
40
|
-
"--debug", action="store_true", help="enables detailed debug logs related to the sync"
|
41
|
-
)
|
42
|
-
main_parser = argparse.ArgumentParser()
|
43
|
-
subparsers = main_parser.add_subparsers(title="commands", dest="command")
|
44
|
-
|
45
|
-
# spec
|
46
|
-
subparsers.add_parser(
|
47
|
-
"spec", help="outputs the json configuration specification", parents=[parent_parser]
|
48
|
-
)
|
49
|
-
|
50
|
-
# check
|
51
|
-
check_parser = subparsers.add_parser(
|
52
|
-
"check", help="checks the config can be used to connect", parents=[parent_parser]
|
53
|
-
)
|
54
|
-
required_check_parser = check_parser.add_argument_group("required named arguments")
|
55
|
-
required_check_parser.add_argument(
|
56
|
-
"--config", type=str, required=True, help="path to the json configuration file"
|
57
|
-
)
|
58
|
-
|
59
|
-
# discover
|
60
|
-
discover_parser = subparsers.add_parser(
|
61
|
-
"discover",
|
62
|
-
help="discover the objects available in the destination",
|
63
|
-
parents=[parent_parser],
|
64
|
-
)
|
65
|
-
required_discover_parser = discover_parser.add_argument_group("required named arguments")
|
66
|
-
required_discover_parser.add_argument(
|
67
|
-
"--config", type=str, required=True, help="path to the json configuration file"
|
68
|
-
)
|
69
|
-
|
70
|
-
# write
|
71
|
-
write_parser = subparsers.add_parser(
|
72
|
-
"write", help="Writes data to the destination", parents=[parent_parser]
|
73
|
-
)
|
74
|
-
write_required = write_parser.add_argument_group("required named arguments")
|
75
|
-
write_required.add_argument(
|
76
|
-
"--config", type=str, required=True, help="path to the JSON configuration file"
|
77
|
-
)
|
78
|
-
write_required.add_argument(
|
79
|
-
"--catalog", type=str, required=True, help="path to the configured catalog JSON file"
|
80
|
-
)
|
81
|
-
|
82
|
-
parsed_args = main_parser.parse_args(args)
|
83
|
-
cmd = parsed_args.command
|
84
|
-
if not cmd:
|
85
|
-
raise Exception("No command entered. ")
|
86
|
-
elif cmd not in ["spec", "check", "discover", "write"]:
|
87
|
-
# This is technically dead code since parse_args() would fail if this was the case
|
88
|
-
# But it's non-obvious enough to warrant placing it here anyways
|
89
|
-
raise Exception(f"Unknown command entered: {cmd}")
|
90
|
-
|
91
|
-
return parsed_args
|
92
|
-
|
93
|
-
|
94
29
|
class Destination(Connector, ABC):
|
95
|
-
VALID_CMDS = {"spec", "check", "
|
96
|
-
|
97
|
-
def discover(self) -> DestinationCatalog:
|
98
|
-
"""Implement to define what objects are available in the destination"""
|
99
|
-
raise NotImplementedError("Discover method is not implemented")
|
30
|
+
VALID_CMDS = {"spec", "check", "write"}
|
100
31
|
|
101
32
|
@abstractmethod
|
102
33
|
def write(
|
@@ -137,9 +68,52 @@ class Destination(Connector, ABC):
|
|
137
68
|
)
|
138
69
|
logger.info("Writing complete.")
|
139
70
|
|
140
|
-
|
141
|
-
|
142
|
-
|
71
|
+
def parse_args(self, args: List[str]) -> argparse.Namespace:
|
72
|
+
"""
|
73
|
+
:param args: commandline arguments
|
74
|
+
:return:
|
75
|
+
"""
|
76
|
+
|
77
|
+
parent_parser = argparse.ArgumentParser(add_help=False)
|
78
|
+
main_parser = argparse.ArgumentParser()
|
79
|
+
subparsers = main_parser.add_subparsers(title="commands", dest="command")
|
80
|
+
|
81
|
+
# spec
|
82
|
+
subparsers.add_parser(
|
83
|
+
"spec", help="outputs the json configuration specification", parents=[parent_parser]
|
84
|
+
)
|
85
|
+
|
86
|
+
# check
|
87
|
+
check_parser = subparsers.add_parser(
|
88
|
+
"check", help="checks the config can be used to connect", parents=[parent_parser]
|
89
|
+
)
|
90
|
+
required_check_parser = check_parser.add_argument_group("required named arguments")
|
91
|
+
required_check_parser.add_argument(
|
92
|
+
"--config", type=str, required=True, help="path to the json configuration file"
|
93
|
+
)
|
94
|
+
|
95
|
+
# write
|
96
|
+
write_parser = subparsers.add_parser(
|
97
|
+
"write", help="Writes data to the destination", parents=[parent_parser]
|
98
|
+
)
|
99
|
+
write_required = write_parser.add_argument_group("required named arguments")
|
100
|
+
write_required.add_argument(
|
101
|
+
"--config", type=str, required=True, help="path to the JSON configuration file"
|
102
|
+
)
|
103
|
+
write_required.add_argument(
|
104
|
+
"--catalog", type=str, required=True, help="path to the configured catalog JSON file"
|
105
|
+
)
|
106
|
+
|
107
|
+
parsed_args = main_parser.parse_args(args)
|
108
|
+
cmd = parsed_args.command
|
109
|
+
if not cmd:
|
110
|
+
raise Exception("No command entered. ")
|
111
|
+
elif cmd not in ["spec", "check", "write"]:
|
112
|
+
# This is technically dead code since parse_args() would fail if this was the case
|
113
|
+
# But it's non-obvious enough to warrant placing it here anyways
|
114
|
+
raise Exception(f"Unknown command entered: {cmd}")
|
115
|
+
|
116
|
+
return parsed_args
|
143
117
|
|
144
118
|
def run_cmd(self, parsed_args: argparse.Namespace) -> Iterable[AirbyteMessage]:
|
145
119
|
cmd = parsed_args.command
|
@@ -163,8 +137,6 @@ class Destination(Connector, ABC):
|
|
163
137
|
|
164
138
|
if cmd == "check":
|
165
139
|
yield self._run_check(config=config)
|
166
|
-
elif cmd == "discover":
|
167
|
-
yield AirbyteMessage(type=Type.DESTINATION_CATALOG, destination_catalog=self.discover())
|
168
140
|
elif cmd == "write":
|
169
141
|
# Wrap in UTF-8 to override any other input encodings
|
170
142
|
wrapped_stdin = io.TextIOWrapper(sys.stdin.buffer, encoding="utf-8")
|
airbyte_cdk/models/__init__.py
CHANGED
@@ -35,10 +35,6 @@ from .airbyte_protocol import (
|
|
35
35
|
ConfiguredAirbyteCatalog,
|
36
36
|
ConfiguredAirbyteStream,
|
37
37
|
ConnectorSpecification,
|
38
|
-
DestinationCatalog,
|
39
|
-
DestinationObject,
|
40
|
-
DestinationObjectProperty,
|
41
|
-
DestinationOperation,
|
42
38
|
DestinationSyncMode,
|
43
39
|
EstimateType,
|
44
40
|
FailureType,
|
@@ -1421,12 +1421,22 @@ definitions:
|
|
1421
1421
|
default: ""
|
1422
1422
|
schema_loader:
|
1423
1423
|
title: Schema Loader
|
1424
|
-
description:
|
1424
|
+
description:
|
1425
|
+
One or many schema loaders can be used to retrieve the schema for the current stream. When
|
1426
|
+
multiple schema loaders are defined, schema properties will be merged together. Schema
|
1427
|
+
loaders defined first taking precedence in the event of a conflict.
|
1425
1428
|
anyOf:
|
1426
1429
|
- "$ref": "#/definitions/InlineSchemaLoader"
|
1427
1430
|
- "$ref": "#/definitions/DynamicSchemaLoader"
|
1428
1431
|
- "$ref": "#/definitions/JsonFileSchemaLoader"
|
1429
1432
|
- "$ref": "#/definitions/CustomSchemaLoader"
|
1433
|
+
- type: array
|
1434
|
+
items:
|
1435
|
+
anyOf:
|
1436
|
+
- "$ref": "#/definitions/InlineSchemaLoader"
|
1437
|
+
- "$ref": "#/definitions/DynamicSchemaLoader"
|
1438
|
+
- "$ref": "#/definitions/JsonFileSchemaLoader"
|
1439
|
+
- "$ref": "#/definitions/CustomSchemaLoader"
|
1430
1440
|
# TODO we have move the transformation to the RecordSelector level in the code but kept this here for
|
1431
1441
|
# compatibility reason. We should eventually move this to align with the code.
|
1432
1442
|
transformations:
|
@@ -4362,4 +4372,4 @@ interpolation:
|
|
4362
4372
|
regex: The regular expression to search for. It must include a capture group.
|
4363
4373
|
return_type: str
|
4364
4374
|
examples:
|
4365
|
-
- '{{ "goodbye, cruel world" | regex_search("goodbye,\s(.*)$") }} -> "cruel world"'
|
4375
|
+
- '{{ "goodbye, cruel world" | regex_search("goodbye,\s(.*)$") }} -> "cruel world"'
|
@@ -4,6 +4,10 @@
|
|
4
4
|
# WHEN DEPRECATED FIELDS ARE ACCESSED
|
5
5
|
|
6
6
|
import warnings
|
7
|
+
|
8
|
+
# ignore the SyntaxWarning in the Airbyte log messages, during the string evaluation
|
9
|
+
warnings.filterwarnings("ignore", category=SyntaxWarning)
|
10
|
+
|
7
11
|
from typing import Any, List
|
8
12
|
|
9
13
|
from pydantic.v1 import BaseModel
|
@@ -12,9 +16,10 @@ from airbyte_cdk.connector_builder.models import LogMessage as ConnectorBuilderL
|
|
12
16
|
|
13
17
|
# format the warning message
|
14
18
|
warnings.formatwarning = (
|
15
|
-
lambda message, category, *args, **kwargs: f"{category.__name__}: {message}"
|
19
|
+
lambda message, category, *args, **kwargs: f"{category.__name__}: {message}\n"
|
16
20
|
)
|
17
21
|
|
22
|
+
|
18
23
|
FIELDS_TAG = "__fields__"
|
19
24
|
DEPRECATED = "deprecated"
|
20
25
|
DEPRECATION_MESSAGE = "deprecation_message"
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
2
|
+
|
1
3
|
# generated by datamodel-codegen:
|
2
4
|
# filename: declarative_component_schema.yaml
|
3
5
|
|
@@ -2151,6 +2153,14 @@ class DeclarativeStream(BaseModel):
|
|
2151
2153
|
DynamicSchemaLoader,
|
2152
2154
|
JsonFileSchemaLoader,
|
2153
2155
|
CustomSchemaLoader,
|
2156
|
+
List[
|
2157
|
+
Union[
|
2158
|
+
InlineSchemaLoader,
|
2159
|
+
DynamicSchemaLoader,
|
2160
|
+
JsonFileSchemaLoader,
|
2161
|
+
CustomSchemaLoader,
|
2162
|
+
]
|
2163
|
+
],
|
2154
2164
|
]
|
2155
2165
|
] = Field(
|
2156
2166
|
None,
|
@@ -504,6 +504,7 @@ from airbyte_cdk.sources.declarative.schema import (
|
|
504
504
|
SchemaTypeIdentifier,
|
505
505
|
TypesMap,
|
506
506
|
)
|
507
|
+
from airbyte_cdk.sources.declarative.schema.composite_schema_loader import CompositeSchemaLoader
|
507
508
|
from airbyte_cdk.sources.declarative.spec import Spec
|
508
509
|
from airbyte_cdk.sources.declarative.stream_slicers import StreamSlicer
|
509
510
|
from airbyte_cdk.sources.declarative.transformations import (
|
@@ -1914,9 +1915,25 @@ class ModelToComponentFactory:
|
|
1914
1915
|
else:
|
1915
1916
|
state_transformations = []
|
1916
1917
|
|
1917
|
-
|
1918
|
+
schema_loader: Union[
|
1919
|
+
CompositeSchemaLoader,
|
1920
|
+
DefaultSchemaLoader,
|
1921
|
+
DynamicSchemaLoader,
|
1922
|
+
InlineSchemaLoader,
|
1923
|
+
JsonFileSchemaLoader,
|
1924
|
+
]
|
1925
|
+
if model.schema_loader and isinstance(model.schema_loader, list):
|
1926
|
+
nested_schema_loaders = [
|
1927
|
+
self._create_component_from_model(model=nested_schema_loader, config=config)
|
1928
|
+
for nested_schema_loader in model.schema_loader
|
1929
|
+
]
|
1930
|
+
schema_loader = CompositeSchemaLoader(
|
1931
|
+
schema_loaders=nested_schema_loaders, parameters={}
|
1932
|
+
)
|
1933
|
+
elif model.schema_loader:
|
1918
1934
|
schema_loader = self._create_component_from_model(
|
1919
|
-
model=model.schema_loader,
|
1935
|
+
model=model.schema_loader, # type: ignore # If defined, schema_loader is guaranteed not to be a list and will be one of the existing base models
|
1936
|
+
config=config,
|
1920
1937
|
)
|
1921
1938
|
else:
|
1922
1939
|
options = model.parameters or {}
|
@@ -0,0 +1,31 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
|
5
|
+
from dataclasses import InitVar, dataclass
|
6
|
+
from typing import Any, Dict, List, Mapping
|
7
|
+
|
8
|
+
from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
|
9
|
+
|
10
|
+
|
11
|
+
@dataclass
|
12
|
+
class CompositeSchemaLoader(SchemaLoader):
|
13
|
+
"""
|
14
|
+
Schema loader that consists of multiple schema loaders that are combined into a single
|
15
|
+
schema. Subsequent schemas do not overwrite existing values so the schema loaders with
|
16
|
+
a higher priority should be defined first.
|
17
|
+
"""
|
18
|
+
|
19
|
+
schema_loaders: List[SchemaLoader]
|
20
|
+
parameters: InitVar[Mapping[str, Any]]
|
21
|
+
|
22
|
+
def get_json_schema(self) -> Mapping[str, Any]:
|
23
|
+
combined_schema: Dict[str, Any] = {
|
24
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
25
|
+
"type": ["null", "object"],
|
26
|
+
"properties": {},
|
27
|
+
}
|
28
|
+
for schema_loader in self.schema_loaders:
|
29
|
+
schema_properties = schema_loader.get_json_schema()["properties"]
|
30
|
+
combined_schema["properties"] = {**schema_properties, **combined_schema["properties"]}
|
31
|
+
return combined_schema
|
@@ -1,10 +1,11 @@
|
|
1
1
|
#
|
2
2
|
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
3
3
|
#
|
4
|
-
|
4
|
+
|
5
|
+
|
5
6
|
from copy import deepcopy
|
6
7
|
from dataclasses import InitVar, dataclass, field
|
7
|
-
from typing import Any,
|
8
|
+
from typing import Any, List, Mapping, MutableMapping, Optional, Union
|
8
9
|
|
9
10
|
import dpath
|
10
11
|
from typing_extensions import deprecated
|
@@ -15,7 +16,7 @@ from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
|
|
15
16
|
from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
|
16
17
|
from airbyte_cdk.sources.declarative.transformations import RecordTransformation
|
17
18
|
from airbyte_cdk.sources.source import ExperimentalClassWarning
|
18
|
-
from airbyte_cdk.sources.types import Config
|
19
|
+
from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
|
19
20
|
|
20
21
|
AIRBYTE_DATA_TYPES: Mapping[str, MutableMapping[str, Any]] = {
|
21
22
|
"string": {"type": ["null", "string"]},
|
@@ -113,38 +114,6 @@ class SchemaTypeIdentifier:
|
|
113
114
|
)
|
114
115
|
|
115
116
|
|
116
|
-
@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
|
117
|
-
class AdditionalPropertyFieldsInferrer(ABC):
|
118
|
-
"""
|
119
|
-
Infers additional fields to be added to each property. For example, if this inferrer returns {"toto": "tata"}, a property that would have looked like this:
|
120
|
-
```
|
121
|
-
"properties": {
|
122
|
-
"Id": {
|
123
|
-
"type": ["null", "string"],
|
124
|
-
},
|
125
|
-
<...>
|
126
|
-
}
|
127
|
-
```
|
128
|
-
... will look like this:
|
129
|
-
```
|
130
|
-
"properties": {
|
131
|
-
"Id": {
|
132
|
-
"type": ["null", "string"],
|
133
|
-
"toto": "tata"
|
134
|
-
},
|
135
|
-
<...>
|
136
|
-
}
|
137
|
-
```
|
138
|
-
"""
|
139
|
-
|
140
|
-
@abstractmethod
|
141
|
-
def infer(self, property_definition: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
|
142
|
-
"""
|
143
|
-
Infers additional property fields from the given property definition.
|
144
|
-
"""
|
145
|
-
pass
|
146
|
-
|
147
|
-
|
148
117
|
@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
|
149
118
|
@dataclass
|
150
119
|
class DynamicSchemaLoader(SchemaLoader):
|
@@ -157,8 +126,6 @@ class DynamicSchemaLoader(SchemaLoader):
|
|
157
126
|
parameters: InitVar[Mapping[str, Any]]
|
158
127
|
schema_type_identifier: SchemaTypeIdentifier
|
159
128
|
schema_transformations: List[RecordTransformation] = field(default_factory=lambda: [])
|
160
|
-
additional_property_fields_inferrer: Optional[AdditionalPropertyFieldsInferrer] = None
|
161
|
-
allow_additional_properties: bool = True
|
162
129
|
|
163
130
|
def get_json_schema(self) -> Mapping[str, Any]:
|
164
131
|
"""
|
@@ -182,26 +149,22 @@ class DynamicSchemaLoader(SchemaLoader):
|
|
182
149
|
property_definition,
|
183
150
|
self.schema_type_identifier.type_pointer,
|
184
151
|
)
|
185
|
-
|
186
|
-
value.update(
|
187
|
-
self.additional_property_fields_inferrer.infer(property_definition)
|
188
|
-
if self.additional_property_fields_inferrer
|
189
|
-
else {}
|
190
|
-
)
|
191
152
|
properties[key] = value
|
192
153
|
|
193
|
-
transformed_properties = self._transform(properties)
|
154
|
+
transformed_properties = self._transform(properties, {})
|
194
155
|
|
195
156
|
return {
|
196
157
|
"$schema": "https://json-schema.org/draft-07/schema#",
|
197
158
|
"type": "object",
|
198
|
-
"additionalProperties":
|
159
|
+
"additionalProperties": True,
|
199
160
|
"properties": transformed_properties,
|
200
161
|
}
|
201
162
|
|
202
163
|
def _transform(
|
203
164
|
self,
|
204
165
|
properties: Mapping[str, Any],
|
166
|
+
stream_state: StreamState,
|
167
|
+
stream_slice: Optional[StreamSlice] = None,
|
205
168
|
) -> Mapping[str, Any]:
|
206
169
|
for transformation in self.schema_transformations:
|
207
170
|
transformation.transform(
|
@@ -227,7 +190,7 @@ class DynamicSchemaLoader(SchemaLoader):
|
|
227
190
|
self,
|
228
191
|
raw_schema: MutableMapping[str, Any],
|
229
192
|
field_type_path: Optional[List[Union[InterpolatedString, str]]],
|
230
|
-
) ->
|
193
|
+
) -> Union[Mapping[str, Any], List[Mapping[str, Any]]]:
|
231
194
|
"""
|
232
195
|
Determines the JSON Schema type for a field, supporting nullable and combined types.
|
233
196
|
"""
|
@@ -257,7 +220,7 @@ class DynamicSchemaLoader(SchemaLoader):
|
|
257
220
|
f"Invalid data type. Available string or two items list of string. Got {mapped_field_type}."
|
258
221
|
)
|
259
222
|
|
260
|
-
def _resolve_complex_type(self, complex_type: ComplexFieldType) ->
|
223
|
+
def _resolve_complex_type(self, complex_type: ComplexFieldType) -> Mapping[str, Any]:
|
261
224
|
if not complex_type.items:
|
262
225
|
return self._get_airbyte_type(complex_type.field_type)
|
263
226
|
|
@@ -292,14 +255,14 @@ class DynamicSchemaLoader(SchemaLoader):
|
|
292
255
|
return field_type
|
293
256
|
|
294
257
|
@staticmethod
|
295
|
-
def _get_airbyte_type(field_type: str) ->
|
258
|
+
def _get_airbyte_type(field_type: str) -> MutableMapping[str, Any]:
|
296
259
|
"""
|
297
260
|
Maps a field type to its corresponding Airbyte type definition.
|
298
261
|
"""
|
299
262
|
if field_type not in AIRBYTE_DATA_TYPES:
|
300
263
|
raise ValueError(f"Invalid Airbyte data type: {field_type}")
|
301
264
|
|
302
|
-
return deepcopy(AIRBYTE_DATA_TYPES[field_type])
|
265
|
+
return deepcopy(AIRBYTE_DATA_TYPES[field_type])
|
303
266
|
|
304
267
|
def _extract_data(
|
305
268
|
self,
|
@@ -2,8 +2,6 @@
|
|
2
2
|
|
3
3
|
from typing import Any, Dict, List, Union, overload
|
4
4
|
|
5
|
-
from airbyte_protocol_dataclasses.models import DestinationSyncMode
|
6
|
-
|
7
5
|
from airbyte_cdk.models import (
|
8
6
|
ConfiguredAirbyteCatalog,
|
9
7
|
ConfiguredAirbyteStream,
|
@@ -34,12 +32,6 @@ class ConfiguredAirbyteStreamBuilder:
|
|
34
32
|
self._stream["sync_mode"] = sync_mode.name
|
35
33
|
return self
|
36
34
|
|
37
|
-
def with_destination_sync_mode(
|
38
|
-
self, sync_mode: DestinationSyncMode
|
39
|
-
) -> "ConfiguredAirbyteStreamBuilder":
|
40
|
-
self._stream["destination_sync_mode"] = sync_mode.name
|
41
|
-
return self
|
42
|
-
|
43
35
|
def with_primary_key(self, pk: List[List[str]]) -> "ConfiguredAirbyteStreamBuilder":
|
44
36
|
self._stream["primary_key"] = pk
|
45
37
|
self._stream["stream"]["source_defined_primary_key"] = pk # type: ignore # we assume that self._stream["stream"] is a Dict[str, Any]
|
@@ -66,7 +58,7 @@ class CatalogBuilder:
|
|
66
58
|
def with_stream(
|
67
59
|
self,
|
68
60
|
name: Union[str, ConfiguredAirbyteStreamBuilder],
|
69
|
-
sync_mode: SyncMode =
|
61
|
+
sync_mode: Union[SyncMode, None] = None,
|
70
62
|
) -> "CatalogBuilder":
|
71
63
|
# As we are introducing a fully fledge ConfiguredAirbyteStreamBuilder, we would like to deprecate the previous interface
|
72
64
|
# with_stream(str, SyncMode)
|
@@ -72,11 +72,7 @@ class HttpRequest:
|
|
72
72
|
elif isinstance(body, bytes):
|
73
73
|
return json.loads(body.decode()) # type: ignore # assumes return type of Mapping[str, Any]
|
74
74
|
elif isinstance(body, str):
|
75
|
-
|
76
|
-
return json.loads(body) # type: ignore # assumes return type of Mapping[str, Any]
|
77
|
-
except json.JSONDecodeError:
|
78
|
-
# one of the body is a mapping while the other isn't so comparison should fail anyway
|
79
|
-
return None
|
75
|
+
return json.loads(body) # type: ignore # assumes return type of Mapping[str, Any]
|
80
76
|
return None
|
81
77
|
|
82
78
|
@staticmethod
|
{airbyte_cdk-6.48.7.dev1.dist-info → airbyte_cdk-6.48.7.post2.dev14899985028.dist-info}/METADATA
RENAMED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: airbyte-cdk
|
3
|
-
Version: 6.48.7.
|
3
|
+
Version: 6.48.7.post2.dev14899985028
|
4
4
|
Summary: A framework for writing Airbyte Connectors.
|
5
5
|
Home-page: https://airbyte.com
|
6
6
|
License: MIT
|
@@ -23,7 +23,7 @@ Provides-Extra: sql
|
|
23
23
|
Provides-Extra: vector-db-based
|
24
24
|
Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
|
25
25
|
Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
|
26
|
-
Requires-Dist: airbyte-protocol-models-dataclasses (
|
26
|
+
Requires-Dist: airbyte-protocol-models-dataclasses (>=0.15,<0.16)
|
27
27
|
Requires-Dist: anyascii (>=0.3.2,<0.4.0)
|
28
28
|
Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
|
29
29
|
Requires-Dist: backoff
|
{airbyte_cdk-6.48.7.dev1.dist-info → airbyte_cdk-6.48.7.post2.dev14899985028.dist-info}/RECORD
RENAMED
@@ -1,7 +1,7 @@
|
|
1
1
|
airbyte_cdk/__init__.py,sha256=52uncJvDQNHvwKxaqzXgnMYTptIl65LDJr2fvlk8-DU,11707
|
2
2
|
airbyte_cdk/cli/__init__.py,sha256=CXsai3MYMLZ_sqi2vPAIVcKDun8VRqlv0cKffBI0iSY,346
|
3
3
|
airbyte_cdk/cli/airbyte_cdk/__init__.py,sha256=8IoEcbdYr7CMAh97Xut5__uHH9vV4LKUtSBNTk3qEWY,2031
|
4
|
-
airbyte_cdk/cli/airbyte_cdk/_connector.py,sha256=
|
4
|
+
airbyte_cdk/cli/airbyte_cdk/_connector.py,sha256=wXwt7tbO-9qFYDDK4P8wbKJX43kbeoRZvK5SNX-Wnfs,5300
|
5
5
|
airbyte_cdk/cli/airbyte_cdk/_image.py,sha256=F0XtvR2CyFi1EPIUBiEnDia9NfCuM7T_itdNj9yyb2E,2907
|
6
6
|
airbyte_cdk/cli/airbyte_cdk/_manifest.py,sha256=aFdeeWgek7oXR3YfZPxk7kBZ64Blmsr0dAXN6BVGiIA,482
|
7
7
|
airbyte_cdk/cli/airbyte_cdk/_secrets.py,sha256=iRA8435sYDcWe6IBv4VAo-3yTIAFqySbDvmUsgpEInA,14712
|
@@ -23,7 +23,7 @@ airbyte_cdk/connector_builder/test_reader/message_grouper.py,sha256=84BAEPIBHMq3
|
|
23
23
|
airbyte_cdk/connector_builder/test_reader/reader.py,sha256=mP1yHK5vG38KxoKoT2QQ7ZNbkdLA1rMAU3EKpucjHls,21098
|
24
24
|
airbyte_cdk/connector_builder/test_reader/types.py,sha256=hPZG3jO03kBaPyW94NI3JHRS1jxXGSNBcN1HFzOxo5Y,2528
|
25
25
|
airbyte_cdk/destinations/__init__.py,sha256=FyDp28PT_YceJD5HDFhA-mrGfX9AONIyMQ4d68CHNxQ,213
|
26
|
-
airbyte_cdk/destinations/destination.py,sha256=
|
26
|
+
airbyte_cdk/destinations/destination.py,sha256=CIq-yb8C_0QvcKCtmStaHfiqn53GEfRAIGGCkJhKP1Q,5880
|
27
27
|
airbyte_cdk/destinations/vector_db_based/README.md,sha256=QAe8c_1Afme4r2TCE10cTSaxUE3zgCBuArSuRQqK8tA,2115
|
28
28
|
airbyte_cdk/destinations/vector_db_based/__init__.py,sha256=eAkzwTjBbXBhJ5GfPO5I53Zgpv5xQFLRQS8n4nuyPt0,1006
|
29
29
|
airbyte_cdk/destinations/vector_db_based/config.py,sha256=1u87eibIWLZ_wuaCvE3yp5ayguM9dGhGXbT8agmkUBg,12468
|
@@ -47,7 +47,7 @@ airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_json_data
|
|
47
47
|
airbyte_cdk/manifest_migrations/migrations/http_requester_url_base_to_url.py,sha256=EX1MVYVpoWypA28qoH48wA0SYZjGdlR8bcSixTDzfgo,1346
|
48
48
|
airbyte_cdk/manifest_migrations/migrations/registry.yaml,sha256=K5KBQ2C1T_dWExEJFuEAe1VO_QqOijOCh90rnUOCEyc,960
|
49
49
|
airbyte_cdk/manifest_migrations/migrations_registry.py,sha256=zly2fwaOxDukqC7eowzrDlvhA2v71FjW74kDzvRXhSY,2619
|
50
|
-
airbyte_cdk/models/__init__.py,sha256=
|
50
|
+
airbyte_cdk/models/__init__.py,sha256=Et9wJWs5VOWynGbb-3aJRhsdAHAiLkNNLxdwqJAuqkw,2114
|
51
51
|
airbyte_cdk/models/airbyte_protocol.py,sha256=oZdKsZ7yPjUt9hvxdWNpxCtgjSV2RWhf4R9Np03sqyY,3613
|
52
52
|
airbyte_cdk/models/airbyte_protocol_serializers.py,sha256=s6SaFB2CMrG_7jTQGn_fhFbQ1FUxhCxf5kq2RWGHMVI,1749
|
53
53
|
airbyte_cdk/models/connector_metadata.py,sha256=Lwb0JKiWvnqHduXjHHgBBBhTsGoLiNjxbG93W_OjcKQ,2875
|
@@ -89,7 +89,7 @@ airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=GoZJ8Oxb
|
|
89
89
|
airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
90
90
|
airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
|
91
91
|
airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
|
92
|
-
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=
|
92
|
+
airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=7tB0-d4lbI97GB5lBI6-qezxLoKZN4QeN_4Vht9TDFo,167270
|
93
93
|
airbyte_cdk/sources/declarative/declarative_source.py,sha256=qmyMnnet92eGc3C22yBtpvD5UZjqdhsAafP_zxI5wp8,1814
|
94
94
|
airbyte_cdk/sources/declarative/declarative_stream.py,sha256=dCRlddBUSaJmBNBz1pSO1r2rTw8AP5d2_vlmIeGs2gg,10767
|
95
95
|
airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
|
@@ -132,15 +132,15 @@ airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW
|
|
132
132
|
airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
|
133
133
|
airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
|
134
134
|
airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
|
135
|
-
airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py,sha256=
|
136
|
-
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=
|
135
|
+
airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py,sha256=Imnj3yef0aqRdLfaUxkIYISUb8YkiPrRH_wBd-x8HjM,5999
|
136
|
+
airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=FjZ2WrItKCqMVxc14XRbMPu11Jhl1Ds3Lugy_2wPoQ8,118235
|
137
137
|
airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
|
138
138
|
airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=nlVvHC511NUyDEEIRBkoeDTAvLqKNp-hRy8D19z8tdk,5941
|
139
139
|
airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=wnRUP0Xeru9Rbu5OexXSDN9QWDo8YU4tT9M2LDVOgGA,802
|
140
140
|
airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=RUyFZS0zslLb7UfQrvqMC--k5CVLNSp7zHw6kbosvKE,9688
|
141
141
|
airbyte_cdk/sources/declarative/parsers/manifest_normalizer.py,sha256=laBy7ebjA-PiNwc-50U4FHvMqS_mmHvnabxgFs4CjGw,17069
|
142
142
|
airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=pJmg78vqE5VfUrF_KJnWjucQ4k9IWFULeAxHCowrHXE,6806
|
143
|
-
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=
|
143
|
+
airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=88ko64Gdfipyg9Uegz-XVCErWms0x5nHNZHCO7CRhVA,167112
|
144
144
|
airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
|
145
145
|
airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
|
146
146
|
airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
|
@@ -209,8 +209,9 @@ airbyte_cdk/sources/declarative/retrievers/file_uploader/noop_file_writer.py,sha
|
|
209
209
|
airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
|
210
210
|
airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=O7qpM71L1_ATIbEKa8y658jdiSJSPw0KmuGKgnaruQU,31008
|
211
211
|
airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
|
212
|
+
airbyte_cdk/sources/declarative/schema/composite_schema_loader.py,sha256=ymGbvxS_QyGc4nnjEyRo5ch8bVedELO41PAUxKXZyMw,1113
|
212
213
|
airbyte_cdk/sources/declarative/schema/default_schema_loader.py,sha256=UnbzlExmwoQiVV8zDg4lhAEaqA_0pRfwbMRe8yqOuWk,1834
|
213
|
-
airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=
|
214
|
+
airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=J8Q_iJYhcSQLWyt0bTZCbDAGpxt9G8FCc6Q9jtGsNzw,10703
|
214
215
|
airbyte_cdk/sources/declarative/schema/inline_schema_loader.py,sha256=bVETE10hRsatRJq3R3BeyRR0wIoK3gcP1gcpVRQ_P5U,464
|
215
216
|
airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py,sha256=5Wl-fqW-pVf_dxJ4yGHMAFfC4JjKHYJhqFJT1xA57F4,4177
|
216
217
|
airbyte_cdk/sources/declarative/schema/schema_loader.py,sha256=kjt8v0N5wWKA5zyLnrDLxf1PJKdUqvQq2RVnAOAzNSY,379
|
@@ -364,12 +365,12 @@ airbyte_cdk/sql/shared/catalog_providers.py,sha256=qiahORhtN6qBUGHhSKmzE00uC4i6W
|
|
364
365
|
airbyte_cdk/sql/shared/sql_processor.py,sha256=1CwfC3fp9dWnHBpKtly7vGduf9ho_MahiwxGFcULG3Y,27687
|
365
366
|
airbyte_cdk/sql/types.py,sha256=XEIhRAo_ASd0kVLBkdLf5bHiRhNple-IJrC9TibcDdY,5880
|
366
367
|
airbyte_cdk/test/__init__.py,sha256=f_XdkOg4_63QT2k3BbKY34209lppwgw-svzfZstQEq4,199
|
367
|
-
airbyte_cdk/test/catalog_builder.py,sha256
|
368
|
+
airbyte_cdk/test/catalog_builder.py,sha256=-y05Cz1x0Dlk6oE9LSKhCozssV2gYBNtMdV5YYOPOtk,3015
|
368
369
|
airbyte_cdk/test/entrypoint_wrapper.py,sha256=TyUmVJyIuGelAv6y8Wy_BnwqIRw_drjfZWKlroljCuQ,9951
|
369
370
|
airbyte_cdk/test/mock_http/__init__.py,sha256=jE5kC6CQ0OXkTqKhciDnNVZHesBFVIA2YvkdFGwva7k,322
|
370
371
|
airbyte_cdk/test/mock_http/matcher.py,sha256=4Qj8UnJKZIs-eodshryce3SN1Ayc8GZpBETmP6hTEyc,1446
|
371
372
|
airbyte_cdk/test/mock_http/mocker.py,sha256=XgsjMtVoeMpRELPyALgrkHFauH9H5irxrz1Kcxh2yFY,8013
|
372
|
-
airbyte_cdk/test/mock_http/request.py,sha256=
|
373
|
+
airbyte_cdk/test/mock_http/request.py,sha256=tdB8cqk2vLgCDTOKffBKsM06llYs4ZecgtH6DKyx6yY,4112
|
373
374
|
airbyte_cdk/test/mock_http/response.py,sha256=s4-cQQqTtmeej0pQDWqmG0vUWpHS-93lIWMpW3zSVyU,662
|
374
375
|
airbyte_cdk/test/mock_http/response_builder.py,sha256=F-v7ebftqGj7YVIMLKdodmU9U8Dq8aIyllWGo2NGwHc,8331
|
375
376
|
airbyte_cdk/test/standard_tests/__init__.py,sha256=YS2bghoGmQ-4GNIbe6RuEmvV-V1kpM1OyxTpebrs0Ig,1338
|
@@ -408,9 +409,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
|
|
408
409
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
409
410
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
410
411
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
411
|
-
airbyte_cdk-6.48.7.
|
412
|
-
airbyte_cdk-6.48.7.
|
413
|
-
airbyte_cdk-6.48.7.
|
414
|
-
airbyte_cdk-6.48.7.
|
415
|
-
airbyte_cdk-6.48.7.
|
416
|
-
airbyte_cdk-6.48.7.
|
412
|
+
airbyte_cdk-6.48.7.post2.dev14899985028.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
413
|
+
airbyte_cdk-6.48.7.post2.dev14899985028.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
414
|
+
airbyte_cdk-6.48.7.post2.dev14899985028.dist-info/METADATA,sha256=g78XypWk60LikFsBy2VCCPA6OIMx1UC-pqGoknnOUuc,6364
|
415
|
+
airbyte_cdk-6.48.7.post2.dev14899985028.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
416
|
+
airbyte_cdk-6.48.7.post2.dev14899985028.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
|
417
|
+
airbyte_cdk-6.48.7.post2.dev14899985028.dist-info/RECORD,,
|
{airbyte_cdk-6.48.7.dev1.dist-info → airbyte_cdk-6.48.7.post2.dev14899985028.dist-info}/LICENSE.txt
RENAMED
File without changes
|
File without changes
|
{airbyte_cdk-6.48.7.dev1.dist-info → airbyte_cdk-6.48.7.post2.dev14899985028.dist-info}/WHEEL
RENAMED
File without changes
|
File without changes
|