airbyte-cdk 6.45.1__py3-none-any.whl → 6.45.1.post46.dev14423672753__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/connector_builder/connector_builder_handler.py +12 -3
- airbyte_cdk/connector_builder/main.py +3 -3
- airbyte_cdk/connector_builder/test_reader/reader.py +2 -0
- airbyte_cdk/sources/concurrent_source/concurrent_source.py +3 -3
- airbyte_cdk/sources/file_based/file_based_source.py +3 -3
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +1 -1
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +3 -3
- airbyte_cdk/sql/shared/sql_processor.py +8 -9
- airbyte_cdk/test/declarative/__init__.py +6 -0
- airbyte_cdk/test/declarative/models/__init__.py +7 -0
- airbyte_cdk/test/declarative/models/scenario.py +74 -0
- airbyte_cdk/test/declarative/test_suites/__init__.py +25 -0
- airbyte_cdk/test/declarative/test_suites/connector_base.py +223 -0
- airbyte_cdk/test/declarative/test_suites/declarative_sources.py +74 -0
- airbyte_cdk/test/declarative/test_suites/destination_base.py +12 -0
- airbyte_cdk/test/declarative/test_suites/source_base.py +128 -0
- airbyte_cdk/test/declarative/utils/__init__.py +0 -0
- airbyte_cdk/test/declarative/utils/job_runner.py +150 -0
- airbyte_cdk/test/entrypoint_wrapper.py +4 -0
- airbyte_cdk/test/fixtures/__init__.py +0 -0
- airbyte_cdk/test/fixtures/auto.py +14 -0
- airbyte_cdk/test/pytest_config/plugin.py +46 -0
- {airbyte_cdk-6.45.1.dist-info → airbyte_cdk-6.45.1.post46.dev14423672753.dist-info}/METADATA +2 -1
- {airbyte_cdk-6.45.1.dist-info → airbyte_cdk-6.45.1.post46.dev14423672753.dist-info}/RECORD +28 -15
- {airbyte_cdk-6.45.1.dist-info → airbyte_cdk-6.45.1.post46.dev14423672753.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.45.1.dist-info → airbyte_cdk-6.45.1.post46.dev14423672753.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.45.1.dist-info → airbyte_cdk-6.45.1.post46.dev14423672753.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.45.1.dist-info → airbyte_cdk-6.45.1.post46.dev14423672753.dist-info}/entry_points.txt +0 -0
@@ -35,8 +35,10 @@ MAX_RECORDS_KEY = "max_records"
|
|
35
35
|
MAX_STREAMS_KEY = "max_streams"
|
36
36
|
|
37
37
|
|
38
|
-
@dataclass
|
38
|
+
@dataclass(kw_only=True)
|
39
39
|
class TestLimits:
|
40
|
+
__test__: bool = False # Prevent pytest from treating this as a test case, despite its name
|
41
|
+
|
40
42
|
max_records: int = field(default=DEFAULT_MAXIMUM_RECORDS)
|
41
43
|
max_pages_per_slice: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE)
|
42
44
|
max_slices: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_SLICES)
|
@@ -51,7 +53,12 @@ def get_limits(config: Mapping[str, Any]) -> TestLimits:
|
|
51
53
|
max_slices = command_config.get(MAX_SLICES_KEY) or DEFAULT_MAXIMUM_NUMBER_OF_SLICES
|
52
54
|
max_records = command_config.get(MAX_RECORDS_KEY) or DEFAULT_MAXIMUM_RECORDS
|
53
55
|
max_streams = command_config.get(MAX_STREAMS_KEY) or DEFAULT_MAXIMUM_STREAMS
|
54
|
-
return TestLimits(
|
56
|
+
return TestLimits(
|
57
|
+
max_records=max_records,
|
58
|
+
max_pages_per_slice=max_pages_per_slice,
|
59
|
+
max_slices=max_slices,
|
60
|
+
max_streams=max_streams,
|
61
|
+
)
|
55
62
|
|
56
63
|
|
57
64
|
def create_source(config: Mapping[str, Any], limits: TestLimits) -> ManifestDeclarativeSource:
|
@@ -79,7 +86,9 @@ def read_stream(
|
|
79
86
|
) -> AirbyteMessage:
|
80
87
|
try:
|
81
88
|
test_read_handler = TestReader(
|
82
|
-
limits.max_pages_per_slice,
|
89
|
+
max_pages_per_slice=limits.max_pages_per_slice,
|
90
|
+
max_slices=limits.max_slices,
|
91
|
+
max_record_limit=limits.max_records,
|
83
92
|
)
|
84
93
|
# The connector builder only supports a single stream
|
85
94
|
stream_name = configured_catalog.streams[0].stream.name
|
@@ -78,9 +78,9 @@ def handle_connector_builder_request(
|
|
78
78
|
if command == "resolve_manifest":
|
79
79
|
return resolve_manifest(source)
|
80
80
|
elif command == "test_read":
|
81
|
-
assert (
|
82
|
-
|
83
|
-
)
|
81
|
+
assert catalog is not None, (
|
82
|
+
"`test_read` requires a valid `ConfiguredAirbyteCatalog`, got None."
|
83
|
+
)
|
84
84
|
return read_stream(source, config, catalog, state, limits)
|
85
85
|
elif command == "full_resolve_manifest":
|
86
86
|
return full_resolve_manifest(source, limits)
|
@@ -49,9 +49,9 @@ class ConcurrentSource:
|
|
49
49
|
too_many_generator = (
|
50
50
|
not is_single_threaded and initial_number_of_partitions_to_generate >= num_workers
|
51
51
|
)
|
52
|
-
assert (
|
53
|
-
|
54
|
-
)
|
52
|
+
assert not too_many_generator, (
|
53
|
+
"It is required to have more workers than threads generating partitions"
|
54
|
+
)
|
55
55
|
threadpool = ThreadPoolManager(
|
56
56
|
concurrent.futures.ThreadPoolExecutor(
|
57
57
|
max_workers=num_workers, thread_name_prefix="workerpool"
|
@@ -282,9 +282,9 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
|
|
282
282
|
and hasattr(self, "_concurrency_level")
|
283
283
|
and self._concurrency_level is not None
|
284
284
|
):
|
285
|
-
assert (
|
286
|
-
|
287
|
-
)
|
285
|
+
assert state_manager is not None, (
|
286
|
+
"No ConnectorStateManager was created, but it is required for incremental syncs. This is unexpected. Please contact Support."
|
287
|
+
)
|
288
288
|
|
289
289
|
cursor = self.cursor_cls(
|
290
290
|
stream_config,
|
@@ -154,7 +154,7 @@ class AvroParser(FileTypeParser):
|
|
154
154
|
# For example: ^-?\d{1,5}(?:\.\d{1,3})?$ would accept 12345.123 and 123456.12345 would be rejected
|
155
155
|
return {
|
156
156
|
"type": "string",
|
157
|
-
"pattern": f"^-?\\d{{{1,max_whole_number_range}}}(?:\\.\\d{1,decimal_range})?$",
|
157
|
+
"pattern": f"^-?\\d{{{1, max_whole_number_range}}}(?:\\.\\d{1, decimal_range})?$",
|
158
158
|
}
|
159
159
|
elif "logicalType" in avro_field:
|
160
160
|
if avro_field["logicalType"] not in AVRO_LOGICAL_TYPE_TO_JSON:
|
@@ -284,9 +284,9 @@ class FileBasedStreamPartition(Partition):
|
|
284
284
|
def to_slice(self) -> Optional[Mapping[str, Any]]:
|
285
285
|
if self._slice is None:
|
286
286
|
return None
|
287
|
-
assert (
|
288
|
-
len(self._slice[
|
289
|
-
)
|
287
|
+
assert len(self._slice["files"]) == 1, (
|
288
|
+
f"Expected 1 file per partition but got {len(self._slice['files'])} for stream {self.stream_name()}"
|
289
|
+
)
|
290
290
|
file = self._slice["files"][0]
|
291
291
|
return {"files": [file]}
|
292
292
|
|
@@ -326,9 +326,9 @@ class SqlProcessorBase(abc.ABC):
|
|
326
326
|
|
327
327
|
if DEBUG_MODE:
|
328
328
|
found_schemas = schemas_list
|
329
|
-
assert (
|
330
|
-
schema_name
|
331
|
-
)
|
329
|
+
assert schema_name in found_schemas, (
|
330
|
+
f"Schema {schema_name} was not created. Found: {found_schemas}"
|
331
|
+
)
|
332
332
|
|
333
333
|
def _quote_identifier(self, identifier: str) -> str:
|
334
334
|
"""Return the given identifier, quoted."""
|
@@ -617,10 +617,10 @@ class SqlProcessorBase(abc.ABC):
|
|
617
617
|
self._execute_sql(
|
618
618
|
f"""
|
619
619
|
INSERT INTO {self._fully_qualified(final_table_name)} (
|
620
|
-
{f
|
620
|
+
{f",{nl} ".join(columns)}
|
621
621
|
)
|
622
622
|
SELECT
|
623
|
-
{f
|
623
|
+
{f",{nl} ".join(columns)}
|
624
624
|
FROM {self._fully_qualified(temp_table_name)}
|
625
625
|
""",
|
626
626
|
)
|
@@ -645,8 +645,7 @@ class SqlProcessorBase(abc.ABC):
|
|
645
645
|
deletion_name = f"{final_table_name}_deleteme"
|
646
646
|
commands = "\n".join(
|
647
647
|
[
|
648
|
-
f"ALTER TABLE {self._fully_qualified(final_table_name)} RENAME "
|
649
|
-
f"TO {deletion_name};",
|
648
|
+
f"ALTER TABLE {self._fully_qualified(final_table_name)} RENAME TO {deletion_name};",
|
650
649
|
f"ALTER TABLE {self._fully_qualified(temp_table_name)} RENAME "
|
651
650
|
f"TO {final_table_name};",
|
652
651
|
f"DROP TABLE {self._fully_qualified(deletion_name)};",
|
@@ -686,10 +685,10 @@ class SqlProcessorBase(abc.ABC):
|
|
686
685
|
{set_clause}
|
687
686
|
WHEN NOT MATCHED THEN INSERT
|
688
687
|
(
|
689
|
-
{f
|
688
|
+
{f",{nl} ".join(columns)}
|
690
689
|
)
|
691
690
|
VALUES (
|
692
|
-
tmp.{f
|
691
|
+
tmp.{f",{nl} tmp.".join(columns)}
|
693
692
|
);
|
694
693
|
""",
|
695
694
|
)
|
@@ -0,0 +1,74 @@
|
|
1
|
+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
|
+
"""Run acceptance tests in PyTest.
|
3
|
+
|
4
|
+
These tests leverage the same `acceptance-test-config.yml` configuration files as the
|
5
|
+
acceptance tests in CAT, but they run in PyTest instead of CAT. This allows us to run
|
6
|
+
the acceptance tests in the same local environment as we are developing in, speeding
|
7
|
+
up iteration cycles.
|
8
|
+
"""
|
9
|
+
|
10
|
+
from __future__ import annotations
|
11
|
+
|
12
|
+
from pathlib import Path
|
13
|
+
from typing import Any, Literal, cast
|
14
|
+
|
15
|
+
import yaml
|
16
|
+
from pydantic import BaseModel
|
17
|
+
|
18
|
+
|
19
|
+
class ConnectorTestScenario(BaseModel):
|
20
|
+
"""Acceptance test instance, as a Pydantic model.
|
21
|
+
|
22
|
+
This class represents an acceptance test instance, which is a single test case
|
23
|
+
that can be run against a connector. It is used to deserialize and validate the
|
24
|
+
acceptance test configuration file.
|
25
|
+
"""
|
26
|
+
|
27
|
+
class AcceptanceTestExpectRecords(BaseModel):
|
28
|
+
path: Path
|
29
|
+
exact_order: bool = False
|
30
|
+
|
31
|
+
class AcceptanceTestFileTypes(BaseModel):
|
32
|
+
skip_test: bool
|
33
|
+
bypass_reason: str
|
34
|
+
|
35
|
+
config_path: Path | None = None
|
36
|
+
config_dict: dict[str, Any] | None = None
|
37
|
+
|
38
|
+
id: str | None = None
|
39
|
+
|
40
|
+
configured_catalog_path: Path | None = None
|
41
|
+
timeout_seconds: int | None = None
|
42
|
+
expect_records: AcceptanceTestExpectRecords | None = None
|
43
|
+
file_types: AcceptanceTestFileTypes | None = None
|
44
|
+
status: Literal["succeed", "failed"] | None = None
|
45
|
+
|
46
|
+
def get_config_dict(self) -> dict[str, Any]:
|
47
|
+
"""Return the config dictionary.
|
48
|
+
|
49
|
+
If a config dictionary has already been loaded, return it. Otherwise, load
|
50
|
+
the config file and return the dictionary.
|
51
|
+
"""
|
52
|
+
if self.config_dict:
|
53
|
+
return self.config_dict
|
54
|
+
|
55
|
+
if self.config_path:
|
56
|
+
return cast(dict[str, Any], yaml.safe_load(self.config_path.read_text()))
|
57
|
+
|
58
|
+
raise ValueError("No config dictionary or path provided.")
|
59
|
+
|
60
|
+
@property
|
61
|
+
def expect_exception(self) -> bool:
|
62
|
+
return self.status and self.status == "failed" or False
|
63
|
+
|
64
|
+
@property
|
65
|
+
def instance_name(self) -> str:
|
66
|
+
return self.config_path.stem if self.config_path else "Unnamed Scenario"
|
67
|
+
|
68
|
+
def __str__(self) -> str:
|
69
|
+
if self.id:
|
70
|
+
return f"'{self.id}' Test Scenario"
|
71
|
+
if self.config_path:
|
72
|
+
return f"'{self.config_path.name}' Test Scenario"
|
73
|
+
|
74
|
+
return f"'{hash(self)}' Test Scenario"
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
|
+
"""Declarative test suites.
|
3
|
+
|
4
|
+
Here we have base classes for a robust set of declarative connector test suites.
|
5
|
+
"""
|
6
|
+
|
7
|
+
from airbyte_cdk.test.declarative.test_suites.connector_base import (
|
8
|
+
ConnectorTestScenario,
|
9
|
+
ConnectorTestSuiteBase,
|
10
|
+
generate_tests,
|
11
|
+
)
|
12
|
+
from airbyte_cdk.test.declarative.test_suites.declarative_sources import (
|
13
|
+
DeclarativeSourceTestSuite,
|
14
|
+
)
|
15
|
+
from airbyte_cdk.test.declarative.test_suites.destination_base import DestinationTestSuiteBase
|
16
|
+
from airbyte_cdk.test.declarative.test_suites.source_base import SourceTestSuiteBase
|
17
|
+
|
18
|
+
__all__ = [
|
19
|
+
"ConnectorTestScenario",
|
20
|
+
"ConnectorTestSuiteBase",
|
21
|
+
"DeclarativeSourceTestSuite",
|
22
|
+
"DestinationTestSuiteBase",
|
23
|
+
"SourceTestSuiteBase",
|
24
|
+
"generate_tests",
|
25
|
+
]
|
@@ -0,0 +1,223 @@
|
|
1
|
+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
|
+
"""Base class for connector test suites."""
|
3
|
+
|
4
|
+
from __future__ import annotations
|
5
|
+
|
6
|
+
import abc
|
7
|
+
import inspect
|
8
|
+
import sys
|
9
|
+
from collections.abc import Callable
|
10
|
+
from pathlib import Path
|
11
|
+
from typing import Any, Literal, cast
|
12
|
+
|
13
|
+
import pytest
|
14
|
+
import yaml
|
15
|
+
from boltons.typeutils import classproperty
|
16
|
+
|
17
|
+
from airbyte_cdk import Connector
|
18
|
+
from airbyte_cdk.models import (
|
19
|
+
AirbyteMessage,
|
20
|
+
Type,
|
21
|
+
)
|
22
|
+
from airbyte_cdk.test import entrypoint_wrapper
|
23
|
+
from airbyte_cdk.test.declarative.models import (
|
24
|
+
ConnectorTestScenario,
|
25
|
+
)
|
26
|
+
from airbyte_cdk.test.declarative.utils.job_runner import IConnector, run_test_job
|
27
|
+
|
28
|
+
ACCEPTANCE_TEST_CONFIG = "acceptance-test-config.yml"
|
29
|
+
MANIFEST_YAML = "manifest.yaml"
|
30
|
+
|
31
|
+
|
32
|
+
class JavaClass(str):
|
33
|
+
"""A string that represents a Java class."""
|
34
|
+
|
35
|
+
|
36
|
+
class DockerImage(str):
|
37
|
+
"""A string that represents a Docker image."""
|
38
|
+
|
39
|
+
|
40
|
+
class RunnableConnector(abc.ABC):
|
41
|
+
"""A connector that can be run in a test scenario."""
|
42
|
+
|
43
|
+
@abc.abstractmethod
|
44
|
+
def launch(cls, args: list[str] | None) -> None: ...
|
45
|
+
|
46
|
+
|
47
|
+
def generate_tests(metafunc: pytest.Metafunc) -> None:
|
48
|
+
"""
|
49
|
+
A helper for pytest_generate_tests hook.
|
50
|
+
|
51
|
+
If a test method (in a class subclassed from our base class)
|
52
|
+
declares an argument 'instance', this function retrieves the
|
53
|
+
'scenarios' attribute from the test class and parametrizes that
|
54
|
+
test with the values from 'scenarios'.
|
55
|
+
|
56
|
+
## Usage
|
57
|
+
|
58
|
+
```python
|
59
|
+
from airbyte_cdk.test.declarative.test_suites.connector_base import (
|
60
|
+
generate_tests,
|
61
|
+
ConnectorTestSuiteBase,
|
62
|
+
)
|
63
|
+
|
64
|
+
def pytest_generate_tests(metafunc):
|
65
|
+
generate_tests(metafunc)
|
66
|
+
|
67
|
+
class TestMyConnector(ConnectorTestSuiteBase):
|
68
|
+
...
|
69
|
+
|
70
|
+
```
|
71
|
+
"""
|
72
|
+
# Check if the test function requires an 'instance' argument
|
73
|
+
if "instance" in metafunc.fixturenames:
|
74
|
+
# Retrieve the test class
|
75
|
+
test_class = metafunc.cls
|
76
|
+
if test_class is None:
|
77
|
+
raise ValueError("Expected a class here.")
|
78
|
+
# Get the 'scenarios' attribute from the class
|
79
|
+
scenarios_attr = getattr(test_class, "get_scenarios", None)
|
80
|
+
if scenarios_attr is None:
|
81
|
+
raise ValueError(
|
82
|
+
f"Test class {test_class} does not have a 'scenarios' attribute. "
|
83
|
+
"Please define the 'scenarios' attribute in the test class."
|
84
|
+
)
|
85
|
+
|
86
|
+
scenarios = test_class.get_scenarios()
|
87
|
+
ids = [str(scenario) for scenario in scenarios]
|
88
|
+
metafunc.parametrize("instance", scenarios, ids=ids)
|
89
|
+
|
90
|
+
|
91
|
+
class ConnectorTestSuiteBase(abc.ABC):
|
92
|
+
"""Base class for connector test suites."""
|
93
|
+
|
94
|
+
connector: type[IConnector] | Callable[[], IConnector] | None = None
|
95
|
+
"""The connector class or a factory function that returns an instance of IConnector."""
|
96
|
+
|
97
|
+
@classmethod
|
98
|
+
def get_test_class_dir(cls) -> Path:
|
99
|
+
"""Get the file path that contains the class."""
|
100
|
+
module = sys.modules[cls.__module__]
|
101
|
+
# Get the directory containing the test file
|
102
|
+
return Path(inspect.getfile(module)).parent
|
103
|
+
|
104
|
+
@classmethod
|
105
|
+
def create_connector(
|
106
|
+
cls,
|
107
|
+
scenario: ConnectorTestScenario,
|
108
|
+
) -> IConnector:
|
109
|
+
"""Instantiate the connector class."""
|
110
|
+
connector = cls.connector # type: ignore
|
111
|
+
if connector:
|
112
|
+
if callable(connector) or isinstance(connector, type):
|
113
|
+
# If the connector is a class or factory function, instantiate it:
|
114
|
+
return cast(IConnector, connector()) # type: ignore [redundant-cast]
|
115
|
+
|
116
|
+
# Otherwise, we can't instantiate the connector. Fail with a clear error message.
|
117
|
+
raise NotImplementedError(
|
118
|
+
"No connector class or connector factory function provided. "
|
119
|
+
"Please provide a class or factory function in `cls.connector`, or "
|
120
|
+
"override `cls.create_connector()` to define a custom initialization process."
|
121
|
+
)
|
122
|
+
|
123
|
+
def run_test_scenario(
|
124
|
+
self,
|
125
|
+
verb: Literal["read", "check", "discover"],
|
126
|
+
test_scenario: ConnectorTestScenario,
|
127
|
+
*,
|
128
|
+
catalog: dict[str, Any] | None = None,
|
129
|
+
) -> entrypoint_wrapper.EntrypointOutput:
|
130
|
+
"""Run a test job from provided CLI args and return the result."""
|
131
|
+
return run_test_job(
|
132
|
+
self.create_connector(test_scenario),
|
133
|
+
verb,
|
134
|
+
test_instance=test_scenario,
|
135
|
+
catalog=catalog,
|
136
|
+
)
|
137
|
+
|
138
|
+
# Test Definitions
|
139
|
+
|
140
|
+
def test_check(
|
141
|
+
self,
|
142
|
+
instance: ConnectorTestScenario,
|
143
|
+
) -> None:
|
144
|
+
"""Run `connection` acceptance tests."""
|
145
|
+
result = self.run_test_scenario(
|
146
|
+
"check",
|
147
|
+
test_scenario=instance,
|
148
|
+
)
|
149
|
+
conn_status_messages: list[AirbyteMessage] = [
|
150
|
+
msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS
|
151
|
+
] # noqa: SLF001 # Non-public API
|
152
|
+
assert len(conn_status_messages) == 1, (
|
153
|
+
f"Expected exactly one CONNECTION_STATUS message. Got: {result._messages}"
|
154
|
+
)
|
155
|
+
|
156
|
+
@classmethod
|
157
|
+
def get_connector_root_dir(cls) -> Path:
|
158
|
+
"""Get the root directory of the connector."""
|
159
|
+
for parent in cls.get_test_class_dir().parents:
|
160
|
+
if (parent / MANIFEST_YAML).exists():
|
161
|
+
return parent
|
162
|
+
if (parent / ACCEPTANCE_TEST_CONFIG).exists():
|
163
|
+
return parent
|
164
|
+
if parent.name == "airbyte_cdk":
|
165
|
+
break
|
166
|
+
# If we reach here, we didn't find the manifest file in any parent directory
|
167
|
+
# Check if the manifest file exists in the current directory
|
168
|
+
for parent in Path.cwd().parents:
|
169
|
+
if (parent / MANIFEST_YAML).exists():
|
170
|
+
return parent
|
171
|
+
if (parent / ACCEPTANCE_TEST_CONFIG).exists():
|
172
|
+
return parent
|
173
|
+
if parent.name == "airbyte_cdk":
|
174
|
+
break
|
175
|
+
|
176
|
+
raise FileNotFoundError(
|
177
|
+
"Could not find connector root directory relative to "
|
178
|
+
f"'{str(cls.get_test_class_dir())}' or '{str(Path.cwd())}'."
|
179
|
+
)
|
180
|
+
|
181
|
+
@classproperty
|
182
|
+
def acceptance_test_config_path(cls) -> Path:
|
183
|
+
"""Get the path to the acceptance test config file."""
|
184
|
+
result = cls.get_connector_root_dir() / ACCEPTANCE_TEST_CONFIG
|
185
|
+
if result.exists():
|
186
|
+
return result
|
187
|
+
|
188
|
+
raise FileNotFoundError(f"Acceptance test config file not found at: {str(result)}")
|
189
|
+
|
190
|
+
@classmethod
|
191
|
+
def get_scenarios(
|
192
|
+
cls,
|
193
|
+
) -> list[ConnectorTestScenario]:
|
194
|
+
"""Get acceptance tests for a given category.
|
195
|
+
|
196
|
+
This has to be a separate function because pytest does not allow
|
197
|
+
parametrization of fixtures with arguments from the test class itself.
|
198
|
+
"""
|
199
|
+
category = "connection"
|
200
|
+
all_tests_config = yaml.safe_load(cls.acceptance_test_config_path.read_text())
|
201
|
+
if "acceptance_tests" not in all_tests_config:
|
202
|
+
raise ValueError(
|
203
|
+
f"Acceptance tests config not found in {cls.acceptance_test_config_path}."
|
204
|
+
f" Found only: {str(all_tests_config)}."
|
205
|
+
)
|
206
|
+
if category not in all_tests_config["acceptance_tests"]:
|
207
|
+
return []
|
208
|
+
if "tests" not in all_tests_config["acceptance_tests"][category]:
|
209
|
+
raise ValueError(f"No tests found for category {category}")
|
210
|
+
|
211
|
+
tests_scenarios = [
|
212
|
+
ConnectorTestScenario.model_validate(test)
|
213
|
+
for test in all_tests_config["acceptance_tests"][category]["tests"]
|
214
|
+
if "iam_role" not in test["config_path"]
|
215
|
+
]
|
216
|
+
connector_root = cls.get_connector_root_dir().absolute()
|
217
|
+
for test in tests_scenarios:
|
218
|
+
if test.config_path:
|
219
|
+
test.config_path = connector_root / test.config_path
|
220
|
+
if test.configured_catalog_path:
|
221
|
+
test.configured_catalog_path = connector_root / test.configured_catalog_path
|
222
|
+
|
223
|
+
return tests_scenarios
|
@@ -0,0 +1,74 @@
|
|
1
|
+
import os
|
2
|
+
from hashlib import md5
|
3
|
+
from pathlib import Path
|
4
|
+
from typing import Any, cast
|
5
|
+
|
6
|
+
import yaml
|
7
|
+
from boltons.typeutils import classproperty
|
8
|
+
|
9
|
+
from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
|
10
|
+
ConcurrentDeclarativeSource,
|
11
|
+
)
|
12
|
+
from airbyte_cdk.test.declarative.models import ConnectorTestScenario
|
13
|
+
from airbyte_cdk.test.declarative.test_suites.connector_base import MANIFEST_YAML
|
14
|
+
from airbyte_cdk.test.declarative.test_suites.source_base import (
|
15
|
+
SourceTestSuiteBase,
|
16
|
+
)
|
17
|
+
from airbyte_cdk.test.declarative.utils.job_runner import IConnector
|
18
|
+
|
19
|
+
|
20
|
+
def md5_checksum(file_path: Path) -> str:
|
21
|
+
with open(file_path, "rb") as file:
|
22
|
+
return md5(file.read()).hexdigest()
|
23
|
+
|
24
|
+
|
25
|
+
class DeclarativeSourceTestSuite(SourceTestSuiteBase):
|
26
|
+
@classproperty
|
27
|
+
def manifest_yaml_path(cls) -> Path:
|
28
|
+
"""Get the path to the manifest.yaml file."""
|
29
|
+
result = cls.get_connector_root_dir() / MANIFEST_YAML
|
30
|
+
if result.exists():
|
31
|
+
return result
|
32
|
+
|
33
|
+
raise FileNotFoundError(
|
34
|
+
f"Manifest YAML file not found at {result}. "
|
35
|
+
"Please ensure that the test suite is run in the correct directory.",
|
36
|
+
)
|
37
|
+
|
38
|
+
@classproperty
|
39
|
+
def components_py_path(cls) -> Path | None:
|
40
|
+
"""Get the path to the components.py file."""
|
41
|
+
result = cls.get_connector_root_dir() / "components.py"
|
42
|
+
if result.exists():
|
43
|
+
return result
|
44
|
+
|
45
|
+
return None
|
46
|
+
|
47
|
+
@classmethod
|
48
|
+
def create_connector(
|
49
|
+
cls,
|
50
|
+
scenario: ConnectorTestScenario,
|
51
|
+
) -> IConnector:
|
52
|
+
"""Create a connector instance for the test suite."""
|
53
|
+
config: dict[str, Any] = scenario.get_config_dict()
|
54
|
+
# catalog = scenario.get_catalog()
|
55
|
+
# state = scenario.get_state()
|
56
|
+
# source_config = scenario.get_source_config()
|
57
|
+
|
58
|
+
manifest_dict = yaml.safe_load(cls.manifest_yaml_path.read_text())
|
59
|
+
if cls.components_py_path and cls.components_py_path.exists():
|
60
|
+
os.environ["AIRBYTE_ENABLE_UNSAFE_CODE"] = "true"
|
61
|
+
config["__injected_components_py"] = cls.components_py_path.read_text()
|
62
|
+
config["__injected_components_py_checksums"] = {
|
63
|
+
"md5": md5_checksum(cls.components_py_path),
|
64
|
+
}
|
65
|
+
|
66
|
+
return cast(
|
67
|
+
IConnector,
|
68
|
+
ConcurrentDeclarativeSource(
|
69
|
+
config=config,
|
70
|
+
catalog=None,
|
71
|
+
state=None,
|
72
|
+
source_config=manifest_dict,
|
73
|
+
),
|
74
|
+
)
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
|
+
"""Base class for destination test suites."""
|
3
|
+
|
4
|
+
from airbyte_cdk.test.declarative.test_suites.connector_base import ConnectorTestSuiteBase
|
5
|
+
|
6
|
+
|
7
|
+
class DestinationTestSuiteBase(ConnectorTestSuiteBase):
|
8
|
+
"""Base class for destination test suites.
|
9
|
+
|
10
|
+
This class provides a base set of functionality for testing destination connectors, and it
|
11
|
+
inherits all generic connector tests from the `ConnectorTestSuiteBase` class.
|
12
|
+
"""
|
@@ -0,0 +1,128 @@
|
|
1
|
+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
|
+
"""Base class for source test suites."""
|
3
|
+
|
4
|
+
from dataclasses import asdict
|
5
|
+
|
6
|
+
from airbyte_cdk.models import (
|
7
|
+
AirbyteMessage,
|
8
|
+
AirbyteStream,
|
9
|
+
ConfiguredAirbyteCatalog,
|
10
|
+
ConfiguredAirbyteStream,
|
11
|
+
DestinationSyncMode,
|
12
|
+
SyncMode,
|
13
|
+
Type,
|
14
|
+
)
|
15
|
+
from airbyte_cdk.test import entrypoint_wrapper
|
16
|
+
from airbyte_cdk.test.declarative.models import (
|
17
|
+
ConnectorTestScenario,
|
18
|
+
)
|
19
|
+
from airbyte_cdk.test.declarative.test_suites.connector_base import (
|
20
|
+
ConnectorTestSuiteBase,
|
21
|
+
)
|
22
|
+
from airbyte_cdk.test.declarative.utils.job_runner import run_test_job
|
23
|
+
|
24
|
+
|
25
|
+
class SourceTestSuiteBase(ConnectorTestSuiteBase):
|
26
|
+
"""Base class for source test suites.
|
27
|
+
|
28
|
+
This class provides a base set of functionality for testing source connectors, and it
|
29
|
+
inherits all generic connector tests from the `ConnectorTestSuiteBase` class.
|
30
|
+
"""
|
31
|
+
|
32
|
+
def test_check(
|
33
|
+
self,
|
34
|
+
instance: ConnectorTestScenario,
|
35
|
+
) -> None:
|
36
|
+
"""Run `connection` acceptance tests."""
|
37
|
+
result: entrypoint_wrapper.EntrypointOutput = run_test_job(
|
38
|
+
self.create_connector(instance),
|
39
|
+
"check",
|
40
|
+
test_instance=instance,
|
41
|
+
)
|
42
|
+
conn_status_messages: list[AirbyteMessage] = [
|
43
|
+
msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS
|
44
|
+
] # noqa: SLF001 # Non-public API
|
45
|
+
num_status_messages = len(conn_status_messages)
|
46
|
+
assert num_status_messages == 1, (
|
47
|
+
f"Expected exactly one CONNECTION_STATUS message. Got {num_status_messages}: \n"
|
48
|
+
+ "\n".join([str(m) for m in result._messages])
|
49
|
+
)
|
50
|
+
|
51
|
+
def test_basic_read(
|
52
|
+
self,
|
53
|
+
instance: ConnectorTestScenario,
|
54
|
+
) -> None:
|
55
|
+
"""Run acceptance tests."""
|
56
|
+
discover_result = run_test_job(
|
57
|
+
self.create_connector(instance),
|
58
|
+
"discover",
|
59
|
+
test_instance=instance,
|
60
|
+
)
|
61
|
+
if instance.expect_exception:
|
62
|
+
assert discover_result.errors, "Expected exception but got none."
|
63
|
+
return
|
64
|
+
|
65
|
+
configured_catalog = ConfiguredAirbyteCatalog(
|
66
|
+
streams=[
|
67
|
+
ConfiguredAirbyteStream(
|
68
|
+
stream=stream,
|
69
|
+
sync_mode=SyncMode.full_refresh,
|
70
|
+
destination_sync_mode=DestinationSyncMode.append_dedup,
|
71
|
+
)
|
72
|
+
for stream in discover_result.catalog.catalog.streams # type: ignore [reportOptionalMemberAccess, union-attr]
|
73
|
+
]
|
74
|
+
)
|
75
|
+
result = run_test_job(
|
76
|
+
self.create_connector(instance),
|
77
|
+
"read",
|
78
|
+
test_instance=instance,
|
79
|
+
catalog=configured_catalog,
|
80
|
+
)
|
81
|
+
|
82
|
+
if not result.records:
|
83
|
+
raise AssertionError("Expected records but got none.") # noqa: TRY003
|
84
|
+
|
85
|
+
def test_fail_with_bad_catalog(
|
86
|
+
self,
|
87
|
+
instance: ConnectorTestScenario,
|
88
|
+
) -> None:
|
89
|
+
"""Test that a bad catalog fails."""
|
90
|
+
invalid_configured_catalog = ConfiguredAirbyteCatalog(
|
91
|
+
streams=[
|
92
|
+
# Create ConfiguredAirbyteStream which is deliberately invalid
|
93
|
+
# with regard to the Airbyte Protocol.
|
94
|
+
# This should cause the connector to fail.
|
95
|
+
ConfiguredAirbyteStream(
|
96
|
+
stream=AirbyteStream(
|
97
|
+
name="__AIRBYTE__stream_that_does_not_exist",
|
98
|
+
json_schema={
|
99
|
+
"type": "object",
|
100
|
+
"properties": {"f1": {"type": "string"}},
|
101
|
+
},
|
102
|
+
supported_sync_modes=[SyncMode.full_refresh],
|
103
|
+
),
|
104
|
+
sync_mode="INVALID", # type: ignore [reportArgumentType]
|
105
|
+
destination_sync_mode="INVALID", # type: ignore [reportArgumentType]
|
106
|
+
)
|
107
|
+
]
|
108
|
+
)
|
109
|
+
# Set expected status to "failed" to ensure the test fails if the connector.
|
110
|
+
instance.status = "failed"
|
111
|
+
result = self.run_test_scenario(
|
112
|
+
"read",
|
113
|
+
test_scenario=instance,
|
114
|
+
catalog=asdict(invalid_configured_catalog),
|
115
|
+
)
|
116
|
+
assert result.errors, "Expected errors but got none."
|
117
|
+
assert result.trace_messages, "Expected trace messages but got none."
|
118
|
+
|
119
|
+
def test_discover(
|
120
|
+
self,
|
121
|
+
instance: ConnectorTestScenario,
|
122
|
+
) -> None:
|
123
|
+
"""Run acceptance tests."""
|
124
|
+
run_test_job(
|
125
|
+
self.create_connector(instance),
|
126
|
+
"check",
|
127
|
+
test_instance=instance,
|
128
|
+
)
|
File without changes
|
@@ -0,0 +1,150 @@
|
|
1
|
+
import tempfile
|
2
|
+
import uuid
|
3
|
+
from dataclasses import asdict
|
4
|
+
from pathlib import Path
|
5
|
+
from typing import Any, Callable, Literal
|
6
|
+
|
7
|
+
import orjson
|
8
|
+
from typing_extensions import Protocol, runtime_checkable
|
9
|
+
|
10
|
+
from airbyte_cdk.models import (
|
11
|
+
ConfiguredAirbyteCatalog,
|
12
|
+
Status,
|
13
|
+
)
|
14
|
+
from airbyte_cdk.test import entrypoint_wrapper
|
15
|
+
from airbyte_cdk.test.declarative.models import (
|
16
|
+
ConnectorTestScenario,
|
17
|
+
)
|
18
|
+
|
19
|
+
|
20
|
+
@runtime_checkable
|
21
|
+
class IConnector(Protocol):
|
22
|
+
"""A connector that can be run in a test scenario."""
|
23
|
+
|
24
|
+
def launch(self, args: list[str] | None) -> None:
|
25
|
+
"""Launch the connector with the given arguments."""
|
26
|
+
...
|
27
|
+
|
28
|
+
|
29
|
+
def run_test_job(
|
30
|
+
connector: IConnector | type[IConnector] | Callable[[], IConnector],
|
31
|
+
verb: Literal["read", "check", "discover"],
|
32
|
+
test_instance: ConnectorTestScenario,
|
33
|
+
*,
|
34
|
+
catalog: ConfiguredAirbyteCatalog | dict[str, Any] | None = None,
|
35
|
+
) -> entrypoint_wrapper.EntrypointOutput:
|
36
|
+
"""Run a test job from provided CLI args and return the result."""
|
37
|
+
if not connector:
|
38
|
+
raise ValueError("Connector is required")
|
39
|
+
|
40
|
+
if catalog and isinstance(catalog, ConfiguredAirbyteCatalog):
|
41
|
+
# Convert the catalog to a dict if it's already a ConfiguredAirbyteCatalog.
|
42
|
+
catalog = asdict(catalog)
|
43
|
+
|
44
|
+
connector_obj: IConnector
|
45
|
+
if isinstance(connector, type) or callable(connector):
|
46
|
+
# If the connector is a class or a factory lambda, instantiate it.
|
47
|
+
connector_obj = connector()
|
48
|
+
elif (
|
49
|
+
isinstance(
|
50
|
+
connector,
|
51
|
+
IConnector,
|
52
|
+
)
|
53
|
+
or True
|
54
|
+
): # TODO: Get a valid protocol check here
|
55
|
+
connector_obj = connector
|
56
|
+
else:
|
57
|
+
raise ValueError(
|
58
|
+
f"Invalid connector input: {type(connector)}",
|
59
|
+
)
|
60
|
+
|
61
|
+
args: list[str] = [verb]
|
62
|
+
if test_instance.config_path:
|
63
|
+
args += ["--config", str(test_instance.config_path)]
|
64
|
+
elif test_instance.config_dict:
|
65
|
+
config_path = (
|
66
|
+
Path(tempfile.gettempdir()) / "airbyte-test" / f"temp_config_{uuid.uuid4().hex}.json"
|
67
|
+
)
|
68
|
+
config_path.parent.mkdir(parents=True, exist_ok=True)
|
69
|
+
config_path.write_text(orjson.dumps(test_instance.config_dict).decode())
|
70
|
+
args += ["--config", str(config_path)]
|
71
|
+
|
72
|
+
catalog_path: Path | None = None
|
73
|
+
if verb not in ["discover", "check"]:
|
74
|
+
# We need a catalog for read.
|
75
|
+
if catalog:
|
76
|
+
# Write the catalog to a temp json file and pass the path to the file as an argument.
|
77
|
+
catalog_path = (
|
78
|
+
Path(tempfile.gettempdir())
|
79
|
+
/ "airbyte-test"
|
80
|
+
/ f"temp_catalog_{uuid.uuid4().hex}.json"
|
81
|
+
)
|
82
|
+
catalog_path.parent.mkdir(parents=True, exist_ok=True)
|
83
|
+
catalog_path.write_text(orjson.dumps(catalog).decode())
|
84
|
+
elif test_instance.configured_catalog_path:
|
85
|
+
catalog_path = Path(test_instance.configured_catalog_path)
|
86
|
+
|
87
|
+
if catalog_path:
|
88
|
+
args += ["--catalog", str(catalog_path)]
|
89
|
+
|
90
|
+
# This is a bit of a hack because the source needs the catalog early.
|
91
|
+
# Because it *also* can fail, we have ot redundantly wrap it in a try/except block.
|
92
|
+
|
93
|
+
result: entrypoint_wrapper.EntrypointOutput = entrypoint_wrapper._run_command( # noqa: SLF001 # Non-public API
|
94
|
+
source=connector_obj, # type: ignore [arg-type]
|
95
|
+
args=args,
|
96
|
+
expecting_exception=test_instance.expect_exception,
|
97
|
+
)
|
98
|
+
if result.errors and not test_instance.expect_exception:
|
99
|
+
raise AssertionError(
|
100
|
+
"\n\n".join(
|
101
|
+
[str(err.trace.error).replace("\\n", "\n") for err in result.errors if err.trace],
|
102
|
+
)
|
103
|
+
)
|
104
|
+
|
105
|
+
if verb == "check":
|
106
|
+
# Check is expected to fail gracefully without an exception.
|
107
|
+
# Instead, we assert that we have a CONNECTION_STATUS message with
|
108
|
+
# a failure status.
|
109
|
+
assert not result.errors, "Expected no errors from check. Got:\n" + "\n".join(
|
110
|
+
[str(error) for error in result.errors]
|
111
|
+
)
|
112
|
+
assert len(result.connection_status_messages) == 1, (
|
113
|
+
"Expected exactly one CONNECTION_STATUS message. Got "
|
114
|
+
f"{len(result.connection_status_messages)}:\n"
|
115
|
+
+ "\n".join([str(msg) for msg in result.connection_status_messages])
|
116
|
+
)
|
117
|
+
if test_instance.expect_exception:
|
118
|
+
conn_status = result.connection_status_messages[0].connectionStatus
|
119
|
+
assert conn_status, (
|
120
|
+
"Expected CONNECTION_STATUS message to be present. Got: \n"
|
121
|
+
+ "\n".join([str(msg) for msg in result.connection_status_messages])
|
122
|
+
)
|
123
|
+
assert conn_status.status == Status.FAILED, (
|
124
|
+
"Expected CONNECTION_STATUS message to be FAILED. Got: \n"
|
125
|
+
+ "\n".join([str(msg) for msg in result.connection_status_messages])
|
126
|
+
)
|
127
|
+
|
128
|
+
return result
|
129
|
+
|
130
|
+
# For all other verbs, we assert check that an exception is raised (or not).
|
131
|
+
if test_instance.expect_exception:
|
132
|
+
if not result.errors:
|
133
|
+
raise AssertionError("Expected exception but got none.")
|
134
|
+
|
135
|
+
return result
|
136
|
+
if result.errors:
|
137
|
+
raise AssertionError(
|
138
|
+
"\n\n".join(
|
139
|
+
[
|
140
|
+
str(err.trace.error).replace(
|
141
|
+
"\\n",
|
142
|
+
"\n",
|
143
|
+
)
|
144
|
+
for err in result.errors
|
145
|
+
if err.trace
|
146
|
+
],
|
147
|
+
)
|
148
|
+
)
|
149
|
+
|
150
|
+
return result
|
@@ -82,6 +82,10 @@ class EntrypointOutput:
|
|
82
82
|
def state_messages(self) -> List[AirbyteMessage]:
|
83
83
|
return self._get_message_by_types([Type.STATE])
|
84
84
|
|
85
|
+
@property
|
86
|
+
def connection_status_messages(self) -> List[AirbyteMessage]:
|
87
|
+
return self._get_message_by_types([Type.CONNECTION_STATUS])
|
88
|
+
|
85
89
|
@property
|
86
90
|
def most_recent_state(self) -> Any:
|
87
91
|
state_messages = self._get_message_by_types([Type.STATE])
|
File without changes
|
@@ -0,0 +1,14 @@
|
|
1
|
+
"""Auto-use fixtures for pytest.
|
2
|
+
|
3
|
+
WARNING: Importing this module will automatically apply these fixtures. If you want to selectively
|
4
|
+
enable fixtures in a different context, you can import directly from the `fixtures.general` module.
|
5
|
+
|
6
|
+
|
7
|
+
Usage:
|
8
|
+
|
9
|
+
```python
|
10
|
+
from airbyte_cdk.test.fixtures import auto
|
11
|
+
# OR
|
12
|
+
from airbyte_cdk.test.fixtures.auto import *
|
13
|
+
```
|
14
|
+
"""
|
@@ -0,0 +1,46 @@
|
|
1
|
+
"""Global pytest configuration for the Airbyte CDK tests."""
|
2
|
+
|
3
|
+
from pathlib import Path
|
4
|
+
from typing import cast
|
5
|
+
|
6
|
+
import pytest
|
7
|
+
|
8
|
+
|
9
|
+
def pytest_collect_file(parent: pytest.Module | None, path: Path) -> pytest.Module | None:
|
10
|
+
"""Collect test files based on their names."""
|
11
|
+
if path.name == "test_connector.py":
|
12
|
+
return cast(pytest.Module, pytest.Module.from_parent(parent, path=path))
|
13
|
+
|
14
|
+
return None
|
15
|
+
|
16
|
+
|
17
|
+
def pytest_configure(config: pytest.Config) -> None:
|
18
|
+
config.addinivalue_line("markers", "connector: mark test as a connector test")
|
19
|
+
|
20
|
+
|
21
|
+
def pytest_addoption(parser: pytest.Parser) -> None:
|
22
|
+
parser.addoption(
|
23
|
+
"--run-connector",
|
24
|
+
action="store_true",
|
25
|
+
default=False,
|
26
|
+
help="run connector tests",
|
27
|
+
)
|
28
|
+
|
29
|
+
|
30
|
+
def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
|
31
|
+
if config.getoption("--run-connector"):
|
32
|
+
return
|
33
|
+
skip_connector = pytest.mark.skip(reason="need --run-connector option to run")
|
34
|
+
for item in items:
|
35
|
+
if "connector" in item.keywords:
|
36
|
+
item.add_marker(skip_connector)
|
37
|
+
|
38
|
+
|
39
|
+
def pytest_runtest_setup(item: pytest.Item) -> None:
|
40
|
+
# This hook is called before each test function is executed
|
41
|
+
print(f"Setting up test: {item.name}")
|
42
|
+
|
43
|
+
|
44
|
+
def pytest_runtest_teardown(item: pytest.Item, nextitem: pytest.Item | None) -> None:
|
45
|
+
# This hook is called after each test function is executed
|
46
|
+
print(f"Tearing down test: {item.name}")
|
{airbyte_cdk-6.45.1.dist-info → airbyte_cdk-6.45.1.post46.dev14423672753.dist-info}/METADATA
RENAMED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: airbyte-cdk
|
3
|
-
Version: 6.45.1
|
3
|
+
Version: 6.45.1.post46.dev14423672753
|
4
4
|
Summary: A framework for writing Airbyte Connectors.
|
5
5
|
Home-page: https://airbyte.com
|
6
6
|
License: MIT
|
@@ -26,6 +26,7 @@ Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
|
|
26
26
|
Requires-Dist: anyascii (>=0.3.2,<0.4.0)
|
27
27
|
Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
|
28
28
|
Requires-Dist: backoff
|
29
|
+
Requires-Dist: boltons (>=25.0.0,<26.0.0)
|
29
30
|
Requires-Dist: cachetools
|
30
31
|
Requires-Dist: cohere (==4.21) ; extra == "vector-db-based"
|
31
32
|
Requires-Dist: cryptography (>=44.0.0,<45.0.0)
|
@@ -7,13 +7,13 @@ airbyte_cdk/config_observation.py,sha256=7SSPxtN0nXPkm4euGNcTTr1iLbwUL01jy-24V1H
|
|
7
7
|
airbyte_cdk/connector.py,sha256=bO23kdGRkl8XKFytOgrrWFc_VagteTHVEF6IsbizVkM,4224
|
8
8
|
airbyte_cdk/connector_builder/README.md,sha256=Hw3wvVewuHG9-QgsAq1jDiKuLlStDxKBz52ftyNRnBw,1665
|
9
9
|
airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
10
|
-
airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=
|
11
|
-
airbyte_cdk/connector_builder/main.py,sha256=
|
10
|
+
airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=ZQE7FU41TxHMqp40BGeEsO86Vvn1C6VkZRpgWKswgh4,6048
|
11
|
+
airbyte_cdk/connector_builder/main.py,sha256=j1pP5N8RsnvQZ4iYxhLdLEHsJ5Ui7IVFBUi6wYMGBkM,3839
|
12
12
|
airbyte_cdk/connector_builder/models.py,sha256=9pIZ98LW_d6fRS39VdnUOf3cxGt4TkC5MJ0_OrzcCRk,1578
|
13
13
|
airbyte_cdk/connector_builder/test_reader/__init__.py,sha256=iTwBMoI9vaJotEgpqZbFjlxRcbxXYypSVJ9YxeHk7wc,120
|
14
14
|
airbyte_cdk/connector_builder/test_reader/helpers.py,sha256=Iczn-_iczS2CaIAunWwyFcX0uLTra8Wh9JVfzm1Gfxo,26765
|
15
15
|
airbyte_cdk/connector_builder/test_reader/message_grouper.py,sha256=84BAEPIBHMq3WCfO14WNvh_q7OsjGgDt0q1FTu8eW-w,6918
|
16
|
-
airbyte_cdk/connector_builder/test_reader/reader.py,sha256=
|
16
|
+
airbyte_cdk/connector_builder/test_reader/reader.py,sha256=qcyCbAZ0Drk994-3QCjjV6t9f1XcvWmhprxvvhGszds,20765
|
17
17
|
airbyte_cdk/connector_builder/test_reader/types.py,sha256=hPZG3jO03kBaPyW94NI3JHRS1jxXGSNBcN1HFzOxo5Y,2528
|
18
18
|
airbyte_cdk/destinations/__init__.py,sha256=FyDp28PT_YceJD5HDFhA-mrGfX9AONIyMQ4d68CHNxQ,213
|
19
19
|
airbyte_cdk/destinations/destination.py,sha256=CIq-yb8C_0QvcKCtmStaHfiqn53GEfRAIGGCkJhKP1Q,5880
|
@@ -39,7 +39,7 @@ airbyte_cdk/sources/__init__.py,sha256=45J83QsFH3Wky3sVapZWg4C58R_i1thm61M06t2c1
|
|
39
39
|
airbyte_cdk/sources/abstract_source.py,sha256=50vxEBRByiNhT4WJkiFvgM-C6PWqKSJgvuNC_aeg2cw,15547
|
40
40
|
airbyte_cdk/sources/concurrent_source/__init__.py,sha256=3D_RJsxQfiLboSCDdNei1Iv-msRp3DXsas6E9kl7dXc,386
|
41
41
|
airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py,sha256=dbDBNcNNg2IZU5pZb3HfZeILU7X5_EhYGSbNqq3JD4I,12711
|
42
|
-
airbyte_cdk/sources/concurrent_source/concurrent_source.py,sha256=
|
42
|
+
airbyte_cdk/sources/concurrent_source/concurrent_source.py,sha256=P8B6EcLKaSstfAD9kDZsTJ0q8vRmdFrxLt-zOA5_By0,7737
|
43
43
|
airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py,sha256=f9PIRPWn2tXu0-bxVeYHL2vYdqCzZ_kgpHg5_Ep-cfQ,6103
|
44
44
|
airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py,sha256=z1t-rAZBsqVidv2fpUlPHE9JgyXsITuGk4AMu96mXSQ,696
|
45
45
|
airbyte_cdk/sources/concurrent_source/stream_thread_exception.py,sha256=-q6mG2145HKQ28rZGD1bUmjPlIZ1S7-Yhewl8Ntu6xI,764
|
@@ -222,11 +222,11 @@ airbyte_cdk/sources/file_based/discovery_policy/__init__.py,sha256=gl3ey6mZbyfra
|
|
222
222
|
airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py,sha256=dCfXX529Rd5rtopg4VeEgTPJjFtqjtjzPq6LCw18Wt0,605
|
223
223
|
airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py,sha256=-xujTidtrq6HC00WKbjQh1CZdT5LMuzkp5BLjqDmfTY,1007
|
224
224
|
airbyte_cdk/sources/file_based/exceptions.py,sha256=WP0qkG6fpWoBpOyyicgp5YNE393VWyegq5qSy0v4QtM,7362
|
225
|
-
airbyte_cdk/sources/file_based/file_based_source.py,sha256=
|
225
|
+
airbyte_cdk/sources/file_based/file_based_source.py,sha256=Xg8OYWnGc-OcVBglvS08uwAWGWHBhEqsBnyODIkOK-4,20051
|
226
226
|
airbyte_cdk/sources/file_based/file_based_stream_permissions_reader.py,sha256=4e7FXqQ9hueacexC0SyrZyjF8oREYHza8pKF9CgKbD8,5050
|
227
227
|
airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=0cmppYO3pZlFiJrs5oorF4JXv4ErhOeEMrdLG7P-Gdk,6742
|
228
228
|
airbyte_cdk/sources/file_based/file_types/__init__.py,sha256=blCLn0-2LC-ZdgcNyDEhqM2RiUvEjEBh-G4-t32ZtuM,1268
|
229
|
-
airbyte_cdk/sources/file_based/file_types/avro_parser.py,sha256=
|
229
|
+
airbyte_cdk/sources/file_based/file_types/avro_parser.py,sha256=USEYqiICXBWpDV443VtNOCmUA-GINzY_Zah74_5w3qQ,10860
|
230
230
|
airbyte_cdk/sources/file_based/file_types/csv_parser.py,sha256=QlCXB-ry3np67Q_VerQEPoWDOTcPTB6Go4ydZxY9ae4,20445
|
231
231
|
airbyte_cdk/sources/file_based/file_types/excel_parser.py,sha256=BeplCq0hmojELU6bZCvvpRLpQ9us81TqbGYwrhd3INo,7188
|
232
232
|
airbyte_cdk/sources/file_based/file_types/file_transfer.py,sha256=HyGRihJxcb_lEsffKhfF3eylLBDy51_PXSwGUFEJ5bA,1265
|
@@ -242,7 +242,7 @@ airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validat
|
|
242
242
|
airbyte_cdk/sources/file_based/stream/__init__.py,sha256=q_zmeOHHg0JK5j1YNSOIsyXGz-wlTl_0E8z5GKVAcVM,543
|
243
243
|
airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py,sha256=9pQh3BHYcxm8CRC8XawfmBxL8O9HggpWwCCbX_ncINE,7509
|
244
244
|
airbyte_cdk/sources/file_based/stream/concurrent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
245
|
-
airbyte_cdk/sources/file_based/stream/concurrent/adapters.py,sha256=
|
245
|
+
airbyte_cdk/sources/file_based/stream/concurrent/adapters.py,sha256=EwWuoCsQRNaoizxbb2-BYVwRYOxr57exw3q3M6zVv1E,13931
|
246
246
|
airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py,sha256=Rx7TwjH8B7e0eee83Tlqxv1bWn-BVXOmlUAH7auM1uM,344
|
247
247
|
airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py,sha256=5dYZMLBEbvCyrCT89lCYdm2FdrLPLuxjdpQSVGP5o0w,1856
|
248
248
|
airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py,sha256=gRTL-9I3ejjQOpLKd6ixe9rB3kGlubCdhUt9ri6AdAI,14880
|
@@ -332,17 +332,30 @@ airbyte_cdk/sql/exceptions.py,sha256=7_-K2c_trPy6kM89I2pwsrnVEtXqOspd9Eqrzf2KD2A
|
|
332
332
|
airbyte_cdk/sql/secrets.py,sha256=FRIafU5YbWzoK8jtAcfExwzMGdswMbs0OOo1O7Y5i-g,4345
|
333
333
|
airbyte_cdk/sql/shared/__init__.py,sha256=-BU9zpzwx7JxSlS7EmFuGmdB9jK_QhhEJUe5dxErrDw,334
|
334
334
|
airbyte_cdk/sql/shared/catalog_providers.py,sha256=qiahORhtN6qBUGHhSKmzE00uC4i6W8unyBKCj7Kw47s,5218
|
335
|
-
airbyte_cdk/sql/shared/sql_processor.py,sha256=
|
335
|
+
airbyte_cdk/sql/shared/sql_processor.py,sha256=1CwfC3fp9dWnHBpKtly7vGduf9ho_MahiwxGFcULG3Y,27687
|
336
336
|
airbyte_cdk/sql/types.py,sha256=XEIhRAo_ASd0kVLBkdLf5bHiRhNple-IJrC9TibcDdY,5880
|
337
337
|
airbyte_cdk/test/__init__.py,sha256=f_XdkOg4_63QT2k3BbKY34209lppwgw-svzfZstQEq4,199
|
338
338
|
airbyte_cdk/test/catalog_builder.py,sha256=-y05Cz1x0Dlk6oE9LSKhCozssV2gYBNtMdV5YYOPOtk,3015
|
339
|
-
airbyte_cdk/test/
|
339
|
+
airbyte_cdk/test/declarative/__init__.py,sha256=IpPLFCCWnlpp_eTGKK_2WGRFfenbnolxYLmkQSZMO3U,205
|
340
|
+
airbyte_cdk/test/declarative/models/__init__.py,sha256=rXoywbDd-oqEhRQLuaEQIDxykMeawHjzzeN3XIpGQN8,132
|
341
|
+
airbyte_cdk/test/declarative/models/scenario.py,sha256=ySzQosxi2TRaw7JbQMFsuBieoH0ufXJhEY8Vu3DXtx4,2378
|
342
|
+
airbyte_cdk/test/declarative/test_suites/__init__.py,sha256=ELgLwaAQ6Gl0RWW5qZleOGeiNvx4o9ldR7JUbwJ9i28,797
|
343
|
+
airbyte_cdk/test/declarative/test_suites/connector_base.py,sha256=oTQLy9TRP2GRhRlbnw2-4E6O5o4R_QgIJG_V57QpKZY,7892
|
344
|
+
airbyte_cdk/test/declarative/test_suites/declarative_sources.py,sha256=CZW5EDBUZkT10bdTSalFwo-P9ARv8z5V6YsrJdhUzjk,2485
|
345
|
+
airbyte_cdk/test/declarative/test_suites/destination_base.py,sha256=L_l1gpNCkzfx-c7mzS1He5hTVbqR39OnfWdrYMglS7E,486
|
346
|
+
airbyte_cdk/test/declarative/test_suites/source_base.py,sha256=SY6yaJ0qghdlwcw5-U04Ni0Hh2-HuqRsDz24cz6iinQ,4575
|
347
|
+
airbyte_cdk/test/declarative/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
348
|
+
airbyte_cdk/test/declarative/utils/job_runner.py,sha256=oIH5YyUOMav7K-YESn-aLgusSOccDFn67P2vsoaAnrk,5480
|
349
|
+
airbyte_cdk/test/entrypoint_wrapper.py,sha256=TyUmVJyIuGelAv6y8Wy_BnwqIRw_drjfZWKlroljCuQ,9951
|
350
|
+
airbyte_cdk/test/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
351
|
+
airbyte_cdk/test/fixtures/auto.py,sha256=wYKu1pBUJX60Bg-K2RwNXi11Txu1Rncc8WNlOvy9Zug,354
|
340
352
|
airbyte_cdk/test/mock_http/__init__.py,sha256=jE5kC6CQ0OXkTqKhciDnNVZHesBFVIA2YvkdFGwva7k,322
|
341
353
|
airbyte_cdk/test/mock_http/matcher.py,sha256=4Qj8UnJKZIs-eodshryce3SN1Ayc8GZpBETmP6hTEyc,1446
|
342
354
|
airbyte_cdk/test/mock_http/mocker.py,sha256=XgsjMtVoeMpRELPyALgrkHFauH9H5irxrz1Kcxh2yFY,8013
|
343
355
|
airbyte_cdk/test/mock_http/request.py,sha256=tdB8cqk2vLgCDTOKffBKsM06llYs4ZecgtH6DKyx6yY,4112
|
344
356
|
airbyte_cdk/test/mock_http/response.py,sha256=s4-cQQqTtmeej0pQDWqmG0vUWpHS-93lIWMpW3zSVyU,662
|
345
357
|
airbyte_cdk/test/mock_http/response_builder.py,sha256=debPx_lRYBaQVSwCoKLa0F8KFk3h0qG7bWxFBATa0cc,7958
|
358
|
+
airbyte_cdk/test/pytest_config/plugin.py,sha256=iAZEnBKRrnIDqrRRh2NTPlQAKiIU2Z4yczx1tio8FKo,1447
|
346
359
|
airbyte_cdk/test/state_builder.py,sha256=kLPql9lNzUJaBg5YYRLJlY_Hy5JLHJDVyKPMZMoYM44,946
|
347
360
|
airbyte_cdk/test/utils/__init__.py,sha256=Hu-1XT2KDoYjDF7-_ziDwv5bY3PueGjANOCbzeOegDg,57
|
348
361
|
airbyte_cdk/test/utils/data.py,sha256=CkCR1_-rujWNmPXFR1IXTMwx1rAl06wAyIKWpDcN02w,820
|
@@ -366,9 +379,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
|
|
366
379
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
367
380
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
368
381
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
369
|
-
airbyte_cdk-6.45.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
370
|
-
airbyte_cdk-6.45.1.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
371
|
-
airbyte_cdk-6.45.1.dist-info/METADATA,sha256=
|
372
|
-
airbyte_cdk-6.45.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
373
|
-
airbyte_cdk-6.45.1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
|
374
|
-
airbyte_cdk-6.45.1.dist-info/RECORD,,
|
382
|
+
airbyte_cdk-6.45.1.post46.dev14423672753.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
383
|
+
airbyte_cdk-6.45.1.post46.dev14423672753.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
384
|
+
airbyte_cdk-6.45.1.post46.dev14423672753.dist-info/METADATA,sha256=3Kw3TrCB_fLtf4uYs6rAGU66z0jAHIg5YBx7u-6YwZg,6135
|
385
|
+
airbyte_cdk-6.45.1.post46.dev14423672753.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
386
|
+
airbyte_cdk-6.45.1.post46.dev14423672753.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
|
387
|
+
airbyte_cdk-6.45.1.post46.dev14423672753.dist-info/RECORD,,
|
{airbyte_cdk-6.45.1.dist-info → airbyte_cdk-6.45.1.post46.dev14423672753.dist-info}/LICENSE.txt
RENAMED
File without changes
|
{airbyte_cdk-6.45.1.dist-info → airbyte_cdk-6.45.1.post46.dev14423672753.dist-info}/LICENSE_SHORT
RENAMED
File without changes
|
File without changes
|
{airbyte_cdk-6.45.1.dist-info → airbyte_cdk-6.45.1.post46.dev14423672753.dist-info}/entry_points.txt
RENAMED
File without changes
|