airbyte-cdk 6.45.0.post23.dev14384860031__py3-none-any.whl → 6.45.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/sources/declarative/interpolation/filters.py +49 -2
- airbyte_cdk/test/entrypoint_wrapper.py +0 -4
- {airbyte_cdk-6.45.0.post23.dev14384860031.dist-info → airbyte_cdk-6.45.1.dist-info}/METADATA +1 -1
- {airbyte_cdk-6.45.0.post23.dev14384860031.dist-info → airbyte_cdk-6.45.1.dist-info}/RECORD +8 -21
- airbyte_cdk/test/declarative/__init__.py +0 -6
- airbyte_cdk/test/declarative/models/__init__.py +0 -7
- airbyte_cdk/test/declarative/models/scenario.py +0 -74
- airbyte_cdk/test/declarative/test_suites/__init__.py +0 -24
- airbyte_cdk/test/declarative/test_suites/connector_base.py +0 -202
- airbyte_cdk/test/declarative/test_suites/declarative_sources.py +0 -48
- airbyte_cdk/test/declarative/test_suites/destination_base.py +0 -12
- airbyte_cdk/test/declarative/test_suites/source_base.py +0 -129
- airbyte_cdk/test/declarative/utils/__init__.py +0 -0
- airbyte_cdk/test/declarative/utils/job_runner.py +0 -128
- airbyte_cdk/test/fixtures/__init__.py +0 -0
- airbyte_cdk/test/fixtures/auto.py +0 -14
- airbyte_cdk/test/pytest_config/plugin.py +0 -46
- {airbyte_cdk-6.45.0.post23.dev14384860031.dist-info → airbyte_cdk-6.45.1.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.45.0.post23.dev14384860031.dist-info → airbyte_cdk-6.45.1.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.45.0.post23.dev14384860031.dist-info → airbyte_cdk-6.45.1.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.45.0.post23.dev14384860031.dist-info → airbyte_cdk-6.45.1.dist-info}/entry_points.txt +0 -0
@@ -4,9 +4,10 @@
|
|
4
4
|
|
5
5
|
import base64
|
6
6
|
import hashlib
|
7
|
+
import hmac as hmac_lib
|
7
8
|
import json
|
8
9
|
import re
|
9
|
-
from typing import Any, Optional
|
10
|
+
from typing import Any, Dict, Optional
|
10
11
|
|
11
12
|
|
12
13
|
def hash(value: Any, hash_type: str = "md5", salt: Optional[str] = None) -> str:
|
@@ -135,5 +136,51 @@ def regex_search(value: str, regex: str) -> str:
|
|
135
136
|
return ""
|
136
137
|
|
137
138
|
|
138
|
-
|
139
|
+
def hmac(value: Any, key: str, hash_type: str = "sha256") -> str:
|
140
|
+
"""
|
141
|
+
Implementation of a custom Jinja2 hmac filter with SHA-256 support.
|
142
|
+
|
143
|
+
This filter creates a Hash-based Message Authentication Code (HMAC) using a cryptographic
|
144
|
+
hash function and a secret key. Currently only supports SHA-256, and returns hexdigest of the signature.
|
145
|
+
|
146
|
+
Example usage in a low code connector:
|
147
|
+
|
148
|
+
auth_headers:
|
149
|
+
$ref: "#/definitions/base_auth"
|
150
|
+
$parameters:
|
151
|
+
signature: "{{ 'message_to_sign' | hmac('my_secret_key') }}"
|
152
|
+
|
153
|
+
:param value: The message to be authenticated
|
154
|
+
:param key: The secret key for the HMAC
|
155
|
+
:param hash_type: Hash algorithm to use (default: sha256)
|
156
|
+
:return: HMAC digest as a hexadecimal string
|
157
|
+
"""
|
158
|
+
# Define allowed hash functions
|
159
|
+
ALLOWED_HASH_TYPES: Dict[str, Any] = {
|
160
|
+
"sha256": hashlib.sha256,
|
161
|
+
}
|
162
|
+
|
163
|
+
if hash_type not in ALLOWED_HASH_TYPES:
|
164
|
+
raise ValueError(
|
165
|
+
f"Hash type '{hash_type}' is not allowed. Allowed types: {', '.join(ALLOWED_HASH_TYPES.keys())}"
|
166
|
+
)
|
167
|
+
|
168
|
+
hmac_obj = hmac_lib.new(
|
169
|
+
key=str(key).encode("utf-8"),
|
170
|
+
msg=str(value).encode("utf-8"),
|
171
|
+
digestmod=ALLOWED_HASH_TYPES[hash_type],
|
172
|
+
)
|
173
|
+
|
174
|
+
return hmac_obj.hexdigest()
|
175
|
+
|
176
|
+
|
177
|
+
_filters_list = [
|
178
|
+
hash,
|
179
|
+
base64encode,
|
180
|
+
base64decode,
|
181
|
+
base64binascii_decode,
|
182
|
+
string,
|
183
|
+
regex_search,
|
184
|
+
hmac,
|
185
|
+
]
|
139
186
|
filters = {f.__name__: f for f in _filters_list}
|
@@ -82,10 +82,6 @@ class EntrypointOutput:
|
|
82
82
|
def state_messages(self) -> List[AirbyteMessage]:
|
83
83
|
return self._get_message_by_types([Type.STATE])
|
84
84
|
|
85
|
-
@property
|
86
|
-
def connection_status_messages(self) -> List[AirbyteMessage]:
|
87
|
-
return self._get_message_by_types([Type.CONNECTION_STATUS])
|
88
|
-
|
89
85
|
@property
|
90
86
|
def most_recent_state(self) -> Any:
|
91
87
|
state_messages = self._get_message_by_types([Type.STATE])
|
@@ -101,7 +101,7 @@ airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py,sha256=9IAJT
|
|
101
101
|
airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py,sha256=2YBOA2NnwAeIKlIhSwUB_W-FaGnPcmrG_liY7b4mV2Y,8365
|
102
102
|
airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py,sha256=10LFv1QPM-agVKl6eaANmEBOfd7gZgBrkoTcMggsieQ,4809
|
103
103
|
airbyte_cdk/sources/declarative/interpolation/__init__.py,sha256=Kh7FxhfetyNVDnAQ9zSxNe4oUbb8CvoW7Mqz7cs2iPg,437
|
104
|
-
airbyte_cdk/sources/declarative/interpolation/filters.py,sha256=
|
104
|
+
airbyte_cdk/sources/declarative/interpolation/filters.py,sha256=cYap5zzOxIJWCLIfbkNlpyfUhjZ8FklLroIG4WGzYVs,5537
|
105
105
|
airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py,sha256=8F3ntT_Mfo8cO9n6dCq8rTfJIpfKmzRCsVtVdhzaoGc,1964
|
106
106
|
airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py,sha256=h36RIng4GZ9v4o_fRmgJjTNOtWmhK7NOILU1oSKPE4Q,2083
|
107
107
|
airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py,sha256=vjwvkLk7_l6YDcFClwjCMcTleRjQBh7-dzny7PUaoG8,1857
|
@@ -336,26 +336,13 @@ airbyte_cdk/sql/shared/sql_processor.py,sha256=fPWLcDaCBXQRV1ugdWIZ6x53FYGQ1BpMF
|
|
336
336
|
airbyte_cdk/sql/types.py,sha256=XEIhRAo_ASd0kVLBkdLf5bHiRhNple-IJrC9TibcDdY,5880
|
337
337
|
airbyte_cdk/test/__init__.py,sha256=f_XdkOg4_63QT2k3BbKY34209lppwgw-svzfZstQEq4,199
|
338
338
|
airbyte_cdk/test/catalog_builder.py,sha256=-y05Cz1x0Dlk6oE9LSKhCozssV2gYBNtMdV5YYOPOtk,3015
|
339
|
-
airbyte_cdk/test/
|
340
|
-
airbyte_cdk/test/declarative/models/__init__.py,sha256=rXoywbDd-oqEhRQLuaEQIDxykMeawHjzzeN3XIpGQN8,132
|
341
|
-
airbyte_cdk/test/declarative/models/scenario.py,sha256=ySzQosxi2TRaw7JbQMFsuBieoH0ufXJhEY8Vu3DXtx4,2378
|
342
|
-
airbyte_cdk/test/declarative/test_suites/__init__.py,sha256=E1KY-xxKjkKXYaAVyCs96-ZMCCoDhR24s2eQhWWOx-c,769
|
343
|
-
airbyte_cdk/test/declarative/test_suites/connector_base.py,sha256=7qFjRI6NkJPOTIHndl8KvWz4eM2PiXMPBSKOBpGKk7o,6760
|
344
|
-
airbyte_cdk/test/declarative/test_suites/declarative_sources.py,sha256=4VPGzG8keyci0x2o6_jNqx7HcAuamumJoPafudH5w_o,1633
|
345
|
-
airbyte_cdk/test/declarative/test_suites/destination_base.py,sha256=L_l1gpNCkzfx-c7mzS1He5hTVbqR39OnfWdrYMglS7E,486
|
346
|
-
airbyte_cdk/test/declarative/test_suites/source_base.py,sha256=1VJ9KXM1nSodFKh2VKXc2cO010JfE-df2JoVGKasrmk,4381
|
347
|
-
airbyte_cdk/test/declarative/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
348
|
-
airbyte_cdk/test/declarative/utils/job_runner.py,sha256=AI10Nt5MEMbJt_zNNBatUPStvMEF1mstHzow-11ZVsE,4845
|
349
|
-
airbyte_cdk/test/entrypoint_wrapper.py,sha256=TyUmVJyIuGelAv6y8Wy_BnwqIRw_drjfZWKlroljCuQ,9951
|
350
|
-
airbyte_cdk/test/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
351
|
-
airbyte_cdk/test/fixtures/auto.py,sha256=wYKu1pBUJX60Bg-K2RwNXi11Txu1Rncc8WNlOvy9Zug,354
|
339
|
+
airbyte_cdk/test/entrypoint_wrapper.py,sha256=9XBii_YguQp0d8cykn3hy102FsJcwIBQzSB7co5ho0s,9802
|
352
340
|
airbyte_cdk/test/mock_http/__init__.py,sha256=jE5kC6CQ0OXkTqKhciDnNVZHesBFVIA2YvkdFGwva7k,322
|
353
341
|
airbyte_cdk/test/mock_http/matcher.py,sha256=4Qj8UnJKZIs-eodshryce3SN1Ayc8GZpBETmP6hTEyc,1446
|
354
342
|
airbyte_cdk/test/mock_http/mocker.py,sha256=XgsjMtVoeMpRELPyALgrkHFauH9H5irxrz1Kcxh2yFY,8013
|
355
343
|
airbyte_cdk/test/mock_http/request.py,sha256=tdB8cqk2vLgCDTOKffBKsM06llYs4ZecgtH6DKyx6yY,4112
|
356
344
|
airbyte_cdk/test/mock_http/response.py,sha256=s4-cQQqTtmeej0pQDWqmG0vUWpHS-93lIWMpW3zSVyU,662
|
357
345
|
airbyte_cdk/test/mock_http/response_builder.py,sha256=debPx_lRYBaQVSwCoKLa0F8KFk3h0qG7bWxFBATa0cc,7958
|
358
|
-
airbyte_cdk/test/pytest_config/plugin.py,sha256=Z2RYpXOKTRsytb0pn0K-PvkQlQXeOppnEcX4IlBEZfI,1457
|
359
346
|
airbyte_cdk/test/state_builder.py,sha256=kLPql9lNzUJaBg5YYRLJlY_Hy5JLHJDVyKPMZMoYM44,946
|
360
347
|
airbyte_cdk/test/utils/__init__.py,sha256=Hu-1XT2KDoYjDF7-_ziDwv5bY3PueGjANOCbzeOegDg,57
|
361
348
|
airbyte_cdk/test/utils/data.py,sha256=CkCR1_-rujWNmPXFR1IXTMwx1rAl06wAyIKWpDcN02w,820
|
@@ -379,9 +366,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
|
|
379
366
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
380
367
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
381
368
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
382
|
-
airbyte_cdk-6.45.
|
383
|
-
airbyte_cdk-6.45.
|
384
|
-
airbyte_cdk-6.45.
|
385
|
-
airbyte_cdk-6.45.
|
386
|
-
airbyte_cdk-6.45.
|
387
|
-
airbyte_cdk-6.45.
|
369
|
+
airbyte_cdk-6.45.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
370
|
+
airbyte_cdk-6.45.1.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
371
|
+
airbyte_cdk-6.45.1.dist-info/METADATA,sha256=l16uftPwU9g9GDgSMmhyD0UWLcVSgNiN5SlMpuUJwMs,6071
|
372
|
+
airbyte_cdk-6.45.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
373
|
+
airbyte_cdk-6.45.1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
|
374
|
+
airbyte_cdk-6.45.1.dist-info/RECORD,,
|
@@ -1,74 +0,0 @@
|
|
1
|
-
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
|
-
"""Run acceptance tests in PyTest.
|
3
|
-
|
4
|
-
These tests leverage the same `acceptance-test-config.yml` configuration files as the
|
5
|
-
acceptance tests in CAT, but they run in PyTest instead of CAT. This allows us to run
|
6
|
-
the acceptance tests in the same local environment as we are developing in, speeding
|
7
|
-
up iteration cycles.
|
8
|
-
"""
|
9
|
-
|
10
|
-
from __future__ import annotations
|
11
|
-
|
12
|
-
from pathlib import Path
|
13
|
-
from typing import Any, Literal, cast
|
14
|
-
|
15
|
-
import yaml
|
16
|
-
from pydantic import BaseModel
|
17
|
-
|
18
|
-
|
19
|
-
class ConnectorTestScenario(BaseModel):
|
20
|
-
"""Acceptance test instance, as a Pydantic model.
|
21
|
-
|
22
|
-
This class represents an acceptance test instance, which is a single test case
|
23
|
-
that can be run against a connector. It is used to deserialize and validate the
|
24
|
-
acceptance test configuration file.
|
25
|
-
"""
|
26
|
-
|
27
|
-
class AcceptanceTestExpectRecords(BaseModel):
|
28
|
-
path: Path
|
29
|
-
exact_order: bool = False
|
30
|
-
|
31
|
-
class AcceptanceTestFileTypes(BaseModel):
|
32
|
-
skip_test: bool
|
33
|
-
bypass_reason: str
|
34
|
-
|
35
|
-
config_path: Path | None = None
|
36
|
-
config_dict: dict[str, Any] | None = None
|
37
|
-
|
38
|
-
id: str | None = None
|
39
|
-
|
40
|
-
configured_catalog_path: Path | None = None
|
41
|
-
timeout_seconds: int | None = None
|
42
|
-
expect_records: AcceptanceTestExpectRecords | None = None
|
43
|
-
file_types: AcceptanceTestFileTypes | None = None
|
44
|
-
status: Literal["succeed", "failed"] | None = None
|
45
|
-
|
46
|
-
def get_config_dict(self) -> dict[str, Any]:
|
47
|
-
"""Return the config dictionary.
|
48
|
-
|
49
|
-
If a config dictionary has already been loaded, return it. Otherwise, load
|
50
|
-
the config file and return the dictionary.
|
51
|
-
"""
|
52
|
-
if self.config_dict:
|
53
|
-
return self.config_dict
|
54
|
-
|
55
|
-
if self.config_path:
|
56
|
-
return cast(dict[str, Any], yaml.safe_load(self.config_path.read_text()))
|
57
|
-
|
58
|
-
raise ValueError("No config dictionary or path provided.")
|
59
|
-
|
60
|
-
@property
|
61
|
-
def expect_exception(self) -> bool:
|
62
|
-
return self.status and self.status == "failed" or False
|
63
|
-
|
64
|
-
@property
|
65
|
-
def instance_name(self) -> str:
|
66
|
-
return self.config_path.stem if self.config_path else "Unnamed Scenario"
|
67
|
-
|
68
|
-
def __str__(self) -> str:
|
69
|
-
if self.id:
|
70
|
-
return f"'{self.id}' Test Scenario"
|
71
|
-
if self.config_path:
|
72
|
-
return f"'{self.config_path.name}' Test Scenario"
|
73
|
-
|
74
|
-
return f"'{hash(self)}' Test Scenario"
|
@@ -1,24 +0,0 @@
|
|
1
|
-
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
|
-
"""Declarative test suites.
|
3
|
-
|
4
|
-
Here we have base classes for a robust set of declarative connector test suites.
|
5
|
-
"""
|
6
|
-
|
7
|
-
from airbyte_cdk.test.declarative.test_suites.connector_base import (
|
8
|
-
ConnectorTestScenario,
|
9
|
-
generate_tests,
|
10
|
-
)
|
11
|
-
from airbyte_cdk.test.declarative.test_suites.declarative_sources import (
|
12
|
-
DeclarativeSourceTestSuite,
|
13
|
-
)
|
14
|
-
from airbyte_cdk.test.declarative.test_suites.destination_base import DestinationTestSuiteBase
|
15
|
-
from airbyte_cdk.test.declarative.test_suites.source_base import SourceTestSuiteBase
|
16
|
-
|
17
|
-
__all__ = [
|
18
|
-
"ConnectorTestScenario",
|
19
|
-
"ConnectorTestSuiteBase",
|
20
|
-
"DeclarativeSourceTestSuite",
|
21
|
-
"DestinationTestSuiteBase",
|
22
|
-
"SourceTestSuiteBase",
|
23
|
-
"generate_tests",
|
24
|
-
]
|
@@ -1,202 +0,0 @@
|
|
1
|
-
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
|
-
"""Base class for connector test suites."""
|
3
|
-
|
4
|
-
from __future__ import annotations
|
5
|
-
|
6
|
-
import abc
|
7
|
-
import functools
|
8
|
-
import inspect
|
9
|
-
import sys
|
10
|
-
from pathlib import Path
|
11
|
-
from typing import Any, Callable, Literal
|
12
|
-
|
13
|
-
import pytest
|
14
|
-
import yaml
|
15
|
-
from pydantic import BaseModel
|
16
|
-
from typing_extensions import override
|
17
|
-
|
18
|
-
from airbyte_cdk import Connector
|
19
|
-
from airbyte_cdk.models import (
|
20
|
-
AirbyteMessage,
|
21
|
-
Type,
|
22
|
-
)
|
23
|
-
from airbyte_cdk.sources.declarative.declarative_source import (
|
24
|
-
AbstractSource,
|
25
|
-
ConcurrentDeclarativeSource,
|
26
|
-
Source,
|
27
|
-
)
|
28
|
-
from airbyte_cdk.test import entrypoint_wrapper
|
29
|
-
from airbyte_cdk.test.declarative.models import (
|
30
|
-
ConnectorTestScenario,
|
31
|
-
)
|
32
|
-
from airbyte_cdk.test.declarative.utils.job_runner import run_test_job
|
33
|
-
|
34
|
-
ACCEPTANCE_TEST_CONFIG = "acceptance-test-config.yml"
|
35
|
-
|
36
|
-
|
37
|
-
class JavaClass(str):
|
38
|
-
"""A string that represents a Java class."""
|
39
|
-
|
40
|
-
|
41
|
-
class DockerImage(str):
|
42
|
-
"""A string that represents a Docker image."""
|
43
|
-
|
44
|
-
|
45
|
-
class RunnableConnector(abc.ABC):
|
46
|
-
"""A connector that can be run in a test scenario."""
|
47
|
-
|
48
|
-
@abc.abstractmethod
|
49
|
-
def launch(cls, args: list[str] | None) -> None: ...
|
50
|
-
|
51
|
-
|
52
|
-
def generate_tests(metafunc) -> None:
|
53
|
-
"""
|
54
|
-
A helper for pytest_generate_tests hook.
|
55
|
-
|
56
|
-
If a test method (in a class subclassed from our base class)
|
57
|
-
declares an argument 'instance', this function retrieves the
|
58
|
-
'scenarios' attribute from the test class and parametrizes that
|
59
|
-
test with the values from 'scenarios'.
|
60
|
-
|
61
|
-
## Usage
|
62
|
-
|
63
|
-
```python
|
64
|
-
from airbyte_cdk.test.declarative.test_suites.connector_base import (
|
65
|
-
generate_tests,
|
66
|
-
ConnectorTestSuiteBase,
|
67
|
-
)
|
68
|
-
|
69
|
-
def pytest_generate_tests(metafunc):
|
70
|
-
generate_tests(metafunc)
|
71
|
-
|
72
|
-
class TestMyConnector(ConnectorTestSuiteBase):
|
73
|
-
...
|
74
|
-
|
75
|
-
```
|
76
|
-
"""
|
77
|
-
# Check if the test function requires an 'instance' argument
|
78
|
-
if "instance" in metafunc.fixturenames:
|
79
|
-
# Retrieve the test class
|
80
|
-
test_class = metafunc.cls
|
81
|
-
if test_class is None:
|
82
|
-
raise ValueError("Expected a class here.")
|
83
|
-
# Get the 'scenarios' attribute from the class
|
84
|
-
scenarios_attr = getattr(test_class, "get_scenarios", None)
|
85
|
-
if scenarios_attr is None:
|
86
|
-
raise ValueError(
|
87
|
-
f"Test class {test_class} does not have a 'scenarios' attribute. "
|
88
|
-
"Please define the 'scenarios' attribute in the test class."
|
89
|
-
)
|
90
|
-
|
91
|
-
scenarios = test_class.get_scenarios()
|
92
|
-
ids = [str(scenario) for scenario in scenarios]
|
93
|
-
metafunc.parametrize("instance", scenarios, ids=ids)
|
94
|
-
|
95
|
-
|
96
|
-
class ConnectorTestSuiteBase(abc.ABC):
|
97
|
-
"""Base class for connector test suites."""
|
98
|
-
|
99
|
-
acceptance_test_file_path = Path("./acceptance-test-config.json")
|
100
|
-
"""The path to the acceptance test config file.
|
101
|
-
|
102
|
-
By default, this is set to the `acceptance-test-config.json` file in
|
103
|
-
the root of the connector source directory.
|
104
|
-
"""
|
105
|
-
|
106
|
-
connector: type[Connector] | Path | JavaClass | DockerImage | None = None
|
107
|
-
"""The connector class or path to the connector to test."""
|
108
|
-
|
109
|
-
working_dir: Path | None = None
|
110
|
-
"""The root directory of the connector source code."""
|
111
|
-
|
112
|
-
@classmethod
|
113
|
-
def create_connector(
|
114
|
-
cls, scenario: ConnectorTestScenario
|
115
|
-
) -> Source | AbstractSource | ConcurrentDeclarativeSource | RunnableConnector:
|
116
|
-
"""Instantiate the connector class."""
|
117
|
-
raise NotImplementedError("Subclasses must implement this method.")
|
118
|
-
|
119
|
-
def run_test_scenario(
|
120
|
-
self,
|
121
|
-
verb: Literal["read", "check", "discover"],
|
122
|
-
test_scenario: ConnectorTestScenario,
|
123
|
-
*,
|
124
|
-
catalog: dict | None = None,
|
125
|
-
) -> entrypoint_wrapper.EntrypointOutput:
|
126
|
-
"""Run a test job from provided CLI args and return the result."""
|
127
|
-
return run_test_job(
|
128
|
-
self.create_connector(test_scenario),
|
129
|
-
verb,
|
130
|
-
test_instance=test_scenario,
|
131
|
-
catalog=catalog,
|
132
|
-
)
|
133
|
-
|
134
|
-
# Test Definitions
|
135
|
-
|
136
|
-
def test_check(
|
137
|
-
self,
|
138
|
-
instance: ConnectorTestScenario,
|
139
|
-
) -> None:
|
140
|
-
"""Run `connection` acceptance tests."""
|
141
|
-
result = self.run_test_scenario(
|
142
|
-
"check",
|
143
|
-
test_scenario=instance,
|
144
|
-
)
|
145
|
-
conn_status_messages: list[AirbyteMessage] = [
|
146
|
-
msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS
|
147
|
-
] # noqa: SLF001 # Non-public API
|
148
|
-
assert len(conn_status_messages) == 1, (
|
149
|
-
"Expected exactly one CONNECTION_STATUS message. Got: \n" + "\n".join(result._messages)
|
150
|
-
)
|
151
|
-
|
152
|
-
@classmethod
|
153
|
-
@property
|
154
|
-
def acceptance_test_config_path(self) -> Path:
|
155
|
-
"""Get the path to the acceptance test config file.
|
156
|
-
|
157
|
-
Check vwd and parent directories of cwd for the config file, and return the first one found.
|
158
|
-
|
159
|
-
Give up if the config file is not found in any parent directory.
|
160
|
-
"""
|
161
|
-
current_dir = Path.cwd()
|
162
|
-
for parent_dir in current_dir.parents:
|
163
|
-
config_path = parent_dir / ACCEPTANCE_TEST_CONFIG
|
164
|
-
if config_path.exists():
|
165
|
-
return config_path
|
166
|
-
raise FileNotFoundError(
|
167
|
-
f"Acceptance test config file not found in any parent directory from : {Path.cwd()}"
|
168
|
-
)
|
169
|
-
|
170
|
-
@classmethod
|
171
|
-
def get_scenarios(
|
172
|
-
cls,
|
173
|
-
) -> list[ConnectorTestScenario]:
|
174
|
-
"""Get acceptance tests for a given category.
|
175
|
-
|
176
|
-
This has to be a separate function because pytest does not allow
|
177
|
-
parametrization of fixtures with arguments from the test class itself.
|
178
|
-
"""
|
179
|
-
category = "connection"
|
180
|
-
all_tests_config = yaml.safe_load(cls.acceptance_test_config_path.read_text())
|
181
|
-
if "acceptance_tests" not in all_tests_config:
|
182
|
-
raise ValueError(
|
183
|
-
f"Acceptance tests config not found in {cls.acceptance_test_config_path}."
|
184
|
-
f" Found only: {str(all_tests_config)}."
|
185
|
-
)
|
186
|
-
if category not in all_tests_config["acceptance_tests"]:
|
187
|
-
return []
|
188
|
-
if "tests" not in all_tests_config["acceptance_tests"][category]:
|
189
|
-
raise ValueError(f"No tests found for category {category}")
|
190
|
-
|
191
|
-
tests_scenarios = [
|
192
|
-
ConnectorTestScenario.model_validate(test)
|
193
|
-
for test in all_tests_config["acceptance_tests"][category]["tests"]
|
194
|
-
if "iam_role" not in test["config_path"]
|
195
|
-
]
|
196
|
-
working_dir = cls.working_dir or Path()
|
197
|
-
for test in tests_scenarios:
|
198
|
-
if test.config_path:
|
199
|
-
test.config_path = working_dir / test.config_path
|
200
|
-
if test.configured_catalog_path:
|
201
|
-
test.configured_catalog_path = working_dir / test.configured_catalog_path
|
202
|
-
return tests_scenarios
|
@@ -1,48 +0,0 @@
|
|
1
|
-
import os
|
2
|
-
from hashlib import md5
|
3
|
-
from pathlib import Path
|
4
|
-
from typing import Any, cast
|
5
|
-
|
6
|
-
import yaml
|
7
|
-
|
8
|
-
from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
|
9
|
-
ConcurrentDeclarativeSource,
|
10
|
-
)
|
11
|
-
from airbyte_cdk.test.declarative.models import ConnectorTestScenario
|
12
|
-
from airbyte_cdk.test.declarative.test_suites.source_base import (
|
13
|
-
SourceTestSuiteBase,
|
14
|
-
)
|
15
|
-
|
16
|
-
|
17
|
-
def md5_checksum(file_path: Path) -> str:
|
18
|
-
with open(file_path, "rb") as file:
|
19
|
-
return md5(file.read()).hexdigest()
|
20
|
-
|
21
|
-
|
22
|
-
class DeclarativeSourceTestSuite(SourceTestSuiteBase):
|
23
|
-
manifest_path = Path("manifest.yaml")
|
24
|
-
components_py_path: Path | None = None
|
25
|
-
|
26
|
-
def create_connector(
|
27
|
-
self, connector_test: ConnectorTestScenario
|
28
|
-
) -> ConcurrentDeclarativeSource:
|
29
|
-
"""Create a connector instance for the test suite."""
|
30
|
-
config = connector_test.get_config_dict()
|
31
|
-
# catalog = connector_test.get_catalog()
|
32
|
-
# state = connector_test.get_state()
|
33
|
-
# source_config = connector_test.get_source_config()
|
34
|
-
|
35
|
-
manifest_dict = yaml.safe_load(self.manifest_path.read_text())
|
36
|
-
if self.components_py_path and self.components_py_path.exists():
|
37
|
-
os.environ["AIRBYTE_ENABLE_UNSAFE_CODE"] = "true"
|
38
|
-
config["__injected_components_py"] = self.components_py_path.read_text()
|
39
|
-
config["__injected_components_py_checksums"] = {
|
40
|
-
"md5": md5_checksum(self.components_py_path),
|
41
|
-
}
|
42
|
-
|
43
|
-
return ConcurrentDeclarativeSource(
|
44
|
-
config=config,
|
45
|
-
catalog=None,
|
46
|
-
state=None,
|
47
|
-
source_config=manifest_dict,
|
48
|
-
)
|
@@ -1,12 +0,0 @@
|
|
1
|
-
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
|
-
"""Base class for destination test suites."""
|
3
|
-
|
4
|
-
from airbyte_cdk.test.declarative.test_suites.connector_base import ConnectorTestSuiteBase
|
5
|
-
|
6
|
-
|
7
|
-
class DestinationTestSuiteBase(ConnectorTestSuiteBase):
|
8
|
-
"""Base class for destination test suites.
|
9
|
-
|
10
|
-
This class provides a base set of functionality for testing destination connectors, and it
|
11
|
-
inherits all generic connector tests from the `ConnectorTestSuiteBase` class.
|
12
|
-
"""
|
@@ -1,129 +0,0 @@
|
|
1
|
-
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
2
|
-
"""Base class for source test suites."""
|
3
|
-
|
4
|
-
from dataclasses import asdict
|
5
|
-
from pathlib import Path
|
6
|
-
|
7
|
-
import pytest
|
8
|
-
|
9
|
-
from airbyte_cdk.models import (
|
10
|
-
AirbyteMessage,
|
11
|
-
AirbyteStream,
|
12
|
-
ConfiguredAirbyteCatalog,
|
13
|
-
ConfiguredAirbyteStream,
|
14
|
-
DestinationSyncMode,
|
15
|
-
SyncMode,
|
16
|
-
Type,
|
17
|
-
)
|
18
|
-
from airbyte_cdk.test import entrypoint_wrapper
|
19
|
-
from airbyte_cdk.test.declarative.models import (
|
20
|
-
ConnectorTestScenario,
|
21
|
-
)
|
22
|
-
from airbyte_cdk.test.declarative.test_suites.connector_base import (
|
23
|
-
ConnectorTestSuiteBase,
|
24
|
-
)
|
25
|
-
from airbyte_cdk.test.declarative.utils.job_runner import run_test_job
|
26
|
-
|
27
|
-
|
28
|
-
class SourceTestSuiteBase(ConnectorTestSuiteBase):
|
29
|
-
"""Base class for source test suites.
|
30
|
-
|
31
|
-
This class provides a base set of functionality for testing source connectors, and it
|
32
|
-
inherits all generic connector tests from the `ConnectorTestSuiteBase` class.
|
33
|
-
"""
|
34
|
-
|
35
|
-
def test_check(
|
36
|
-
self,
|
37
|
-
instance: ConnectorTestScenario,
|
38
|
-
) -> None:
|
39
|
-
"""Run `connection` acceptance tests."""
|
40
|
-
result: entrypoint_wrapper.EntrypointOutput = run_test_job(
|
41
|
-
self.create_connector(instance),
|
42
|
-
"check",
|
43
|
-
test_instance=instance,
|
44
|
-
)
|
45
|
-
conn_status_messages: list[AirbyteMessage] = [
|
46
|
-
msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS
|
47
|
-
] # noqa: SLF001 # Non-public API
|
48
|
-
assert len(conn_status_messages) == 1, (
|
49
|
-
"Expected exactly one CONNECTION_STATUS message. Got: \n" + "\n".join(result._messages)
|
50
|
-
)
|
51
|
-
|
52
|
-
def test_basic_read(
|
53
|
-
self,
|
54
|
-
instance: ConnectorTestScenario,
|
55
|
-
) -> None:
|
56
|
-
"""Run acceptance tests."""
|
57
|
-
discover_result = run_test_job(
|
58
|
-
self.create_connector(instance),
|
59
|
-
"discover",
|
60
|
-
test_instance=instance,
|
61
|
-
)
|
62
|
-
if instance.expect_exception:
|
63
|
-
assert discover_result.errors, "Expected exception but got none."
|
64
|
-
return
|
65
|
-
|
66
|
-
configured_catalog = ConfiguredAirbyteCatalog(
|
67
|
-
streams=[
|
68
|
-
ConfiguredAirbyteStream(
|
69
|
-
stream=stream,
|
70
|
-
sync_mode=SyncMode.full_refresh,
|
71
|
-
destination_sync_mode=DestinationSyncMode.append_dedup,
|
72
|
-
)
|
73
|
-
for stream in discover_result.catalog.catalog.streams
|
74
|
-
]
|
75
|
-
)
|
76
|
-
result = run_test_job(
|
77
|
-
self.create_connector(instance),
|
78
|
-
"read",
|
79
|
-
test_instance=instance,
|
80
|
-
catalog=configured_catalog,
|
81
|
-
)
|
82
|
-
|
83
|
-
if not result.records:
|
84
|
-
raise AssertionError("Expected records but got none.") # noqa: TRY003
|
85
|
-
|
86
|
-
def test_fail_with_bad_catalog(
|
87
|
-
self,
|
88
|
-
instance: ConnectorTestScenario,
|
89
|
-
) -> None:
|
90
|
-
"""Test that a bad catalog fails."""
|
91
|
-
invalid_configured_catalog = ConfiguredAirbyteCatalog(
|
92
|
-
streams=[
|
93
|
-
# Create ConfiguredAirbyteStream which is deliberately invalid
|
94
|
-
# with regard to the Airbyte Protocol.
|
95
|
-
# This should cause the connector to fail.
|
96
|
-
ConfiguredAirbyteStream(
|
97
|
-
stream=AirbyteStream(
|
98
|
-
name="__AIRBYTE__stream_that_does_not_exist",
|
99
|
-
json_schema={
|
100
|
-
"type": "object",
|
101
|
-
"properties": {"f1": {"type": "string"}},
|
102
|
-
},
|
103
|
-
supported_sync_modes=[SyncMode.full_refresh],
|
104
|
-
),
|
105
|
-
sync_mode="INVALID",
|
106
|
-
destination_sync_mode="INVALID",
|
107
|
-
)
|
108
|
-
]
|
109
|
-
)
|
110
|
-
# Set expected status to "failed" to ensure the test fails if the connector.
|
111
|
-
instance.status = "failed"
|
112
|
-
result = self.run_test_scenario(
|
113
|
-
"read",
|
114
|
-
test_scenario=instance,
|
115
|
-
catalog=asdict(invalid_configured_catalog),
|
116
|
-
)
|
117
|
-
assert result.errors, "Expected errors but got none."
|
118
|
-
assert result.trace_messages, "Expected trace messages but got none."
|
119
|
-
|
120
|
-
def test_discover(
|
121
|
-
self,
|
122
|
-
instance: ConnectorTestScenario,
|
123
|
-
) -> None:
|
124
|
-
"""Run acceptance tests."""
|
125
|
-
run_test_job(
|
126
|
-
self.create_connector(instance),
|
127
|
-
"check",
|
128
|
-
test_instance=instance,
|
129
|
-
)
|
File without changes
|
@@ -1,128 +0,0 @@
|
|
1
|
-
import tempfile
|
2
|
-
import uuid
|
3
|
-
from pathlib import Path
|
4
|
-
from typing import Any, Callable, Literal
|
5
|
-
|
6
|
-
import orjson
|
7
|
-
|
8
|
-
from airbyte_cdk import Connector
|
9
|
-
from airbyte_cdk.models import (
|
10
|
-
Status,
|
11
|
-
)
|
12
|
-
from airbyte_cdk.sources.abstract_source import AbstractSource
|
13
|
-
from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
|
14
|
-
from airbyte_cdk.test import entrypoint_wrapper
|
15
|
-
from airbyte_cdk.test.declarative.models import (
|
16
|
-
ConnectorTestScenario,
|
17
|
-
)
|
18
|
-
|
19
|
-
|
20
|
-
def run_test_job(
|
21
|
-
connector: Connector | type[Connector] | Callable[[], Connector],
|
22
|
-
verb: Literal["read", "check", "discover"],
|
23
|
-
test_instance: ConnectorTestScenario,
|
24
|
-
*,
|
25
|
-
catalog: dict[str, Any] | None = None,
|
26
|
-
) -> entrypoint_wrapper.EntrypointOutput:
|
27
|
-
"""Run a test job from provided CLI args and return the result."""
|
28
|
-
if not connector:
|
29
|
-
raise ValueError("Connector is required")
|
30
|
-
|
31
|
-
connector_obj: Connector
|
32
|
-
if isinstance(connector, type):
|
33
|
-
connector_obj = connector()
|
34
|
-
elif isinstance(connector, Connector):
|
35
|
-
connector_obj = connector
|
36
|
-
elif isinstance(connector, DeclarativeSource | AbstractSource):
|
37
|
-
connector_obj = connector
|
38
|
-
elif isinstance(connector, Callable):
|
39
|
-
try:
|
40
|
-
connector_obj = connector()
|
41
|
-
except Exception as ex:
|
42
|
-
if not test_instance.expect_exception:
|
43
|
-
raise
|
44
|
-
|
45
|
-
return entrypoint_wrapper.EntrypointOutput(
|
46
|
-
messages=[],
|
47
|
-
uncaught_exception=ex,
|
48
|
-
)
|
49
|
-
else:
|
50
|
-
raise ValueError(f"Invalid source type: {type(connector)}")
|
51
|
-
|
52
|
-
args: list[str] = [verb]
|
53
|
-
if test_instance.config_path:
|
54
|
-
args += ["--config", str(test_instance.config_path)]
|
55
|
-
elif test_instance.config_dict:
|
56
|
-
config_path = (
|
57
|
-
Path(tempfile.gettempdir()) / "airbyte-test" / f"temp_config_{uuid.uuid4().hex}.json"
|
58
|
-
)
|
59
|
-
config_path.parent.mkdir(parents=True, exist_ok=True)
|
60
|
-
config_path.write_text(orjson.dumps(test_instance.config_dict).decode())
|
61
|
-
args += ["--config", str(config_path)]
|
62
|
-
|
63
|
-
catalog_path: Path | None = None
|
64
|
-
if verb not in ["discover", "check"]:
|
65
|
-
# We need a catalog for read.
|
66
|
-
if catalog:
|
67
|
-
# Write the catalog to a temp json file and pass the path to the file as an argument.
|
68
|
-
catalog_path = (
|
69
|
-
Path(tempfile.gettempdir())
|
70
|
-
/ "airbyte-test"
|
71
|
-
/ f"temp_catalog_{uuid.uuid4().hex}.json"
|
72
|
-
)
|
73
|
-
catalog_path.parent.mkdir(parents=True, exist_ok=True)
|
74
|
-
catalog_path.write_text(orjson.dumps(catalog).decode())
|
75
|
-
elif test_instance.configured_catalog_path:
|
76
|
-
catalog_path = Path(test_instance.configured_catalog_path)
|
77
|
-
|
78
|
-
if catalog_path:
|
79
|
-
args += ["--catalog", str(catalog_path)]
|
80
|
-
|
81
|
-
# This is a bit of a hack because the source needs the catalog early.
|
82
|
-
# Because it *also* can fail, we have ot redundantly wrap it in a try/except block.
|
83
|
-
|
84
|
-
result: entrypoint_wrapper.EntrypointOutput = entrypoint_wrapper._run_command( # noqa: SLF001 # Non-public API
|
85
|
-
source=connector_obj,
|
86
|
-
args=args,
|
87
|
-
expecting_exception=test_instance.expect_exception,
|
88
|
-
)
|
89
|
-
if result.errors and not test_instance.expect_exception:
|
90
|
-
raise AssertionError(
|
91
|
-
"\n\n".join(
|
92
|
-
[str(err.trace.error).replace("\\n", "\n") for err in result.errors],
|
93
|
-
)
|
94
|
-
)
|
95
|
-
|
96
|
-
if verb == "check":
|
97
|
-
# Check is expected to fail gracefully without an exception.
|
98
|
-
# Instead, we assert that we have a CONNECTION_STATUS message with
|
99
|
-
# a failure status.
|
100
|
-
assert not result.errors, "Expected no errors from check. Got:\n" + "\n".join(
|
101
|
-
[str(error) for error in result.errors]
|
102
|
-
)
|
103
|
-
assert len(result.connection_status_messages) == 1, (
|
104
|
-
"Expected exactly one CONNECTION_STATUS message. Got "
|
105
|
-
f"{len(result.connection_status_messages)}:\n"
|
106
|
-
+ "\n".join([str(msg) for msg in result.connection_status_messages])
|
107
|
-
)
|
108
|
-
if test_instance.expect_exception:
|
109
|
-
assert result.connection_status_messages[0].connectionStatus.status == Status.FAILED, (
|
110
|
-
"Expected CONNECTION_STATUS message to be FAILED. Got: \n"
|
111
|
-
+ "\n".join([str(msg) for msg in result.connection_status_messages])
|
112
|
-
)
|
113
|
-
return result
|
114
|
-
|
115
|
-
# For all other verbs, we assert check that an exception is raised (or not).
|
116
|
-
if test_instance.expect_exception:
|
117
|
-
if not result.errors:
|
118
|
-
raise AssertionError("Expected exception but got none.")
|
119
|
-
|
120
|
-
return result
|
121
|
-
if result.errors:
|
122
|
-
raise AssertionError(
|
123
|
-
"\n\n".join(
|
124
|
-
[str(err.trace.error).replace("\\n", "\n") for err in result.errors],
|
125
|
-
)
|
126
|
-
)
|
127
|
-
|
128
|
-
return result
|
File without changes
|
@@ -1,14 +0,0 @@
|
|
1
|
-
"""Auto-use fixtures for pytest.
|
2
|
-
|
3
|
-
WARNING: Importing this module will automatically apply these fixtures. If you want to selectively
|
4
|
-
enable fixtures in a different context, you can import directly from the `fixtures.general` module.
|
5
|
-
|
6
|
-
|
7
|
-
Usage:
|
8
|
-
|
9
|
-
```python
|
10
|
-
from airbyte_cdk.test.fixtures import auto
|
11
|
-
# OR
|
12
|
-
from airbyte_cdk.test.fixtures.auto import *
|
13
|
-
```
|
14
|
-
"""
|
@@ -1,46 +0,0 @@
|
|
1
|
-
"""Global pytest configuration for the Airbyte CDK tests."""
|
2
|
-
|
3
|
-
from pathlib import Path
|
4
|
-
from typing import Optional
|
5
|
-
|
6
|
-
import pytest
|
7
|
-
|
8
|
-
|
9
|
-
def pytest_collect_file(parent: Optional[pytest.Module], path: Path) -> pytest.Module | None:
|
10
|
-
"""Collect test files based on their names."""
|
11
|
-
if path.name == "test_connector.py":
|
12
|
-
return cast(pytest.Module, pytest.Module.from_parent(parent, path=path))
|
13
|
-
|
14
|
-
return None
|
15
|
-
|
16
|
-
|
17
|
-
def pytest_configure(config: pytest.Config) -> None:
|
18
|
-
config.addinivalue_line("markers", "connector: mark test as a connector test")
|
19
|
-
|
20
|
-
|
21
|
-
def pytest_addoption(parser: pytest.Parser) -> None:
|
22
|
-
parser.addoption(
|
23
|
-
"--run-connector",
|
24
|
-
action="store_true",
|
25
|
-
default=False,
|
26
|
-
help="run connector tests",
|
27
|
-
)
|
28
|
-
|
29
|
-
|
30
|
-
def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
|
31
|
-
if config.getoption("--run-connector"):
|
32
|
-
return
|
33
|
-
skip_connector = pytest.mark.skip(reason="need --run-connector option to run")
|
34
|
-
for item in items:
|
35
|
-
if "connector" in item.keywords:
|
36
|
-
item.add_marker(skip_connector)
|
37
|
-
|
38
|
-
|
39
|
-
def pytest_runtest_setup(item: pytest.Item) -> None:
|
40
|
-
# This hook is called before each test function is executed
|
41
|
-
print(f"Setting up test: {item.name}")
|
42
|
-
|
43
|
-
|
44
|
-
def pytest_runtest_teardown(item: pytest.Item, nextitem: Optional[pytest.Item]) -> None:
|
45
|
-
# This hook is called after each test function is executed
|
46
|
-
print(f"Tearing down test: {item.name}")
|
{airbyte_cdk-6.45.0.post23.dev14384860031.dist-info → airbyte_cdk-6.45.1.dist-info}/LICENSE.txt
RENAMED
File without changes
|
{airbyte_cdk-6.45.0.post23.dev14384860031.dist-info → airbyte_cdk-6.45.1.dist-info}/LICENSE_SHORT
RENAMED
File without changes
|
File without changes
|
{airbyte_cdk-6.45.0.post23.dev14384860031.dist-info → airbyte_cdk-6.45.1.dist-info}/entry_points.txt
RENAMED
File without changes
|