airbyte-cdk 6.45.0.post23.dev14384860031__py3-none-any.whl → 6.45.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,9 +4,10 @@
4
4
 
5
5
  import base64
6
6
  import hashlib
7
+ import hmac as hmac_lib
7
8
  import json
8
9
  import re
9
- from typing import Any, Optional
10
+ from typing import Any, Dict, Optional
10
11
 
11
12
 
12
13
  def hash(value: Any, hash_type: str = "md5", salt: Optional[str] = None) -> str:
@@ -135,5 +136,51 @@ def regex_search(value: str, regex: str) -> str:
135
136
  return ""
136
137
 
137
138
 
138
- _filters_list = [hash, base64encode, base64decode, base64binascii_decode, string, regex_search]
139
+ def hmac(value: Any, key: str, hash_type: str = "sha256") -> str:
140
+ """
141
+ Implementation of a custom Jinja2 hmac filter with SHA-256 support.
142
+
143
+ This filter creates a Hash-based Message Authentication Code (HMAC) using a cryptographic
144
+ hash function and a secret key. Currently only supports SHA-256, and returns hexdigest of the signature.
145
+
146
+ Example usage in a low code connector:
147
+
148
+ auth_headers:
149
+ $ref: "#/definitions/base_auth"
150
+ $parameters:
151
+ signature: "{{ 'message_to_sign' | hmac('my_secret_key') }}"
152
+
153
+ :param value: The message to be authenticated
154
+ :param key: The secret key for the HMAC
155
+ :param hash_type: Hash algorithm to use (default: sha256)
156
+ :return: HMAC digest as a hexadecimal string
157
+ """
158
+ # Define allowed hash functions
159
+ ALLOWED_HASH_TYPES: Dict[str, Any] = {
160
+ "sha256": hashlib.sha256,
161
+ }
162
+
163
+ if hash_type not in ALLOWED_HASH_TYPES:
164
+ raise ValueError(
165
+ f"Hash type '{hash_type}' is not allowed. Allowed types: {', '.join(ALLOWED_HASH_TYPES.keys())}"
166
+ )
167
+
168
+ hmac_obj = hmac_lib.new(
169
+ key=str(key).encode("utf-8"),
170
+ msg=str(value).encode("utf-8"),
171
+ digestmod=ALLOWED_HASH_TYPES[hash_type],
172
+ )
173
+
174
+ return hmac_obj.hexdigest()
175
+
176
+
177
+ _filters_list = [
178
+ hash,
179
+ base64encode,
180
+ base64decode,
181
+ base64binascii_decode,
182
+ string,
183
+ regex_search,
184
+ hmac,
185
+ ]
139
186
  filters = {f.__name__: f for f in _filters_list}
@@ -82,10 +82,6 @@ class EntrypointOutput:
82
82
  def state_messages(self) -> List[AirbyteMessage]:
83
83
  return self._get_message_by_types([Type.STATE])
84
84
 
85
- @property
86
- def connection_status_messages(self) -> List[AirbyteMessage]:
87
- return self._get_message_by_types([Type.CONNECTION_STATUS])
88
-
89
85
  @property
90
86
  def most_recent_state(self) -> Any:
91
87
  state_messages = self._get_message_by_types([Type.STATE])
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: airbyte-cdk
3
- Version: 6.45.0.post23.dev14384860031
3
+ Version: 6.45.1
4
4
  Summary: A framework for writing Airbyte Connectors.
5
5
  Home-page: https://airbyte.com
6
6
  License: MIT
@@ -101,7 +101,7 @@ airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py,sha256=9IAJT
101
101
  airbyte_cdk/sources/declarative/incremental/per_partition_with_global.py,sha256=2YBOA2NnwAeIKlIhSwUB_W-FaGnPcmrG_liY7b4mV2Y,8365
102
102
  airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py,sha256=10LFv1QPM-agVKl6eaANmEBOfd7gZgBrkoTcMggsieQ,4809
103
103
  airbyte_cdk/sources/declarative/interpolation/__init__.py,sha256=Kh7FxhfetyNVDnAQ9zSxNe4oUbb8CvoW7Mqz7cs2iPg,437
104
- airbyte_cdk/sources/declarative/interpolation/filters.py,sha256=9tqTLFlmIp47QtN6zTToLh-YmO9b4Mbylc3bivEa448,4210
104
+ airbyte_cdk/sources/declarative/interpolation/filters.py,sha256=cYap5zzOxIJWCLIfbkNlpyfUhjZ8FklLroIG4WGzYVs,5537
105
105
  airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py,sha256=8F3ntT_Mfo8cO9n6dCq8rTfJIpfKmzRCsVtVdhzaoGc,1964
106
106
  airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py,sha256=h36RIng4GZ9v4o_fRmgJjTNOtWmhK7NOILU1oSKPE4Q,2083
107
107
  airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py,sha256=vjwvkLk7_l6YDcFClwjCMcTleRjQBh7-dzny7PUaoG8,1857
@@ -336,26 +336,13 @@ airbyte_cdk/sql/shared/sql_processor.py,sha256=fPWLcDaCBXQRV1ugdWIZ6x53FYGQ1BpMF
336
336
  airbyte_cdk/sql/types.py,sha256=XEIhRAo_ASd0kVLBkdLf5bHiRhNple-IJrC9TibcDdY,5880
337
337
  airbyte_cdk/test/__init__.py,sha256=f_XdkOg4_63QT2k3BbKY34209lppwgw-svzfZstQEq4,199
338
338
  airbyte_cdk/test/catalog_builder.py,sha256=-y05Cz1x0Dlk6oE9LSKhCozssV2gYBNtMdV5YYOPOtk,3015
339
- airbyte_cdk/test/declarative/__init__.py,sha256=IpPLFCCWnlpp_eTGKK_2WGRFfenbnolxYLmkQSZMO3U,205
340
- airbyte_cdk/test/declarative/models/__init__.py,sha256=rXoywbDd-oqEhRQLuaEQIDxykMeawHjzzeN3XIpGQN8,132
341
- airbyte_cdk/test/declarative/models/scenario.py,sha256=ySzQosxi2TRaw7JbQMFsuBieoH0ufXJhEY8Vu3DXtx4,2378
342
- airbyte_cdk/test/declarative/test_suites/__init__.py,sha256=E1KY-xxKjkKXYaAVyCs96-ZMCCoDhR24s2eQhWWOx-c,769
343
- airbyte_cdk/test/declarative/test_suites/connector_base.py,sha256=7qFjRI6NkJPOTIHndl8KvWz4eM2PiXMPBSKOBpGKk7o,6760
344
- airbyte_cdk/test/declarative/test_suites/declarative_sources.py,sha256=4VPGzG8keyci0x2o6_jNqx7HcAuamumJoPafudH5w_o,1633
345
- airbyte_cdk/test/declarative/test_suites/destination_base.py,sha256=L_l1gpNCkzfx-c7mzS1He5hTVbqR39OnfWdrYMglS7E,486
346
- airbyte_cdk/test/declarative/test_suites/source_base.py,sha256=1VJ9KXM1nSodFKh2VKXc2cO010JfE-df2JoVGKasrmk,4381
347
- airbyte_cdk/test/declarative/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
348
- airbyte_cdk/test/declarative/utils/job_runner.py,sha256=AI10Nt5MEMbJt_zNNBatUPStvMEF1mstHzow-11ZVsE,4845
349
- airbyte_cdk/test/entrypoint_wrapper.py,sha256=TyUmVJyIuGelAv6y8Wy_BnwqIRw_drjfZWKlroljCuQ,9951
350
- airbyte_cdk/test/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
351
- airbyte_cdk/test/fixtures/auto.py,sha256=wYKu1pBUJX60Bg-K2RwNXi11Txu1Rncc8WNlOvy9Zug,354
339
+ airbyte_cdk/test/entrypoint_wrapper.py,sha256=9XBii_YguQp0d8cykn3hy102FsJcwIBQzSB7co5ho0s,9802
352
340
  airbyte_cdk/test/mock_http/__init__.py,sha256=jE5kC6CQ0OXkTqKhciDnNVZHesBFVIA2YvkdFGwva7k,322
353
341
  airbyte_cdk/test/mock_http/matcher.py,sha256=4Qj8UnJKZIs-eodshryce3SN1Ayc8GZpBETmP6hTEyc,1446
354
342
  airbyte_cdk/test/mock_http/mocker.py,sha256=XgsjMtVoeMpRELPyALgrkHFauH9H5irxrz1Kcxh2yFY,8013
355
343
  airbyte_cdk/test/mock_http/request.py,sha256=tdB8cqk2vLgCDTOKffBKsM06llYs4ZecgtH6DKyx6yY,4112
356
344
  airbyte_cdk/test/mock_http/response.py,sha256=s4-cQQqTtmeej0pQDWqmG0vUWpHS-93lIWMpW3zSVyU,662
357
345
  airbyte_cdk/test/mock_http/response_builder.py,sha256=debPx_lRYBaQVSwCoKLa0F8KFk3h0qG7bWxFBATa0cc,7958
358
- airbyte_cdk/test/pytest_config/plugin.py,sha256=Z2RYpXOKTRsytb0pn0K-PvkQlQXeOppnEcX4IlBEZfI,1457
359
346
  airbyte_cdk/test/state_builder.py,sha256=kLPql9lNzUJaBg5YYRLJlY_Hy5JLHJDVyKPMZMoYM44,946
360
347
  airbyte_cdk/test/utils/__init__.py,sha256=Hu-1XT2KDoYjDF7-_ziDwv5bY3PueGjANOCbzeOegDg,57
361
348
  airbyte_cdk/test/utils/data.py,sha256=CkCR1_-rujWNmPXFR1IXTMwx1rAl06wAyIKWpDcN02w,820
@@ -379,9 +366,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
379
366
  airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
380
367
  airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
381
368
  airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
382
- airbyte_cdk-6.45.0.post23.dev14384860031.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
383
- airbyte_cdk-6.45.0.post23.dev14384860031.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
384
- airbyte_cdk-6.45.0.post23.dev14384860031.dist-info/METADATA,sha256=9vnnu8hLTvth6IXJb8KCu045Tl6ulUD0afpW1H0qziU,6093
385
- airbyte_cdk-6.45.0.post23.dev14384860031.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
386
- airbyte_cdk-6.45.0.post23.dev14384860031.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
387
- airbyte_cdk-6.45.0.post23.dev14384860031.dist-info/RECORD,,
369
+ airbyte_cdk-6.45.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
370
+ airbyte_cdk-6.45.1.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
371
+ airbyte_cdk-6.45.1.dist-info/METADATA,sha256=l16uftPwU9g9GDgSMmhyD0UWLcVSgNiN5SlMpuUJwMs,6071
372
+ airbyte_cdk-6.45.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
373
+ airbyte_cdk-6.45.1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
374
+ airbyte_cdk-6.45.1.dist-info/RECORD,,
@@ -1,6 +0,0 @@
1
- # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
2
- """Declarative tests framework.
3
-
4
- This module provides fixtures and utilities for testing Airbyte sources and destinations
5
- in a declarative way.
6
- """
@@ -1,7 +0,0 @@
1
- from airbyte_cdk.test.declarative.models.scenario import (
2
- ConnectorTestScenario,
3
- )
4
-
5
- __all__ = [
6
- "ConnectorTestScenario",
7
- ]
@@ -1,74 +0,0 @@
1
- # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
2
- """Run acceptance tests in PyTest.
3
-
4
- These tests leverage the same `acceptance-test-config.yml` configuration files as the
5
- acceptance tests in CAT, but they run in PyTest instead of CAT. This allows us to run
6
- the acceptance tests in the same local environment as we are developing in, speeding
7
- up iteration cycles.
8
- """
9
-
10
- from __future__ import annotations
11
-
12
- from pathlib import Path
13
- from typing import Any, Literal, cast
14
-
15
- import yaml
16
- from pydantic import BaseModel
17
-
18
-
19
- class ConnectorTestScenario(BaseModel):
20
- """Acceptance test instance, as a Pydantic model.
21
-
22
- This class represents an acceptance test instance, which is a single test case
23
- that can be run against a connector. It is used to deserialize and validate the
24
- acceptance test configuration file.
25
- """
26
-
27
- class AcceptanceTestExpectRecords(BaseModel):
28
- path: Path
29
- exact_order: bool = False
30
-
31
- class AcceptanceTestFileTypes(BaseModel):
32
- skip_test: bool
33
- bypass_reason: str
34
-
35
- config_path: Path | None = None
36
- config_dict: dict[str, Any] | None = None
37
-
38
- id: str | None = None
39
-
40
- configured_catalog_path: Path | None = None
41
- timeout_seconds: int | None = None
42
- expect_records: AcceptanceTestExpectRecords | None = None
43
- file_types: AcceptanceTestFileTypes | None = None
44
- status: Literal["succeed", "failed"] | None = None
45
-
46
- def get_config_dict(self) -> dict[str, Any]:
47
- """Return the config dictionary.
48
-
49
- If a config dictionary has already been loaded, return it. Otherwise, load
50
- the config file and return the dictionary.
51
- """
52
- if self.config_dict:
53
- return self.config_dict
54
-
55
- if self.config_path:
56
- return cast(dict[str, Any], yaml.safe_load(self.config_path.read_text()))
57
-
58
- raise ValueError("No config dictionary or path provided.")
59
-
60
- @property
61
- def expect_exception(self) -> bool:
62
- return self.status and self.status == "failed" or False
63
-
64
- @property
65
- def instance_name(self) -> str:
66
- return self.config_path.stem if self.config_path else "Unnamed Scenario"
67
-
68
- def __str__(self) -> str:
69
- if self.id:
70
- return f"'{self.id}' Test Scenario"
71
- if self.config_path:
72
- return f"'{self.config_path.name}' Test Scenario"
73
-
74
- return f"'{hash(self)}' Test Scenario"
@@ -1,24 +0,0 @@
1
- # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
2
- """Declarative test suites.
3
-
4
- Here we have base classes for a robust set of declarative connector test suites.
5
- """
6
-
7
- from airbyte_cdk.test.declarative.test_suites.connector_base import (
8
- ConnectorTestScenario,
9
- generate_tests,
10
- )
11
- from airbyte_cdk.test.declarative.test_suites.declarative_sources import (
12
- DeclarativeSourceTestSuite,
13
- )
14
- from airbyte_cdk.test.declarative.test_suites.destination_base import DestinationTestSuiteBase
15
- from airbyte_cdk.test.declarative.test_suites.source_base import SourceTestSuiteBase
16
-
17
- __all__ = [
18
- "ConnectorTestScenario",
19
- "ConnectorTestSuiteBase",
20
- "DeclarativeSourceTestSuite",
21
- "DestinationTestSuiteBase",
22
- "SourceTestSuiteBase",
23
- "generate_tests",
24
- ]
@@ -1,202 +0,0 @@
1
- # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
2
- """Base class for connector test suites."""
3
-
4
- from __future__ import annotations
5
-
6
- import abc
7
- import functools
8
- import inspect
9
- import sys
10
- from pathlib import Path
11
- from typing import Any, Callable, Literal
12
-
13
- import pytest
14
- import yaml
15
- from pydantic import BaseModel
16
- from typing_extensions import override
17
-
18
- from airbyte_cdk import Connector
19
- from airbyte_cdk.models import (
20
- AirbyteMessage,
21
- Type,
22
- )
23
- from airbyte_cdk.sources.declarative.declarative_source import (
24
- AbstractSource,
25
- ConcurrentDeclarativeSource,
26
- Source,
27
- )
28
- from airbyte_cdk.test import entrypoint_wrapper
29
- from airbyte_cdk.test.declarative.models import (
30
- ConnectorTestScenario,
31
- )
32
- from airbyte_cdk.test.declarative.utils.job_runner import run_test_job
33
-
34
- ACCEPTANCE_TEST_CONFIG = "acceptance-test-config.yml"
35
-
36
-
37
- class JavaClass(str):
38
- """A string that represents a Java class."""
39
-
40
-
41
- class DockerImage(str):
42
- """A string that represents a Docker image."""
43
-
44
-
45
- class RunnableConnector(abc.ABC):
46
- """A connector that can be run in a test scenario."""
47
-
48
- @abc.abstractmethod
49
- def launch(cls, args: list[str] | None) -> None: ...
50
-
51
-
52
- def generate_tests(metafunc) -> None:
53
- """
54
- A helper for pytest_generate_tests hook.
55
-
56
- If a test method (in a class subclassed from our base class)
57
- declares an argument 'instance', this function retrieves the
58
- 'scenarios' attribute from the test class and parametrizes that
59
- test with the values from 'scenarios'.
60
-
61
- ## Usage
62
-
63
- ```python
64
- from airbyte_cdk.test.declarative.test_suites.connector_base import (
65
- generate_tests,
66
- ConnectorTestSuiteBase,
67
- )
68
-
69
- def pytest_generate_tests(metafunc):
70
- generate_tests(metafunc)
71
-
72
- class TestMyConnector(ConnectorTestSuiteBase):
73
- ...
74
-
75
- ```
76
- """
77
- # Check if the test function requires an 'instance' argument
78
- if "instance" in metafunc.fixturenames:
79
- # Retrieve the test class
80
- test_class = metafunc.cls
81
- if test_class is None:
82
- raise ValueError("Expected a class here.")
83
- # Get the 'scenarios' attribute from the class
84
- scenarios_attr = getattr(test_class, "get_scenarios", None)
85
- if scenarios_attr is None:
86
- raise ValueError(
87
- f"Test class {test_class} does not have a 'scenarios' attribute. "
88
- "Please define the 'scenarios' attribute in the test class."
89
- )
90
-
91
- scenarios = test_class.get_scenarios()
92
- ids = [str(scenario) for scenario in scenarios]
93
- metafunc.parametrize("instance", scenarios, ids=ids)
94
-
95
-
96
- class ConnectorTestSuiteBase(abc.ABC):
97
- """Base class for connector test suites."""
98
-
99
- acceptance_test_file_path = Path("./acceptance-test-config.json")
100
- """The path to the acceptance test config file.
101
-
102
- By default, this is set to the `acceptance-test-config.json` file in
103
- the root of the connector source directory.
104
- """
105
-
106
- connector: type[Connector] | Path | JavaClass | DockerImage | None = None
107
- """The connector class or path to the connector to test."""
108
-
109
- working_dir: Path | None = None
110
- """The root directory of the connector source code."""
111
-
112
- @classmethod
113
- def create_connector(
114
- cls, scenario: ConnectorTestScenario
115
- ) -> Source | AbstractSource | ConcurrentDeclarativeSource | RunnableConnector:
116
- """Instantiate the connector class."""
117
- raise NotImplementedError("Subclasses must implement this method.")
118
-
119
- def run_test_scenario(
120
- self,
121
- verb: Literal["read", "check", "discover"],
122
- test_scenario: ConnectorTestScenario,
123
- *,
124
- catalog: dict | None = None,
125
- ) -> entrypoint_wrapper.EntrypointOutput:
126
- """Run a test job from provided CLI args and return the result."""
127
- return run_test_job(
128
- self.create_connector(test_scenario),
129
- verb,
130
- test_instance=test_scenario,
131
- catalog=catalog,
132
- )
133
-
134
- # Test Definitions
135
-
136
- def test_check(
137
- self,
138
- instance: ConnectorTestScenario,
139
- ) -> None:
140
- """Run `connection` acceptance tests."""
141
- result = self.run_test_scenario(
142
- "check",
143
- test_scenario=instance,
144
- )
145
- conn_status_messages: list[AirbyteMessage] = [
146
- msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS
147
- ] # noqa: SLF001 # Non-public API
148
- assert len(conn_status_messages) == 1, (
149
- "Expected exactly one CONNECTION_STATUS message. Got: \n" + "\n".join(result._messages)
150
- )
151
-
152
- @classmethod
153
- @property
154
- def acceptance_test_config_path(self) -> Path:
155
- """Get the path to the acceptance test config file.
156
-
157
- Check vwd and parent directories of cwd for the config file, and return the first one found.
158
-
159
- Give up if the config file is not found in any parent directory.
160
- """
161
- current_dir = Path.cwd()
162
- for parent_dir in current_dir.parents:
163
- config_path = parent_dir / ACCEPTANCE_TEST_CONFIG
164
- if config_path.exists():
165
- return config_path
166
- raise FileNotFoundError(
167
- f"Acceptance test config file not found in any parent directory from : {Path.cwd()}"
168
- )
169
-
170
- @classmethod
171
- def get_scenarios(
172
- cls,
173
- ) -> list[ConnectorTestScenario]:
174
- """Get acceptance tests for a given category.
175
-
176
- This has to be a separate function because pytest does not allow
177
- parametrization of fixtures with arguments from the test class itself.
178
- """
179
- category = "connection"
180
- all_tests_config = yaml.safe_load(cls.acceptance_test_config_path.read_text())
181
- if "acceptance_tests" not in all_tests_config:
182
- raise ValueError(
183
- f"Acceptance tests config not found in {cls.acceptance_test_config_path}."
184
- f" Found only: {str(all_tests_config)}."
185
- )
186
- if category not in all_tests_config["acceptance_tests"]:
187
- return []
188
- if "tests" not in all_tests_config["acceptance_tests"][category]:
189
- raise ValueError(f"No tests found for category {category}")
190
-
191
- tests_scenarios = [
192
- ConnectorTestScenario.model_validate(test)
193
- for test in all_tests_config["acceptance_tests"][category]["tests"]
194
- if "iam_role" not in test["config_path"]
195
- ]
196
- working_dir = cls.working_dir or Path()
197
- for test in tests_scenarios:
198
- if test.config_path:
199
- test.config_path = working_dir / test.config_path
200
- if test.configured_catalog_path:
201
- test.configured_catalog_path = working_dir / test.configured_catalog_path
202
- return tests_scenarios
@@ -1,48 +0,0 @@
1
- import os
2
- from hashlib import md5
3
- from pathlib import Path
4
- from typing import Any, cast
5
-
6
- import yaml
7
-
8
- from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
9
- ConcurrentDeclarativeSource,
10
- )
11
- from airbyte_cdk.test.declarative.models import ConnectorTestScenario
12
- from airbyte_cdk.test.declarative.test_suites.source_base import (
13
- SourceTestSuiteBase,
14
- )
15
-
16
-
17
- def md5_checksum(file_path: Path) -> str:
18
- with open(file_path, "rb") as file:
19
- return md5(file.read()).hexdigest()
20
-
21
-
22
- class DeclarativeSourceTestSuite(SourceTestSuiteBase):
23
- manifest_path = Path("manifest.yaml")
24
- components_py_path: Path | None = None
25
-
26
- def create_connector(
27
- self, connector_test: ConnectorTestScenario
28
- ) -> ConcurrentDeclarativeSource:
29
- """Create a connector instance for the test suite."""
30
- config = connector_test.get_config_dict()
31
- # catalog = connector_test.get_catalog()
32
- # state = connector_test.get_state()
33
- # source_config = connector_test.get_source_config()
34
-
35
- manifest_dict = yaml.safe_load(self.manifest_path.read_text())
36
- if self.components_py_path and self.components_py_path.exists():
37
- os.environ["AIRBYTE_ENABLE_UNSAFE_CODE"] = "true"
38
- config["__injected_components_py"] = self.components_py_path.read_text()
39
- config["__injected_components_py_checksums"] = {
40
- "md5": md5_checksum(self.components_py_path),
41
- }
42
-
43
- return ConcurrentDeclarativeSource(
44
- config=config,
45
- catalog=None,
46
- state=None,
47
- source_config=manifest_dict,
48
- )
@@ -1,12 +0,0 @@
1
- # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
2
- """Base class for destination test suites."""
3
-
4
- from airbyte_cdk.test.declarative.test_suites.connector_base import ConnectorTestSuiteBase
5
-
6
-
7
- class DestinationTestSuiteBase(ConnectorTestSuiteBase):
8
- """Base class for destination test suites.
9
-
10
- This class provides a base set of functionality for testing destination connectors, and it
11
- inherits all generic connector tests from the `ConnectorTestSuiteBase` class.
12
- """
@@ -1,129 +0,0 @@
1
- # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
2
- """Base class for source test suites."""
3
-
4
- from dataclasses import asdict
5
- from pathlib import Path
6
-
7
- import pytest
8
-
9
- from airbyte_cdk.models import (
10
- AirbyteMessage,
11
- AirbyteStream,
12
- ConfiguredAirbyteCatalog,
13
- ConfiguredAirbyteStream,
14
- DestinationSyncMode,
15
- SyncMode,
16
- Type,
17
- )
18
- from airbyte_cdk.test import entrypoint_wrapper
19
- from airbyte_cdk.test.declarative.models import (
20
- ConnectorTestScenario,
21
- )
22
- from airbyte_cdk.test.declarative.test_suites.connector_base import (
23
- ConnectorTestSuiteBase,
24
- )
25
- from airbyte_cdk.test.declarative.utils.job_runner import run_test_job
26
-
27
-
28
- class SourceTestSuiteBase(ConnectorTestSuiteBase):
29
- """Base class for source test suites.
30
-
31
- This class provides a base set of functionality for testing source connectors, and it
32
- inherits all generic connector tests from the `ConnectorTestSuiteBase` class.
33
- """
34
-
35
- def test_check(
36
- self,
37
- instance: ConnectorTestScenario,
38
- ) -> None:
39
- """Run `connection` acceptance tests."""
40
- result: entrypoint_wrapper.EntrypointOutput = run_test_job(
41
- self.create_connector(instance),
42
- "check",
43
- test_instance=instance,
44
- )
45
- conn_status_messages: list[AirbyteMessage] = [
46
- msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS
47
- ] # noqa: SLF001 # Non-public API
48
- assert len(conn_status_messages) == 1, (
49
- "Expected exactly one CONNECTION_STATUS message. Got: \n" + "\n".join(result._messages)
50
- )
51
-
52
- def test_basic_read(
53
- self,
54
- instance: ConnectorTestScenario,
55
- ) -> None:
56
- """Run acceptance tests."""
57
- discover_result = run_test_job(
58
- self.create_connector(instance),
59
- "discover",
60
- test_instance=instance,
61
- )
62
- if instance.expect_exception:
63
- assert discover_result.errors, "Expected exception but got none."
64
- return
65
-
66
- configured_catalog = ConfiguredAirbyteCatalog(
67
- streams=[
68
- ConfiguredAirbyteStream(
69
- stream=stream,
70
- sync_mode=SyncMode.full_refresh,
71
- destination_sync_mode=DestinationSyncMode.append_dedup,
72
- )
73
- for stream in discover_result.catalog.catalog.streams
74
- ]
75
- )
76
- result = run_test_job(
77
- self.create_connector(instance),
78
- "read",
79
- test_instance=instance,
80
- catalog=configured_catalog,
81
- )
82
-
83
- if not result.records:
84
- raise AssertionError("Expected records but got none.") # noqa: TRY003
85
-
86
- def test_fail_with_bad_catalog(
87
- self,
88
- instance: ConnectorTestScenario,
89
- ) -> None:
90
- """Test that a bad catalog fails."""
91
- invalid_configured_catalog = ConfiguredAirbyteCatalog(
92
- streams=[
93
- # Create ConfiguredAirbyteStream which is deliberately invalid
94
- # with regard to the Airbyte Protocol.
95
- # This should cause the connector to fail.
96
- ConfiguredAirbyteStream(
97
- stream=AirbyteStream(
98
- name="__AIRBYTE__stream_that_does_not_exist",
99
- json_schema={
100
- "type": "object",
101
- "properties": {"f1": {"type": "string"}},
102
- },
103
- supported_sync_modes=[SyncMode.full_refresh],
104
- ),
105
- sync_mode="INVALID",
106
- destination_sync_mode="INVALID",
107
- )
108
- ]
109
- )
110
- # Set expected status to "failed" to ensure the test fails if the connector.
111
- instance.status = "failed"
112
- result = self.run_test_scenario(
113
- "read",
114
- test_scenario=instance,
115
- catalog=asdict(invalid_configured_catalog),
116
- )
117
- assert result.errors, "Expected errors but got none."
118
- assert result.trace_messages, "Expected trace messages but got none."
119
-
120
- def test_discover(
121
- self,
122
- instance: ConnectorTestScenario,
123
- ) -> None:
124
- """Run acceptance tests."""
125
- run_test_job(
126
- self.create_connector(instance),
127
- "check",
128
- test_instance=instance,
129
- )
File without changes
@@ -1,128 +0,0 @@
1
- import tempfile
2
- import uuid
3
- from pathlib import Path
4
- from typing import Any, Callable, Literal
5
-
6
- import orjson
7
-
8
- from airbyte_cdk import Connector
9
- from airbyte_cdk.models import (
10
- Status,
11
- )
12
- from airbyte_cdk.sources.abstract_source import AbstractSource
13
- from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
14
- from airbyte_cdk.test import entrypoint_wrapper
15
- from airbyte_cdk.test.declarative.models import (
16
- ConnectorTestScenario,
17
- )
18
-
19
-
20
- def run_test_job(
21
- connector: Connector | type[Connector] | Callable[[], Connector],
22
- verb: Literal["read", "check", "discover"],
23
- test_instance: ConnectorTestScenario,
24
- *,
25
- catalog: dict[str, Any] | None = None,
26
- ) -> entrypoint_wrapper.EntrypointOutput:
27
- """Run a test job from provided CLI args and return the result."""
28
- if not connector:
29
- raise ValueError("Connector is required")
30
-
31
- connector_obj: Connector
32
- if isinstance(connector, type):
33
- connector_obj = connector()
34
- elif isinstance(connector, Connector):
35
- connector_obj = connector
36
- elif isinstance(connector, DeclarativeSource | AbstractSource):
37
- connector_obj = connector
38
- elif isinstance(connector, Callable):
39
- try:
40
- connector_obj = connector()
41
- except Exception as ex:
42
- if not test_instance.expect_exception:
43
- raise
44
-
45
- return entrypoint_wrapper.EntrypointOutput(
46
- messages=[],
47
- uncaught_exception=ex,
48
- )
49
- else:
50
- raise ValueError(f"Invalid source type: {type(connector)}")
51
-
52
- args: list[str] = [verb]
53
- if test_instance.config_path:
54
- args += ["--config", str(test_instance.config_path)]
55
- elif test_instance.config_dict:
56
- config_path = (
57
- Path(tempfile.gettempdir()) / "airbyte-test" / f"temp_config_{uuid.uuid4().hex}.json"
58
- )
59
- config_path.parent.mkdir(parents=True, exist_ok=True)
60
- config_path.write_text(orjson.dumps(test_instance.config_dict).decode())
61
- args += ["--config", str(config_path)]
62
-
63
- catalog_path: Path | None = None
64
- if verb not in ["discover", "check"]:
65
- # We need a catalog for read.
66
- if catalog:
67
- # Write the catalog to a temp json file and pass the path to the file as an argument.
68
- catalog_path = (
69
- Path(tempfile.gettempdir())
70
- / "airbyte-test"
71
- / f"temp_catalog_{uuid.uuid4().hex}.json"
72
- )
73
- catalog_path.parent.mkdir(parents=True, exist_ok=True)
74
- catalog_path.write_text(orjson.dumps(catalog).decode())
75
- elif test_instance.configured_catalog_path:
76
- catalog_path = Path(test_instance.configured_catalog_path)
77
-
78
- if catalog_path:
79
- args += ["--catalog", str(catalog_path)]
80
-
81
- # This is a bit of a hack because the source needs the catalog early.
82
- # Because it *also* can fail, we have ot redundantly wrap it in a try/except block.
83
-
84
- result: entrypoint_wrapper.EntrypointOutput = entrypoint_wrapper._run_command( # noqa: SLF001 # Non-public API
85
- source=connector_obj,
86
- args=args,
87
- expecting_exception=test_instance.expect_exception,
88
- )
89
- if result.errors and not test_instance.expect_exception:
90
- raise AssertionError(
91
- "\n\n".join(
92
- [str(err.trace.error).replace("\\n", "\n") for err in result.errors],
93
- )
94
- )
95
-
96
- if verb == "check":
97
- # Check is expected to fail gracefully without an exception.
98
- # Instead, we assert that we have a CONNECTION_STATUS message with
99
- # a failure status.
100
- assert not result.errors, "Expected no errors from check. Got:\n" + "\n".join(
101
- [str(error) for error in result.errors]
102
- )
103
- assert len(result.connection_status_messages) == 1, (
104
- "Expected exactly one CONNECTION_STATUS message. Got "
105
- f"{len(result.connection_status_messages)}:\n"
106
- + "\n".join([str(msg) for msg in result.connection_status_messages])
107
- )
108
- if test_instance.expect_exception:
109
- assert result.connection_status_messages[0].connectionStatus.status == Status.FAILED, (
110
- "Expected CONNECTION_STATUS message to be FAILED. Got: \n"
111
- + "\n".join([str(msg) for msg in result.connection_status_messages])
112
- )
113
- return result
114
-
115
- # For all other verbs, we assert check that an exception is raised (or not).
116
- if test_instance.expect_exception:
117
- if not result.errors:
118
- raise AssertionError("Expected exception but got none.")
119
-
120
- return result
121
- if result.errors:
122
- raise AssertionError(
123
- "\n\n".join(
124
- [str(err.trace.error).replace("\\n", "\n") for err in result.errors],
125
- )
126
- )
127
-
128
- return result
File without changes
@@ -1,14 +0,0 @@
1
- """Auto-use fixtures for pytest.
2
-
3
- WARNING: Importing this module will automatically apply these fixtures. If you want to selectively
4
- enable fixtures in a different context, you can import directly from the `fixtures.general` module.
5
-
6
-
7
- Usage:
8
-
9
- ```python
10
- from airbyte_cdk.test.fixtures import auto
11
- # OR
12
- from airbyte_cdk.test.fixtures.auto import *
13
- ```
14
- """
@@ -1,46 +0,0 @@
1
- """Global pytest configuration for the Airbyte CDK tests."""
2
-
3
- from pathlib import Path
4
- from typing import Optional
5
-
6
- import pytest
7
-
8
-
9
- def pytest_collect_file(parent: Optional[pytest.Module], path: Path) -> pytest.Module | None:
10
- """Collect test files based on their names."""
11
- if path.name == "test_connector.py":
12
- return cast(pytest.Module, pytest.Module.from_parent(parent, path=path))
13
-
14
- return None
15
-
16
-
17
- def pytest_configure(config: pytest.Config) -> None:
18
- config.addinivalue_line("markers", "connector: mark test as a connector test")
19
-
20
-
21
- def pytest_addoption(parser: pytest.Parser) -> None:
22
- parser.addoption(
23
- "--run-connector",
24
- action="store_true",
25
- default=False,
26
- help="run connector tests",
27
- )
28
-
29
-
30
- def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
31
- if config.getoption("--run-connector"):
32
- return
33
- skip_connector = pytest.mark.skip(reason="need --run-connector option to run")
34
- for item in items:
35
- if "connector" in item.keywords:
36
- item.add_marker(skip_connector)
37
-
38
-
39
- def pytest_runtest_setup(item: pytest.Item) -> None:
40
- # This hook is called before each test function is executed
41
- print(f"Setting up test: {item.name}")
42
-
43
-
44
- def pytest_runtest_teardown(item: pytest.Item, nextitem: Optional[pytest.Item]) -> None:
45
- # This hook is called after each test function is executed
46
- print(f"Tearing down test: {item.name}")