airbyte-cdk 6.48.7.dev2__py3-none-any.whl → 6.48.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/cli/airbyte_cdk/_connector.py +18 -20
- airbyte_cdk/cli/airbyte_cdk/_image.py +16 -18
- airbyte_cdk/cli/airbyte_cdk/_secrets.py +14 -33
- airbyte_cdk/destinations/destination.py +50 -78
- airbyte_cdk/models/__init__.py +0 -4
- airbyte_cdk/models/airbyte_protocol_serializers.py +3 -2
- airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py +6 -1
- airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +12 -49
- airbyte_cdk/test/catalog_builder.py +1 -9
- airbyte_cdk/test/entrypoint_wrapper.py +4 -0
- airbyte_cdk/test/mock_http/request.py +1 -5
- airbyte_cdk/test/standard_tests/_job_runner.py +9 -6
- airbyte_cdk/test/standard_tests/connector_base.py +22 -15
- airbyte_cdk/test/standard_tests/declarative_sources.py +8 -4
- airbyte_cdk/test/standard_tests/models/scenario.py +14 -3
- airbyte_cdk/test/standard_tests/source_base.py +24 -0
- airbyte_cdk/test/standard_tests/util.py +1 -1
- airbyte_cdk/utils/connector_paths.py +223 -0
- airbyte_cdk/utils/docker.py +116 -29
- {airbyte_cdk-6.48.7.dev2.dist-info → airbyte_cdk-6.48.8.dist-info}/METADATA +2 -2
- {airbyte_cdk-6.48.7.dev2.dist-info → airbyte_cdk-6.48.8.dist-info}/RECORD +25 -27
- airbyte_cdk/cli/airbyte_cdk/_util.py +0 -69
- airbyte_cdk/test/standard_tests/test_resources.py +0 -69
- airbyte_cdk/utils/docker_image_templates.py +0 -136
- {airbyte_cdk-6.48.7.dev2.dist-info → airbyte_cdk-6.48.8.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.48.7.dev2.dist-info → airbyte_cdk-6.48.8.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.48.7.dev2.dist-info → airbyte_cdk-6.48.8.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.48.7.dev2.dist-info → airbyte_cdk-6.48.8.dist-info}/entry_points.txt +0 -0
@@ -56,12 +56,15 @@ class IConnector(Protocol):
|
|
56
56
|
|
57
57
|
def run_test_job(
|
58
58
|
connector: IConnector | type[IConnector] | Callable[[], IConnector],
|
59
|
-
verb: Literal["read", "check", "discover"],
|
60
|
-
test_scenario: ConnectorTestScenario,
|
59
|
+
verb: Literal["spec", "read", "check", "discover"],
|
61
60
|
*,
|
61
|
+
test_scenario: ConnectorTestScenario | None = None,
|
62
62
|
catalog: ConfiguredAirbyteCatalog | dict[str, Any] | None = None,
|
63
63
|
) -> entrypoint_wrapper.EntrypointOutput:
|
64
64
|
"""Run a test scenario from provided CLI args and return the result."""
|
65
|
+
# Use default (empty) scenario if not provided:
|
66
|
+
test_scenario = test_scenario or ConnectorTestScenario()
|
67
|
+
|
65
68
|
if not connector:
|
66
69
|
raise ValueError("Connector is required")
|
67
70
|
|
@@ -81,14 +84,14 @@ def run_test_job(
|
|
81
84
|
)
|
82
85
|
|
83
86
|
args: list[str] = [verb]
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
+
config_dict = test_scenario.get_config_dict(empty_if_missing=True)
|
88
|
+
if config_dict and verb != "spec":
|
89
|
+
# Write the config to a temp json file and pass the path to the file as an argument.
|
87
90
|
config_path = (
|
88
91
|
Path(tempfile.gettempdir()) / "airbyte-test" / f"temp_config_{uuid.uuid4().hex}.json"
|
89
92
|
)
|
90
93
|
config_path.parent.mkdir(parents=True, exist_ok=True)
|
91
|
-
config_path.write_text(orjson.dumps(
|
94
|
+
config_path.write_text(orjson.dumps(config_dict).decode())
|
92
95
|
args += ["--config", str(config_path)]
|
93
96
|
|
94
97
|
catalog_path: Path | None = None
|
@@ -24,7 +24,7 @@ from airbyte_cdk.test.standard_tests._job_runner import IConnector, run_test_job
|
|
24
24
|
from airbyte_cdk.test.standard_tests.models import (
|
25
25
|
ConnectorTestScenario,
|
26
26
|
)
|
27
|
-
from airbyte_cdk.
|
27
|
+
from airbyte_cdk.utils.connector_paths import (
|
28
28
|
ACCEPTANCE_TEST_CONFIG,
|
29
29
|
find_connector_root,
|
30
30
|
)
|
@@ -89,7 +89,7 @@ class ConnectorTestSuiteBase(abc.ABC):
|
|
89
89
|
@classmethod
|
90
90
|
def create_connector(
|
91
91
|
cls,
|
92
|
-
scenario: ConnectorTestScenario,
|
92
|
+
scenario: ConnectorTestScenario | None,
|
93
93
|
) -> IConnector:
|
94
94
|
"""Instantiate the connector class."""
|
95
95
|
connector = cls.connector # type: ignore
|
@@ -147,28 +147,35 @@ class ConnectorTestSuiteBase(abc.ABC):
|
|
147
147
|
This has to be a separate function because pytest does not allow
|
148
148
|
parametrization of fixtures with arguments from the test class itself.
|
149
149
|
"""
|
150
|
-
|
150
|
+
categories = ["connection", "spec"]
|
151
151
|
all_tests_config = yaml.safe_load(cls.acceptance_test_config_path.read_text())
|
152
152
|
if "acceptance_tests" not in all_tests_config:
|
153
153
|
raise ValueError(
|
154
154
|
f"Acceptance tests config not found in {cls.acceptance_test_config_path}."
|
155
155
|
f" Found only: {str(all_tests_config)}."
|
156
156
|
)
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
157
|
+
|
158
|
+
test_scenarios: list[ConnectorTestScenario] = []
|
159
|
+
for category in categories:
|
160
|
+
if (
|
161
|
+
category not in all_tests_config["acceptance_tests"]
|
162
|
+
or "tests" not in all_tests_config["acceptance_tests"][category]
|
163
|
+
):
|
164
|
+
continue
|
165
|
+
|
166
|
+
test_scenarios.extend(
|
167
|
+
[
|
168
|
+
ConnectorTestScenario.model_validate(test)
|
169
|
+
for test in all_tests_config["acceptance_tests"][category]["tests"]
|
170
|
+
if "config_path" in test and "iam_role" not in test["config_path"]
|
171
|
+
]
|
172
|
+
)
|
173
|
+
|
167
174
|
connector_root = cls.get_connector_root_dir().absolute()
|
168
|
-
for test in
|
175
|
+
for test in test_scenarios:
|
169
176
|
if test.config_path:
|
170
177
|
test.config_path = connector_root / test.config_path
|
171
178
|
if test.configured_catalog_path:
|
172
179
|
test.configured_catalog_path = connector_root / test.configured_catalog_path
|
173
180
|
|
174
|
-
return
|
181
|
+
return test_scenarios
|
@@ -12,7 +12,7 @@ from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
|
|
12
12
|
from airbyte_cdk.test.standard_tests._job_runner import IConnector
|
13
13
|
from airbyte_cdk.test.standard_tests.models import ConnectorTestScenario
|
14
14
|
from airbyte_cdk.test.standard_tests.source_base import SourceTestSuiteBase
|
15
|
-
from airbyte_cdk.
|
15
|
+
from airbyte_cdk.utils.connector_paths import MANIFEST_YAML
|
16
16
|
|
17
17
|
|
18
18
|
def md5_checksum(file_path: Path) -> str:
|
@@ -64,7 +64,7 @@ class DeclarativeSourceTestSuite(SourceTestSuiteBase):
|
|
64
64
|
@classmethod
|
65
65
|
def create_connector(
|
66
66
|
cls,
|
67
|
-
scenario: ConnectorTestScenario,
|
67
|
+
scenario: ConnectorTestScenario | None,
|
68
68
|
) -> IConnector:
|
69
69
|
"""Create a connector scenario for the test suite.
|
70
70
|
|
@@ -73,9 +73,13 @@ class DeclarativeSourceTestSuite(SourceTestSuiteBase):
|
|
73
73
|
|
74
74
|
Subclasses should not need to override this method.
|
75
75
|
"""
|
76
|
-
|
77
|
-
|
76
|
+
scenario = scenario or ConnectorTestScenario() # Use default (empty) scenario if None
|
78
77
|
manifest_dict = yaml.safe_load(cls.manifest_yaml_path.read_text())
|
78
|
+
config = {
|
79
|
+
"__injected_manifest": manifest_dict,
|
80
|
+
}
|
81
|
+
config.update(scenario.get_config_dict(empty_if_missing=True))
|
82
|
+
|
79
83
|
if cls.components_py_path and cls.components_py_path.exists():
|
80
84
|
os.environ["AIRBYTE_ENABLE_UNSAFE_CODE"] = "true"
|
81
85
|
config["__injected_components_py"] = cls.components_py_path.read_text()
|
@@ -43,18 +43,29 @@ class ConnectorTestScenario(BaseModel):
|
|
43
43
|
file_types: AcceptanceTestFileTypes | None = None
|
44
44
|
status: Literal["succeed", "failed"] | None = None
|
45
45
|
|
46
|
-
def get_config_dict(
|
46
|
+
def get_config_dict(
|
47
|
+
self,
|
48
|
+
*,
|
49
|
+
empty_if_missing: bool,
|
50
|
+
) -> dict[str, Any]:
|
47
51
|
"""Return the config dictionary.
|
48
52
|
|
49
53
|
If a config dictionary has already been loaded, return it. Otherwise, load
|
50
54
|
the config file and return the dictionary.
|
55
|
+
|
56
|
+
If `self.config_dict` and `self.config_path` are both `None`:
|
57
|
+
- return an empty dictionary if `empty_if_missing` is True
|
58
|
+
- raise a ValueError if `empty_if_missing` is False
|
51
59
|
"""
|
52
|
-
if self.config_dict:
|
60
|
+
if self.config_dict is not None:
|
53
61
|
return self.config_dict
|
54
62
|
|
55
|
-
if self.config_path:
|
63
|
+
if self.config_path is not None:
|
56
64
|
return cast(dict[str, Any], yaml.safe_load(self.config_path.read_text()))
|
57
65
|
|
66
|
+
if empty_if_missing:
|
67
|
+
return {}
|
68
|
+
|
58
69
|
raise ValueError("No config dictionary or path provided.")
|
59
70
|
|
60
71
|
@property
|
@@ -64,6 +64,30 @@ class SourceTestSuiteBase(ConnectorTestSuiteBase):
|
|
64
64
|
test_scenario=scenario,
|
65
65
|
)
|
66
66
|
|
67
|
+
def test_spec(self) -> None:
|
68
|
+
"""Standard test for `spec`.
|
69
|
+
|
70
|
+
This test does not require a `scenario` input, since `spec`
|
71
|
+
does not require any inputs.
|
72
|
+
|
73
|
+
We assume `spec` should always succeed and it should always generate
|
74
|
+
a valid `SPEC` message.
|
75
|
+
|
76
|
+
Note: the parsing of messages by type also implicitly validates that
|
77
|
+
the generated `SPEC` message is valid JSON.
|
78
|
+
"""
|
79
|
+
result = run_test_job(
|
80
|
+
verb="spec",
|
81
|
+
test_scenario=None,
|
82
|
+
connector=self.create_connector(scenario=None),
|
83
|
+
)
|
84
|
+
# If an error occurs, it will be raised above.
|
85
|
+
|
86
|
+
assert len(result.spec_messages) == 1, (
|
87
|
+
"Expected exactly 1 spec message but got {len(result.spec_messages)}",
|
88
|
+
result.errors,
|
89
|
+
)
|
90
|
+
|
67
91
|
def test_basic_read(
|
68
92
|
self,
|
69
93
|
scenario: ConnectorTestScenario,
|
@@ -12,7 +12,7 @@ from airbyte_cdk.test.standard_tests.declarative_sources import (
|
|
12
12
|
)
|
13
13
|
from airbyte_cdk.test.standard_tests.destination_base import DestinationTestSuiteBase
|
14
14
|
from airbyte_cdk.test.standard_tests.source_base import SourceTestSuiteBase
|
15
|
-
from airbyte_cdk.
|
15
|
+
from airbyte_cdk.utils.connector_paths import (
|
16
16
|
METADATA_YAML,
|
17
17
|
find_connector_root_from_name,
|
18
18
|
)
|
@@ -0,0 +1,223 @@
|
|
1
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
2
|
+
"""Resources and utilities for locating Airbyte Connectors."""
|
3
|
+
|
4
|
+
from contextlib import suppress
|
5
|
+
from pathlib import Path
|
6
|
+
|
7
|
+
ACCEPTANCE_TEST_CONFIG = "acceptance-test-config.yml"
|
8
|
+
MANIFEST_YAML = "manifest.yaml"
|
9
|
+
METADATA_YAML = "metadata.yaml"
|
10
|
+
|
11
|
+
|
12
|
+
def resolve_airbyte_repo_root(
|
13
|
+
from_dir: Path,
|
14
|
+
) -> Path:
|
15
|
+
"""Resolve the Airbyte repository root directory.
|
16
|
+
|
17
|
+
This function will resolve the Airbyte repository root directory based on the
|
18
|
+
current working directory. If the current working directory is not within the
|
19
|
+
Airbyte repository, it will look for the 'airbyte' or 'airbyte-enterprise'
|
20
|
+
directory in the parent directories.
|
21
|
+
|
22
|
+
Sibling directories are also considered, so if the working directory is '~/repos/airbyte-cdk',
|
23
|
+
it will find the 'airbyte' directory in '~/repos/airbyte'. The 'airbyte' directory
|
24
|
+
will be preferred over 'airbyte-enterprise' if both are present as sibling directories and
|
25
|
+
neither is a parent directory.
|
26
|
+
|
27
|
+
If we reach the root of the filesystem without finding the 'airbyte' directory,
|
28
|
+
a FileNotFoundError will be raised.
|
29
|
+
|
30
|
+
Raises:
|
31
|
+
FileNotFoundError: If the Airbyte repository root directory cannot be found.
|
32
|
+
"""
|
33
|
+
|
34
|
+
def _is_airbyte_repo_root(path: Path) -> bool:
|
35
|
+
"""Check if the given path is the Airbyte repository root."""
|
36
|
+
return all(
|
37
|
+
[
|
38
|
+
(path.name == "airbyte" or path.name == "airbyte-enterprise"),
|
39
|
+
(path / "airbyte-integrations").is_dir(),
|
40
|
+
]
|
41
|
+
)
|
42
|
+
|
43
|
+
def _find_in_adjacent_dirs(current_dir: Path) -> Path | None:
|
44
|
+
"""Check if 'airbyte' or 'airbyte-enterprise' exists as a sibling, parent, or child."""
|
45
|
+
# Check parents
|
46
|
+
parent_dir = current_dir.parent
|
47
|
+
if _is_airbyte_repo_root(parent_dir):
|
48
|
+
return parent_dir
|
49
|
+
|
50
|
+
# Check siblings
|
51
|
+
if _is_airbyte_repo_root(parent_dir / "airbyte"):
|
52
|
+
return parent_dir / "airbyte"
|
53
|
+
if _is_airbyte_repo_root(parent_dir / "airbyte-enterprise"):
|
54
|
+
return parent_dir / "airbyte-enterprise"
|
55
|
+
|
56
|
+
# Check children only if no "airbyte" or "airbyte-enterprise" in parent
|
57
|
+
if not any(
|
58
|
+
[
|
59
|
+
"airbyte" in current_dir.parts,
|
60
|
+
"airbyte-enterprise" in current_dir.parts,
|
61
|
+
]
|
62
|
+
):
|
63
|
+
if _is_airbyte_repo_root(current_dir / "airbyte"):
|
64
|
+
return current_dir / "airbyte"
|
65
|
+
if _is_airbyte_repo_root(current_dir / "airbyte-enterprise"):
|
66
|
+
return current_dir / "airbyte-enterprise"
|
67
|
+
|
68
|
+
return None
|
69
|
+
|
70
|
+
current_dir = from_dir.resolve().absolute()
|
71
|
+
while current_dir != current_dir.parent: # abort when we reach file system root
|
72
|
+
found_dir = _find_in_adjacent_dirs(current_dir)
|
73
|
+
if found_dir:
|
74
|
+
return found_dir
|
75
|
+
|
76
|
+
# Move up one directory
|
77
|
+
current_dir = current_dir.parent
|
78
|
+
|
79
|
+
raise FileNotFoundError(
|
80
|
+
f"Could not find the Airbyte repository root directory. Current directory: {from_dir}"
|
81
|
+
)
|
82
|
+
|
83
|
+
|
84
|
+
def resolve_connector_name_and_directory(
|
85
|
+
connector_ref: str | Path | None = None,
|
86
|
+
*,
|
87
|
+
connector_directory: Path | None = None,
|
88
|
+
) -> tuple[str, Path]:
|
89
|
+
"""Resolve the connector name and directory.
|
90
|
+
|
91
|
+
This function will resolve the connector name and directory based on the provided
|
92
|
+
reference. If no input ref is provided, it will look within the
|
93
|
+
current working directory. If the current working directory is not a connector
|
94
|
+
directory (e.g. starting with 'source-') and no connector name or path is provided,
|
95
|
+
the process will fail.
|
96
|
+
If ref is sent as a string containing "/" or "\\", it will be treated as a path to the
|
97
|
+
connector directory.
|
98
|
+
|
99
|
+
raises:
|
100
|
+
ValueError: If the connector name or directory cannot be resolved.
|
101
|
+
FileNotFoundError: If the connector directory does not exist or cannot be found.
|
102
|
+
"""
|
103
|
+
connector_name: str | None = None
|
104
|
+
|
105
|
+
# Resolve connector_ref to connector_name or connector_directory (if provided)
|
106
|
+
if connector_ref:
|
107
|
+
if isinstance(connector_ref, str):
|
108
|
+
if "/" in connector_ref or "\\" in connector_ref:
|
109
|
+
# If the connector name is a path, treat it as a directory
|
110
|
+
connector_directory = Path(connector_ref)
|
111
|
+
else:
|
112
|
+
# Otherwise, treat it as a connector name
|
113
|
+
connector_name = connector_ref
|
114
|
+
elif isinstance(connector_ref, Path):
|
115
|
+
connector_directory = connector_ref
|
116
|
+
else:
|
117
|
+
raise ValueError(
|
118
|
+
"connector_ref must be a string or Path. "
|
119
|
+
f"Received type '{type(connector_ref).__name__}': {connector_ref!r}",
|
120
|
+
)
|
121
|
+
|
122
|
+
if not connector_directory:
|
123
|
+
if connector_name:
|
124
|
+
connector_directory = find_connector_root_from_name(connector_name)
|
125
|
+
else:
|
126
|
+
cwd = Path().resolve().absolute()
|
127
|
+
if cwd.name.startswith("source-") or cwd.name.startswith("destination-"):
|
128
|
+
connector_directory = cwd
|
129
|
+
else:
|
130
|
+
raise ValueError(
|
131
|
+
"The 'connector' input must be provided if not "
|
132
|
+
"running from a connector directory. "
|
133
|
+
f"Could not infer connector directory from: {cwd}"
|
134
|
+
)
|
135
|
+
|
136
|
+
if not connector_name:
|
137
|
+
connector_name = connector_directory.name
|
138
|
+
|
139
|
+
if connector_directory:
|
140
|
+
connector_directory = connector_directory.resolve().absolute()
|
141
|
+
elif connector_name:
|
142
|
+
connector_directory = find_connector_root_from_name(connector_name)
|
143
|
+
else:
|
144
|
+
raise ValueError(
|
145
|
+
f"Could not infer connector_name or connector_directory from input ref: {connector_ref}",
|
146
|
+
)
|
147
|
+
|
148
|
+
return connector_name, connector_directory
|
149
|
+
|
150
|
+
|
151
|
+
def resolve_connector_name(
|
152
|
+
connector_directory: Path,
|
153
|
+
) -> str:
|
154
|
+
"""Resolve the connector name.
|
155
|
+
|
156
|
+
This function will resolve the connector name based on the provided connector directory.
|
157
|
+
If the current working directory is not a connector directory
|
158
|
+
(e.g. starting with 'source-'), the process will fail.
|
159
|
+
|
160
|
+
Raises:
|
161
|
+
FileNotFoundError: If the connector directory does not exist or cannot be found.
|
162
|
+
"""
|
163
|
+
if not connector_directory:
|
164
|
+
raise FileNotFoundError(
|
165
|
+
"Connector directory does not exist or cannot be found. Please provide a valid "
|
166
|
+
"connector directory."
|
167
|
+
)
|
168
|
+
connector_name = connector_directory.absolute().name
|
169
|
+
if not connector_name.startswith("source-") and not connector_name.startswith("destination-"):
|
170
|
+
raise ValueError(
|
171
|
+
f"Connector directory '{connector_name}' does not look like a valid connector directory. "
|
172
|
+
f"Full path: {connector_directory.absolute()}"
|
173
|
+
)
|
174
|
+
return connector_name
|
175
|
+
|
176
|
+
|
177
|
+
def find_connector_root(from_paths: list[Path]) -> Path:
|
178
|
+
"""Find the root directory of the connector."""
|
179
|
+
for path in from_paths:
|
180
|
+
# If we reach here, we didn't find the manifest file in any parent directory
|
181
|
+
# Check if the manifest file exists in the current directory
|
182
|
+
for parent in [path, *path.parents]:
|
183
|
+
if (parent / METADATA_YAML).exists():
|
184
|
+
return parent
|
185
|
+
if (parent / MANIFEST_YAML).exists():
|
186
|
+
return parent
|
187
|
+
if (parent / ACCEPTANCE_TEST_CONFIG).exists():
|
188
|
+
return parent
|
189
|
+
if parent.name == "airbyte_cdk":
|
190
|
+
break
|
191
|
+
|
192
|
+
raise FileNotFoundError(
|
193
|
+
"Could not find connector root directory relative to the provided directories: "
|
194
|
+
f"'{str(from_paths)}'."
|
195
|
+
)
|
196
|
+
|
197
|
+
|
198
|
+
def find_connector_root_from_name(connector_name: str) -> Path:
|
199
|
+
"""Find the root directory of the connector from its name."""
|
200
|
+
with suppress(FileNotFoundError):
|
201
|
+
return find_connector_root([Path(connector_name)])
|
202
|
+
|
203
|
+
# If the connector name is not found, check if we are in the airbyte monorepo
|
204
|
+
# and try to find the connector root from the current directory.
|
205
|
+
|
206
|
+
cwd: Path = Path().absolute()
|
207
|
+
|
208
|
+
try:
|
209
|
+
airbyte_repo_root: Path = resolve_airbyte_repo_root(cwd)
|
210
|
+
except FileNotFoundError as ex:
|
211
|
+
raise FileNotFoundError(
|
212
|
+
"Could not find connector root directory relative and we are not in the airbyte repo."
|
213
|
+
) from ex
|
214
|
+
|
215
|
+
expected_connector_dir: Path = (
|
216
|
+
airbyte_repo_root / "airbyte-integrations" / "connectors" / connector_name
|
217
|
+
)
|
218
|
+
if not expected_connector_dir.exists():
|
219
|
+
raise FileNotFoundError(
|
220
|
+
f"Could not find connector directory '{expected_connector_dir}' relative to the airbyte repo."
|
221
|
+
)
|
222
|
+
|
223
|
+
return expected_connector_dir
|
airbyte_cdk/utils/docker.py
CHANGED
@@ -12,14 +12,10 @@ from enum import Enum
|
|
12
12
|
from pathlib import Path
|
13
13
|
|
14
14
|
import click
|
15
|
+
import requests
|
15
16
|
|
16
17
|
from airbyte_cdk.models.connector_metadata import ConnectorLanguage, MetadataFile
|
17
|
-
from airbyte_cdk.utils.
|
18
|
-
DOCKERIGNORE_TEMPLATE,
|
19
|
-
JAVA_CONNECTOR_DOCKERFILE_TEMPLATE,
|
20
|
-
MANIFEST_ONLY_DOCKERFILE_TEMPLATE,
|
21
|
-
PYTHON_CONNECTOR_DOCKERFILE_TEMPLATE,
|
22
|
-
)
|
18
|
+
from airbyte_cdk.utils.connector_paths import resolve_airbyte_repo_root
|
23
19
|
|
24
20
|
|
25
21
|
@dataclass(kw_only=True)
|
@@ -145,6 +141,7 @@ def build_connector_image(
|
|
145
141
|
tag: str,
|
146
142
|
primary_arch: ArchEnum = ArchEnum.ARM64, # Assume MacBook M series by default
|
147
143
|
no_verify: bool = False,
|
144
|
+
dockerfile_override: Path | None = None,
|
148
145
|
) -> None:
|
149
146
|
"""Build a connector Docker image.
|
150
147
|
|
@@ -167,10 +164,36 @@ def build_connector_image(
|
|
167
164
|
ConnectorImageBuildError: If the image build or tag operation fails.
|
168
165
|
"""
|
169
166
|
connector_kebab_name = connector_name
|
170
|
-
connector_snake_name = connector_kebab_name.replace("-", "_")
|
171
167
|
|
172
|
-
|
173
|
-
|
168
|
+
if dockerfile_override:
|
169
|
+
dockerfile_path = dockerfile_override
|
170
|
+
else:
|
171
|
+
dockerfile_path = connector_directory / "build" / "docker" / "Dockerfile"
|
172
|
+
dockerignore_path = connector_directory / "build" / "docker" / "Dockerfile.dockerignore"
|
173
|
+
try:
|
174
|
+
dockerfile_text, dockerignore_text = get_dockerfile_templates(
|
175
|
+
metadata=metadata,
|
176
|
+
connector_directory=connector_directory,
|
177
|
+
)
|
178
|
+
except FileNotFoundError:
|
179
|
+
# If the Dockerfile and .dockerignore are not found in the connector directory,
|
180
|
+
# download the templates from the Airbyte repo. This is a fallback
|
181
|
+
# in case the Airbyte repo not checked out locally.
|
182
|
+
try:
|
183
|
+
dockerfile_text, dockerignore_text = _download_dockerfile_defs(
|
184
|
+
connector_language=metadata.data.language,
|
185
|
+
)
|
186
|
+
except requests.HTTPError as e:
|
187
|
+
raise ConnectorImageBuildError(
|
188
|
+
build_args=[],
|
189
|
+
error_text=(
|
190
|
+
"Could not locate local dockerfile templates and "
|
191
|
+
f"failed to download Dockerfile templates from github: {e}"
|
192
|
+
),
|
193
|
+
) from e
|
194
|
+
|
195
|
+
dockerfile_path.write_text(dockerfile_text)
|
196
|
+
dockerignore_path.write_text(dockerignore_text)
|
174
197
|
|
175
198
|
extra_build_script: str = ""
|
176
199
|
build_customization_path = connector_directory / "build_customization.py"
|
@@ -185,14 +208,9 @@ def build_connector_image(
|
|
185
208
|
)
|
186
209
|
|
187
210
|
base_image = metadata.data.connectorBuildOptions.baseImage
|
188
|
-
|
189
|
-
dockerfile_path.write_text(get_dockerfile_template(metadata))
|
190
|
-
dockerignore_path.write_text(DOCKERIGNORE_TEMPLATE)
|
191
|
-
|
192
211
|
build_args: dict[str, str | None] = {
|
193
212
|
"BASE_IMAGE": base_image,
|
194
|
-
"
|
195
|
-
"CONNECTOR_KEBAB_NAME": connector_kebab_name,
|
213
|
+
"CONNECTOR_NAME": connector_kebab_name,
|
196
214
|
"EXTRA_BUILD_SCRIPT": extra_build_script,
|
197
215
|
}
|
198
216
|
|
@@ -246,31 +264,100 @@ def build_connector_image(
|
|
246
264
|
sys.exit(0)
|
247
265
|
|
248
266
|
|
249
|
-
def
|
267
|
+
def _download_dockerfile_defs(
|
268
|
+
connector_language: ConnectorLanguage,
|
269
|
+
) -> tuple[str, str]:
|
270
|
+
"""Download the Dockerfile and .dockerignore templates for the specified connector language.
|
271
|
+
|
272
|
+
We use the requests library to download from the master branch hosted on GitHub.
|
273
|
+
|
274
|
+
Args:
|
275
|
+
connector_language: The language of the connector.
|
276
|
+
|
277
|
+
Returns:
|
278
|
+
A tuple containing the Dockerfile and .dockerignore templates as strings.
|
279
|
+
|
280
|
+
Raises:
|
281
|
+
ValueError: If the connector language is not supported.
|
282
|
+
requests.HTTPError: If the download fails.
|
283
|
+
"""
|
284
|
+
print("Downloading Dockerfile and .dockerignore templates from GitHub...")
|
285
|
+
# Map ConnectorLanguage to template directory
|
286
|
+
language_to_template_suffix = {
|
287
|
+
ConnectorLanguage.PYTHON: "python-connector",
|
288
|
+
ConnectorLanguage.JAVA: "java-connector",
|
289
|
+
ConnectorLanguage.MANIFEST_ONLY: "manifest-only-connector",
|
290
|
+
}
|
291
|
+
|
292
|
+
if connector_language not in language_to_template_suffix:
|
293
|
+
raise ValueError(f"Unsupported connector language: {connector_language}")
|
294
|
+
|
295
|
+
template_suffix = language_to_template_suffix[connector_language]
|
296
|
+
base_url = f"https://github.com/airbytehq/airbyte/raw/master/docker-images/"
|
297
|
+
|
298
|
+
dockerfile_url = f"{base_url}/Dockerfile.{template_suffix}"
|
299
|
+
dockerignore_url = f"{base_url}/Dockerfile.{template_suffix}.dockerignore"
|
300
|
+
|
301
|
+
dockerfile_resp = requests.get(dockerfile_url)
|
302
|
+
dockerfile_resp.raise_for_status()
|
303
|
+
dockerfile_text = dockerfile_resp.text
|
304
|
+
|
305
|
+
dockerignore_resp = requests.get(dockerignore_url)
|
306
|
+
dockerignore_resp.raise_for_status()
|
307
|
+
dockerignore_text = dockerignore_resp.text
|
308
|
+
|
309
|
+
return dockerfile_text, dockerignore_text
|
310
|
+
|
311
|
+
|
312
|
+
def get_dockerfile_templates(
|
250
313
|
metadata: MetadataFile,
|
251
|
-
|
314
|
+
connector_directory: Path,
|
315
|
+
) -> tuple[str, str]:
|
252
316
|
"""Get the Dockerfile template for the connector.
|
253
317
|
|
254
318
|
Args:
|
255
319
|
metadata: The metadata of the connector.
|
256
320
|
connector_name: The name of the connector.
|
257
321
|
|
322
|
+
Raises:
|
323
|
+
ValueError: If the connector language is not supported.
|
324
|
+
FileNotFoundError: If the Dockerfile or .dockerignore is not found.
|
325
|
+
|
258
326
|
Returns:
|
259
|
-
|
327
|
+
A tuple containing the Dockerfile and .dockerignore templates as strings.
|
260
328
|
"""
|
261
|
-
if metadata.data.language
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
329
|
+
if metadata.data.language not in [
|
330
|
+
ConnectorLanguage.PYTHON,
|
331
|
+
ConnectorLanguage.MANIFEST_ONLY,
|
332
|
+
ConnectorLanguage.JAVA,
|
333
|
+
]:
|
334
|
+
raise ValueError(
|
335
|
+
f"Unsupported connector language: {metadata.data.language}. "
|
336
|
+
"Please check the connector's metadata file."
|
337
|
+
)
|
269
338
|
|
270
|
-
|
271
|
-
|
272
|
-
|
339
|
+
airbyte_repo_root = resolve_airbyte_repo_root(
|
340
|
+
from_dir=connector_directory,
|
341
|
+
)
|
342
|
+
# airbyte_repo_root successfully resolved
|
343
|
+
dockerfile_path = (
|
344
|
+
airbyte_repo_root / "docker-images" / f"Dockerfile.{metadata.data.language.value}-connector"
|
273
345
|
)
|
346
|
+
dockerignore_path = (
|
347
|
+
airbyte_repo_root
|
348
|
+
/ "docker-images"
|
349
|
+
/ f"Dockerfile.{metadata.data.language.value}-connector.dockerignore"
|
350
|
+
)
|
351
|
+
if not dockerfile_path.exists():
|
352
|
+
raise FileNotFoundError(
|
353
|
+
f"Dockerfile for {metadata.data.language.value} connector not found at {dockerfile_path}"
|
354
|
+
)
|
355
|
+
if not dockerignore_path.exists():
|
356
|
+
raise FileNotFoundError(
|
357
|
+
f".dockerignore for {metadata.data.language.value} connector not found at {dockerignore_path}"
|
358
|
+
)
|
359
|
+
|
360
|
+
return dockerfile_path.read_text(), dockerignore_path.read_text()
|
274
361
|
|
275
362
|
|
276
363
|
def run_docker_command(
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: airbyte-cdk
|
3
|
-
Version: 6.48.
|
3
|
+
Version: 6.48.8
|
4
4
|
Summary: A framework for writing Airbyte Connectors.
|
5
5
|
Home-page: https://airbyte.com
|
6
6
|
License: MIT
|
@@ -23,7 +23,7 @@ Provides-Extra: sql
|
|
23
23
|
Provides-Extra: vector-db-based
|
24
24
|
Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
|
25
25
|
Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
|
26
|
-
Requires-Dist: airbyte-protocol-models-dataclasses (
|
26
|
+
Requires-Dist: airbyte-protocol-models-dataclasses (>=0.15,<0.16)
|
27
27
|
Requires-Dist: anyascii (>=0.3.2,<0.4.0)
|
28
28
|
Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
|
29
29
|
Requires-Dist: backoff
|