airbyte-cdk 6.48.7.dev2__py3-none-any.whl → 6.48.8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. airbyte_cdk/cli/airbyte_cdk/_connector.py +18 -20
  2. airbyte_cdk/cli/airbyte_cdk/_image.py +16 -18
  3. airbyte_cdk/cli/airbyte_cdk/_secrets.py +14 -33
  4. airbyte_cdk/destinations/destination.py +50 -78
  5. airbyte_cdk/models/__init__.py +0 -4
  6. airbyte_cdk/models/airbyte_protocol_serializers.py +3 -2
  7. airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py +6 -1
  8. airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +12 -49
  9. airbyte_cdk/test/catalog_builder.py +1 -9
  10. airbyte_cdk/test/entrypoint_wrapper.py +4 -0
  11. airbyte_cdk/test/mock_http/request.py +1 -5
  12. airbyte_cdk/test/standard_tests/_job_runner.py +9 -6
  13. airbyte_cdk/test/standard_tests/connector_base.py +22 -15
  14. airbyte_cdk/test/standard_tests/declarative_sources.py +8 -4
  15. airbyte_cdk/test/standard_tests/models/scenario.py +14 -3
  16. airbyte_cdk/test/standard_tests/source_base.py +24 -0
  17. airbyte_cdk/test/standard_tests/util.py +1 -1
  18. airbyte_cdk/utils/connector_paths.py +223 -0
  19. airbyte_cdk/utils/docker.py +116 -29
  20. {airbyte_cdk-6.48.7.dev2.dist-info → airbyte_cdk-6.48.8.dist-info}/METADATA +2 -2
  21. {airbyte_cdk-6.48.7.dev2.dist-info → airbyte_cdk-6.48.8.dist-info}/RECORD +25 -27
  22. airbyte_cdk/cli/airbyte_cdk/_util.py +0 -69
  23. airbyte_cdk/test/standard_tests/test_resources.py +0 -69
  24. airbyte_cdk/utils/docker_image_templates.py +0 -136
  25. {airbyte_cdk-6.48.7.dev2.dist-info → airbyte_cdk-6.48.8.dist-info}/LICENSE.txt +0 -0
  26. {airbyte_cdk-6.48.7.dev2.dist-info → airbyte_cdk-6.48.8.dist-info}/LICENSE_SHORT +0 -0
  27. {airbyte_cdk-6.48.7.dev2.dist-info → airbyte_cdk-6.48.8.dist-info}/WHEEL +0 -0
  28. {airbyte_cdk-6.48.7.dev2.dist-info → airbyte_cdk-6.48.8.dist-info}/entry_points.txt +0 -0
@@ -56,12 +56,15 @@ class IConnector(Protocol):
56
56
 
57
57
  def run_test_job(
58
58
  connector: IConnector | type[IConnector] | Callable[[], IConnector],
59
- verb: Literal["read", "check", "discover"],
60
- test_scenario: ConnectorTestScenario,
59
+ verb: Literal["spec", "read", "check", "discover"],
61
60
  *,
61
+ test_scenario: ConnectorTestScenario | None = None,
62
62
  catalog: ConfiguredAirbyteCatalog | dict[str, Any] | None = None,
63
63
  ) -> entrypoint_wrapper.EntrypointOutput:
64
64
  """Run a test scenario from provided CLI args and return the result."""
65
+ # Use default (empty) scenario if not provided:
66
+ test_scenario = test_scenario or ConnectorTestScenario()
67
+
65
68
  if not connector:
66
69
  raise ValueError("Connector is required")
67
70
 
@@ -81,14 +84,14 @@ def run_test_job(
81
84
  )
82
85
 
83
86
  args: list[str] = [verb]
84
- if test_scenario.config_path:
85
- args += ["--config", str(test_scenario.config_path)]
86
- elif test_scenario.config_dict:
87
+ config_dict = test_scenario.get_config_dict(empty_if_missing=True)
88
+ if config_dict and verb != "spec":
89
+ # Write the config to a temp json file and pass the path to the file as an argument.
87
90
  config_path = (
88
91
  Path(tempfile.gettempdir()) / "airbyte-test" / f"temp_config_{uuid.uuid4().hex}.json"
89
92
  )
90
93
  config_path.parent.mkdir(parents=True, exist_ok=True)
91
- config_path.write_text(orjson.dumps(test_scenario.config_dict).decode())
94
+ config_path.write_text(orjson.dumps(config_dict).decode())
92
95
  args += ["--config", str(config_path)]
93
96
 
94
97
  catalog_path: Path | None = None
@@ -24,7 +24,7 @@ from airbyte_cdk.test.standard_tests._job_runner import IConnector, run_test_job
24
24
  from airbyte_cdk.test.standard_tests.models import (
25
25
  ConnectorTestScenario,
26
26
  )
27
- from airbyte_cdk.test.standard_tests.test_resources import (
27
+ from airbyte_cdk.utils.connector_paths import (
28
28
  ACCEPTANCE_TEST_CONFIG,
29
29
  find_connector_root,
30
30
  )
@@ -89,7 +89,7 @@ class ConnectorTestSuiteBase(abc.ABC):
89
89
  @classmethod
90
90
  def create_connector(
91
91
  cls,
92
- scenario: ConnectorTestScenario,
92
+ scenario: ConnectorTestScenario | None,
93
93
  ) -> IConnector:
94
94
  """Instantiate the connector class."""
95
95
  connector = cls.connector # type: ignore
@@ -147,28 +147,35 @@ class ConnectorTestSuiteBase(abc.ABC):
147
147
  This has to be a separate function because pytest does not allow
148
148
  parametrization of fixtures with arguments from the test class itself.
149
149
  """
150
- category = "connection"
150
+ categories = ["connection", "spec"]
151
151
  all_tests_config = yaml.safe_load(cls.acceptance_test_config_path.read_text())
152
152
  if "acceptance_tests" not in all_tests_config:
153
153
  raise ValueError(
154
154
  f"Acceptance tests config not found in {cls.acceptance_test_config_path}."
155
155
  f" Found only: {str(all_tests_config)}."
156
156
  )
157
- if category not in all_tests_config["acceptance_tests"]:
158
- return []
159
- if "tests" not in all_tests_config["acceptance_tests"][category]:
160
- raise ValueError(f"No tests found for category {category}")
161
-
162
- tests_scenarios = [
163
- ConnectorTestScenario.model_validate(test)
164
- for test in all_tests_config["acceptance_tests"][category]["tests"]
165
- if "iam_role" not in test["config_path"]
166
- ]
157
+
158
+ test_scenarios: list[ConnectorTestScenario] = []
159
+ for category in categories:
160
+ if (
161
+ category not in all_tests_config["acceptance_tests"]
162
+ or "tests" not in all_tests_config["acceptance_tests"][category]
163
+ ):
164
+ continue
165
+
166
+ test_scenarios.extend(
167
+ [
168
+ ConnectorTestScenario.model_validate(test)
169
+ for test in all_tests_config["acceptance_tests"][category]["tests"]
170
+ if "config_path" in test and "iam_role" not in test["config_path"]
171
+ ]
172
+ )
173
+
167
174
  connector_root = cls.get_connector_root_dir().absolute()
168
- for test in tests_scenarios:
175
+ for test in test_scenarios:
169
176
  if test.config_path:
170
177
  test.config_path = connector_root / test.config_path
171
178
  if test.configured_catalog_path:
172
179
  test.configured_catalog_path = connector_root / test.configured_catalog_path
173
180
 
174
- return tests_scenarios
181
+ return test_scenarios
@@ -12,7 +12,7 @@ from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
12
12
  from airbyte_cdk.test.standard_tests._job_runner import IConnector
13
13
  from airbyte_cdk.test.standard_tests.models import ConnectorTestScenario
14
14
  from airbyte_cdk.test.standard_tests.source_base import SourceTestSuiteBase
15
- from airbyte_cdk.test.standard_tests.test_resources import MANIFEST_YAML
15
+ from airbyte_cdk.utils.connector_paths import MANIFEST_YAML
16
16
 
17
17
 
18
18
  def md5_checksum(file_path: Path) -> str:
@@ -64,7 +64,7 @@ class DeclarativeSourceTestSuite(SourceTestSuiteBase):
64
64
  @classmethod
65
65
  def create_connector(
66
66
  cls,
67
- scenario: ConnectorTestScenario,
67
+ scenario: ConnectorTestScenario | None,
68
68
  ) -> IConnector:
69
69
  """Create a connector scenario for the test suite.
70
70
 
@@ -73,9 +73,13 @@ class DeclarativeSourceTestSuite(SourceTestSuiteBase):
73
73
 
74
74
  Subclasses should not need to override this method.
75
75
  """
76
- config: dict[str, Any] = scenario.get_config_dict()
77
-
76
+ scenario = scenario or ConnectorTestScenario() # Use default (empty) scenario if None
78
77
  manifest_dict = yaml.safe_load(cls.manifest_yaml_path.read_text())
78
+ config = {
79
+ "__injected_manifest": manifest_dict,
80
+ }
81
+ config.update(scenario.get_config_dict(empty_if_missing=True))
82
+
79
83
  if cls.components_py_path and cls.components_py_path.exists():
80
84
  os.environ["AIRBYTE_ENABLE_UNSAFE_CODE"] = "true"
81
85
  config["__injected_components_py"] = cls.components_py_path.read_text()
@@ -43,18 +43,29 @@ class ConnectorTestScenario(BaseModel):
43
43
  file_types: AcceptanceTestFileTypes | None = None
44
44
  status: Literal["succeed", "failed"] | None = None
45
45
 
46
- def get_config_dict(self) -> dict[str, Any]:
46
+ def get_config_dict(
47
+ self,
48
+ *,
49
+ empty_if_missing: bool,
50
+ ) -> dict[str, Any]:
47
51
  """Return the config dictionary.
48
52
 
49
53
  If a config dictionary has already been loaded, return it. Otherwise, load
50
54
  the config file and return the dictionary.
55
+
56
+ If `self.config_dict` and `self.config_path` are both `None`:
57
+ - return an empty dictionary if `empty_if_missing` is True
58
+ - raise a ValueError if `empty_if_missing` is False
51
59
  """
52
- if self.config_dict:
60
+ if self.config_dict is not None:
53
61
  return self.config_dict
54
62
 
55
- if self.config_path:
63
+ if self.config_path is not None:
56
64
  return cast(dict[str, Any], yaml.safe_load(self.config_path.read_text()))
57
65
 
66
+ if empty_if_missing:
67
+ return {}
68
+
58
69
  raise ValueError("No config dictionary or path provided.")
59
70
 
60
71
  @property
@@ -64,6 +64,30 @@ class SourceTestSuiteBase(ConnectorTestSuiteBase):
64
64
  test_scenario=scenario,
65
65
  )
66
66
 
67
+ def test_spec(self) -> None:
68
+ """Standard test for `spec`.
69
+
70
+ This test does not require a `scenario` input, since `spec`
71
+ does not require any inputs.
72
+
73
+ We assume `spec` should always succeed and it should always generate
74
+ a valid `SPEC` message.
75
+
76
+ Note: the parsing of messages by type also implicitly validates that
77
+ the generated `SPEC` message is valid JSON.
78
+ """
79
+ result = run_test_job(
80
+ verb="spec",
81
+ test_scenario=None,
82
+ connector=self.create_connector(scenario=None),
83
+ )
84
+ # If an error occurs, it will be raised above.
85
+
86
+ assert len(result.spec_messages) == 1, (
87
+ "Expected exactly 1 spec message but got {len(result.spec_messages)}",
88
+ result.errors,
89
+ )
90
+
67
91
  def test_basic_read(
68
92
  self,
69
93
  scenario: ConnectorTestScenario,
@@ -12,7 +12,7 @@ from airbyte_cdk.test.standard_tests.declarative_sources import (
12
12
  )
13
13
  from airbyte_cdk.test.standard_tests.destination_base import DestinationTestSuiteBase
14
14
  from airbyte_cdk.test.standard_tests.source_base import SourceTestSuiteBase
15
- from airbyte_cdk.test.standard_tests.test_resources import (
15
+ from airbyte_cdk.utils.connector_paths import (
16
16
  METADATA_YAML,
17
17
  find_connector_root_from_name,
18
18
  )
@@ -0,0 +1,223 @@
1
+ # Copyright (c) 2025 Airbyte, Inc., all rights reserved.
2
+ """Resources and utilities for locating Airbyte Connectors."""
3
+
4
+ from contextlib import suppress
5
+ from pathlib import Path
6
+
7
+ ACCEPTANCE_TEST_CONFIG = "acceptance-test-config.yml"
8
+ MANIFEST_YAML = "manifest.yaml"
9
+ METADATA_YAML = "metadata.yaml"
10
+
11
+
12
+ def resolve_airbyte_repo_root(
13
+ from_dir: Path,
14
+ ) -> Path:
15
+ """Resolve the Airbyte repository root directory.
16
+
17
+ This function will resolve the Airbyte repository root directory based on the
18
+ current working directory. If the current working directory is not within the
19
+ Airbyte repository, it will look for the 'airbyte' or 'airbyte-enterprise'
20
+ directory in the parent directories.
21
+
22
+ Sibling directories are also considered, so if the working directory is '~/repos/airbyte-cdk',
23
+ it will find the 'airbyte' directory in '~/repos/airbyte'. The 'airbyte' directory
24
+ will be preferred over 'airbyte-enterprise' if both are present as sibling directories and
25
+ neither is a parent directory.
26
+
27
+ If we reach the root of the filesystem without finding the 'airbyte' directory,
28
+ a FileNotFoundError will be raised.
29
+
30
+ Raises:
31
+ FileNotFoundError: If the Airbyte repository root directory cannot be found.
32
+ """
33
+
34
+ def _is_airbyte_repo_root(path: Path) -> bool:
35
+ """Check if the given path is the Airbyte repository root."""
36
+ return all(
37
+ [
38
+ (path.name == "airbyte" or path.name == "airbyte-enterprise"),
39
+ (path / "airbyte-integrations").is_dir(),
40
+ ]
41
+ )
42
+
43
+ def _find_in_adjacent_dirs(current_dir: Path) -> Path | None:
44
+ """Check if 'airbyte' or 'airbyte-enterprise' exists as a sibling, parent, or child."""
45
+ # Check parents
46
+ parent_dir = current_dir.parent
47
+ if _is_airbyte_repo_root(parent_dir):
48
+ return parent_dir
49
+
50
+ # Check siblings
51
+ if _is_airbyte_repo_root(parent_dir / "airbyte"):
52
+ return parent_dir / "airbyte"
53
+ if _is_airbyte_repo_root(parent_dir / "airbyte-enterprise"):
54
+ return parent_dir / "airbyte-enterprise"
55
+
56
+ # Check children only if no "airbyte" or "airbyte-enterprise" in parent
57
+ if not any(
58
+ [
59
+ "airbyte" in current_dir.parts,
60
+ "airbyte-enterprise" in current_dir.parts,
61
+ ]
62
+ ):
63
+ if _is_airbyte_repo_root(current_dir / "airbyte"):
64
+ return current_dir / "airbyte"
65
+ if _is_airbyte_repo_root(current_dir / "airbyte-enterprise"):
66
+ return current_dir / "airbyte-enterprise"
67
+
68
+ return None
69
+
70
+ current_dir = from_dir.resolve().absolute()
71
+ while current_dir != current_dir.parent: # abort when we reach file system root
72
+ found_dir = _find_in_adjacent_dirs(current_dir)
73
+ if found_dir:
74
+ return found_dir
75
+
76
+ # Move up one directory
77
+ current_dir = current_dir.parent
78
+
79
+ raise FileNotFoundError(
80
+ f"Could not find the Airbyte repository root directory. Current directory: {from_dir}"
81
+ )
82
+
83
+
84
+ def resolve_connector_name_and_directory(
85
+ connector_ref: str | Path | None = None,
86
+ *,
87
+ connector_directory: Path | None = None,
88
+ ) -> tuple[str, Path]:
89
+ """Resolve the connector name and directory.
90
+
91
+ This function will resolve the connector name and directory based on the provided
92
+ reference. If no input ref is provided, it will look within the
93
+ current working directory. If the current working directory is not a connector
94
+ directory (e.g. starting with 'source-') and no connector name or path is provided,
95
+ the process will fail.
96
+ If ref is sent as a string containing "/" or "\\", it will be treated as a path to the
97
+ connector directory.
98
+
99
+ raises:
100
+ ValueError: If the connector name or directory cannot be resolved.
101
+ FileNotFoundError: If the connector directory does not exist or cannot be found.
102
+ """
103
+ connector_name: str | None = None
104
+
105
+ # Resolve connector_ref to connector_name or connector_directory (if provided)
106
+ if connector_ref:
107
+ if isinstance(connector_ref, str):
108
+ if "/" in connector_ref or "\\" in connector_ref:
109
+ # If the connector name is a path, treat it as a directory
110
+ connector_directory = Path(connector_ref)
111
+ else:
112
+ # Otherwise, treat it as a connector name
113
+ connector_name = connector_ref
114
+ elif isinstance(connector_ref, Path):
115
+ connector_directory = connector_ref
116
+ else:
117
+ raise ValueError(
118
+ "connector_ref must be a string or Path. "
119
+ f"Received type '{type(connector_ref).__name__}': {connector_ref!r}",
120
+ )
121
+
122
+ if not connector_directory:
123
+ if connector_name:
124
+ connector_directory = find_connector_root_from_name(connector_name)
125
+ else:
126
+ cwd = Path().resolve().absolute()
127
+ if cwd.name.startswith("source-") or cwd.name.startswith("destination-"):
128
+ connector_directory = cwd
129
+ else:
130
+ raise ValueError(
131
+ "The 'connector' input must be provided if not "
132
+ "running from a connector directory. "
133
+ f"Could not infer connector directory from: {cwd}"
134
+ )
135
+
136
+ if not connector_name:
137
+ connector_name = connector_directory.name
138
+
139
+ if connector_directory:
140
+ connector_directory = connector_directory.resolve().absolute()
141
+ elif connector_name:
142
+ connector_directory = find_connector_root_from_name(connector_name)
143
+ else:
144
+ raise ValueError(
145
+ f"Could not infer connector_name or connector_directory from input ref: {connector_ref}",
146
+ )
147
+
148
+ return connector_name, connector_directory
149
+
150
+
151
+ def resolve_connector_name(
152
+ connector_directory: Path,
153
+ ) -> str:
154
+ """Resolve the connector name.
155
+
156
+ This function will resolve the connector name based on the provided connector directory.
157
+ If the current working directory is not a connector directory
158
+ (e.g. starting with 'source-'), the process will fail.
159
+
160
+ Raises:
161
+ FileNotFoundError: If the connector directory does not exist or cannot be found.
162
+ """
163
+ if not connector_directory:
164
+ raise FileNotFoundError(
165
+ "Connector directory does not exist or cannot be found. Please provide a valid "
166
+ "connector directory."
167
+ )
168
+ connector_name = connector_directory.absolute().name
169
+ if not connector_name.startswith("source-") and not connector_name.startswith("destination-"):
170
+ raise ValueError(
171
+ f"Connector directory '{connector_name}' does not look like a valid connector directory. "
172
+ f"Full path: {connector_directory.absolute()}"
173
+ )
174
+ return connector_name
175
+
176
+
177
+ def find_connector_root(from_paths: list[Path]) -> Path:
178
+ """Find the root directory of the connector."""
179
+ for path in from_paths:
180
+ # If we reach here, we didn't find the manifest file in any parent directory
181
+ # Check if the manifest file exists in the current directory
182
+ for parent in [path, *path.parents]:
183
+ if (parent / METADATA_YAML).exists():
184
+ return parent
185
+ if (parent / MANIFEST_YAML).exists():
186
+ return parent
187
+ if (parent / ACCEPTANCE_TEST_CONFIG).exists():
188
+ return parent
189
+ if parent.name == "airbyte_cdk":
190
+ break
191
+
192
+ raise FileNotFoundError(
193
+ "Could not find connector root directory relative to the provided directories: "
194
+ f"'{str(from_paths)}'."
195
+ )
196
+
197
+
198
+ def find_connector_root_from_name(connector_name: str) -> Path:
199
+ """Find the root directory of the connector from its name."""
200
+ with suppress(FileNotFoundError):
201
+ return find_connector_root([Path(connector_name)])
202
+
203
+ # If the connector name is not found, check if we are in the airbyte monorepo
204
+ # and try to find the connector root from the current directory.
205
+
206
+ cwd: Path = Path().absolute()
207
+
208
+ try:
209
+ airbyte_repo_root: Path = resolve_airbyte_repo_root(cwd)
210
+ except FileNotFoundError as ex:
211
+ raise FileNotFoundError(
212
+ "Could not find connector root directory relative and we are not in the airbyte repo."
213
+ ) from ex
214
+
215
+ expected_connector_dir: Path = (
216
+ airbyte_repo_root / "airbyte-integrations" / "connectors" / connector_name
217
+ )
218
+ if not expected_connector_dir.exists():
219
+ raise FileNotFoundError(
220
+ f"Could not find connector directory '{expected_connector_dir}' relative to the airbyte repo."
221
+ )
222
+
223
+ return expected_connector_dir
@@ -12,14 +12,10 @@ from enum import Enum
12
12
  from pathlib import Path
13
13
 
14
14
  import click
15
+ import requests
15
16
 
16
17
  from airbyte_cdk.models.connector_metadata import ConnectorLanguage, MetadataFile
17
- from airbyte_cdk.utils.docker_image_templates import (
18
- DOCKERIGNORE_TEMPLATE,
19
- JAVA_CONNECTOR_DOCKERFILE_TEMPLATE,
20
- MANIFEST_ONLY_DOCKERFILE_TEMPLATE,
21
- PYTHON_CONNECTOR_DOCKERFILE_TEMPLATE,
22
- )
18
+ from airbyte_cdk.utils.connector_paths import resolve_airbyte_repo_root
23
19
 
24
20
 
25
21
  @dataclass(kw_only=True)
@@ -145,6 +141,7 @@ def build_connector_image(
145
141
  tag: str,
146
142
  primary_arch: ArchEnum = ArchEnum.ARM64, # Assume MacBook M series by default
147
143
  no_verify: bool = False,
144
+ dockerfile_override: Path | None = None,
148
145
  ) -> None:
149
146
  """Build a connector Docker image.
150
147
 
@@ -167,10 +164,36 @@ def build_connector_image(
167
164
  ConnectorImageBuildError: If the image build or tag operation fails.
168
165
  """
169
166
  connector_kebab_name = connector_name
170
- connector_snake_name = connector_kebab_name.replace("-", "_")
171
167
 
172
- dockerfile_path = connector_directory / "build" / "docker" / "Dockerfile"
173
- dockerignore_path = connector_directory / "build" / "docker" / "Dockerfile.dockerignore"
168
+ if dockerfile_override:
169
+ dockerfile_path = dockerfile_override
170
+ else:
171
+ dockerfile_path = connector_directory / "build" / "docker" / "Dockerfile"
172
+ dockerignore_path = connector_directory / "build" / "docker" / "Dockerfile.dockerignore"
173
+ try:
174
+ dockerfile_text, dockerignore_text = get_dockerfile_templates(
175
+ metadata=metadata,
176
+ connector_directory=connector_directory,
177
+ )
178
+ except FileNotFoundError:
179
+ # If the Dockerfile and .dockerignore are not found in the connector directory,
180
+ # download the templates from the Airbyte repo. This is a fallback
181
+ # in case the Airbyte repo not checked out locally.
182
+ try:
183
+ dockerfile_text, dockerignore_text = _download_dockerfile_defs(
184
+ connector_language=metadata.data.language,
185
+ )
186
+ except requests.HTTPError as e:
187
+ raise ConnectorImageBuildError(
188
+ build_args=[],
189
+ error_text=(
190
+ "Could not locate local dockerfile templates and "
191
+ f"failed to download Dockerfile templates from github: {e}"
192
+ ),
193
+ ) from e
194
+
195
+ dockerfile_path.write_text(dockerfile_text)
196
+ dockerignore_path.write_text(dockerignore_text)
174
197
 
175
198
  extra_build_script: str = ""
176
199
  build_customization_path = connector_directory / "build_customization.py"
@@ -185,14 +208,9 @@ def build_connector_image(
185
208
  )
186
209
 
187
210
  base_image = metadata.data.connectorBuildOptions.baseImage
188
-
189
- dockerfile_path.write_text(get_dockerfile_template(metadata))
190
- dockerignore_path.write_text(DOCKERIGNORE_TEMPLATE)
191
-
192
211
  build_args: dict[str, str | None] = {
193
212
  "BASE_IMAGE": base_image,
194
- "CONNECTOR_SNAKE_NAME": connector_snake_name,
195
- "CONNECTOR_KEBAB_NAME": connector_kebab_name,
213
+ "CONNECTOR_NAME": connector_kebab_name,
196
214
  "EXTRA_BUILD_SCRIPT": extra_build_script,
197
215
  }
198
216
 
@@ -246,31 +264,100 @@ def build_connector_image(
246
264
  sys.exit(0)
247
265
 
248
266
 
249
- def get_dockerfile_template(
267
+ def _download_dockerfile_defs(
268
+ connector_language: ConnectorLanguage,
269
+ ) -> tuple[str, str]:
270
+ """Download the Dockerfile and .dockerignore templates for the specified connector language.
271
+
272
+ We use the requests library to download from the master branch hosted on GitHub.
273
+
274
+ Args:
275
+ connector_language: The language of the connector.
276
+
277
+ Returns:
278
+ A tuple containing the Dockerfile and .dockerignore templates as strings.
279
+
280
+ Raises:
281
+ ValueError: If the connector language is not supported.
282
+ requests.HTTPError: If the download fails.
283
+ """
284
+ print("Downloading Dockerfile and .dockerignore templates from GitHub...")
285
+ # Map ConnectorLanguage to template directory
286
+ language_to_template_suffix = {
287
+ ConnectorLanguage.PYTHON: "python-connector",
288
+ ConnectorLanguage.JAVA: "java-connector",
289
+ ConnectorLanguage.MANIFEST_ONLY: "manifest-only-connector",
290
+ }
291
+
292
+ if connector_language not in language_to_template_suffix:
293
+ raise ValueError(f"Unsupported connector language: {connector_language}")
294
+
295
+ template_suffix = language_to_template_suffix[connector_language]
296
+ base_url = f"https://github.com/airbytehq/airbyte/raw/master/docker-images/"
297
+
298
+ dockerfile_url = f"{base_url}/Dockerfile.{template_suffix}"
299
+ dockerignore_url = f"{base_url}/Dockerfile.{template_suffix}.dockerignore"
300
+
301
+ dockerfile_resp = requests.get(dockerfile_url)
302
+ dockerfile_resp.raise_for_status()
303
+ dockerfile_text = dockerfile_resp.text
304
+
305
+ dockerignore_resp = requests.get(dockerignore_url)
306
+ dockerignore_resp.raise_for_status()
307
+ dockerignore_text = dockerignore_resp.text
308
+
309
+ return dockerfile_text, dockerignore_text
310
+
311
+
312
+ def get_dockerfile_templates(
250
313
  metadata: MetadataFile,
251
- ) -> str:
314
+ connector_directory: Path,
315
+ ) -> tuple[str, str]:
252
316
  """Get the Dockerfile template for the connector.
253
317
 
254
318
  Args:
255
319
  metadata: The metadata of the connector.
256
320
  connector_name: The name of the connector.
257
321
 
322
+ Raises:
323
+ ValueError: If the connector language is not supported.
324
+ FileNotFoundError: If the Dockerfile or .dockerignore is not found.
325
+
258
326
  Returns:
259
- The Dockerfile template as a string.
327
+ A tuple containing the Dockerfile and .dockerignore templates as strings.
260
328
  """
261
- if metadata.data.language == ConnectorLanguage.PYTHON:
262
- return PYTHON_CONNECTOR_DOCKERFILE_TEMPLATE
263
-
264
- if metadata.data.language == ConnectorLanguage.MANIFEST_ONLY:
265
- return MANIFEST_ONLY_DOCKERFILE_TEMPLATE
266
-
267
- if metadata.data.language == ConnectorLanguage.JAVA:
268
- return JAVA_CONNECTOR_DOCKERFILE_TEMPLATE
329
+ if metadata.data.language not in [
330
+ ConnectorLanguage.PYTHON,
331
+ ConnectorLanguage.MANIFEST_ONLY,
332
+ ConnectorLanguage.JAVA,
333
+ ]:
334
+ raise ValueError(
335
+ f"Unsupported connector language: {metadata.data.language}. "
336
+ "Please check the connector's metadata file."
337
+ )
269
338
 
270
- raise ValueError(
271
- f"Unsupported connector language: {metadata.data.language}. "
272
- "Please check the connector's metadata file."
339
+ airbyte_repo_root = resolve_airbyte_repo_root(
340
+ from_dir=connector_directory,
341
+ )
342
+ # airbyte_repo_root successfully resolved
343
+ dockerfile_path = (
344
+ airbyte_repo_root / "docker-images" / f"Dockerfile.{metadata.data.language.value}-connector"
273
345
  )
346
+ dockerignore_path = (
347
+ airbyte_repo_root
348
+ / "docker-images"
349
+ / f"Dockerfile.{metadata.data.language.value}-connector.dockerignore"
350
+ )
351
+ if not dockerfile_path.exists():
352
+ raise FileNotFoundError(
353
+ f"Dockerfile for {metadata.data.language.value} connector not found at {dockerfile_path}"
354
+ )
355
+ if not dockerignore_path.exists():
356
+ raise FileNotFoundError(
357
+ f".dockerignore for {metadata.data.language.value} connector not found at {dockerignore_path}"
358
+ )
359
+
360
+ return dockerfile_path.read_text(), dockerignore_path.read_text()
274
361
 
275
362
 
276
363
  def run_docker_command(
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: airbyte-cdk
3
- Version: 6.48.7.dev2
3
+ Version: 6.48.8
4
4
  Summary: A framework for writing Airbyte Connectors.
5
5
  Home-page: https://airbyte.com
6
6
  License: MIT
@@ -23,7 +23,7 @@ Provides-Extra: sql
23
23
  Provides-Extra: vector-db-based
24
24
  Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
25
25
  Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
26
- Requires-Dist: airbyte-protocol-models-dataclasses (==0.15.0.dev1746621859)
26
+ Requires-Dist: airbyte-protocol-models-dataclasses (>=0.15,<0.16)
27
27
  Requires-Dist: anyascii (>=0.3.2,<0.4.0)
28
28
  Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
29
29
  Requires-Dist: backoff