airbyte-cdk 6.54.11__py3-none-any.whl → 6.55.1__py3-none-any.whl

airbyte_cdk/cli/airbyte_cdk/_connector.py

@@ -101,7 +101,7 @@ def connector_cli_group() -> None:
     pass
 
 
-@connector_cli_group.command()
+@connector_cli_group.command("test")
 @click.argument(
     "connector",
     required=False,
@@ -114,10 +114,18 @@ def connector_cli_group() -> None:
     default=False,
     help="Only collect tests, do not run them.",
 )
-def test(
+@click.option(
+    "--pytest-arg",
+    "pytest_args",  # ← map --pytest-arg into pytest_args
+    type=str,
+    multiple=True,
+    help="Additional argument(s) to pass to pytest. Can be specified multiple times.",
+)
+def connector_test(
     connector: str | Path | None = None,
     *,
     collect_only: bool = False,
+    pytest_args: list[str] | None = None,
 ) -> None:
     """Run connector tests.
 
@@ -130,19 +138,36 @@ def test(
     directory. If the current working directory is not a connector directory (e.g. starting
     with 'source-') and no connector name or path is provided, the process will fail.
     """
+    click.echo("Connector test command executed.")
+    connector_name, connector_directory = resolve_connector_name_and_directory(connector)
+
+    pytest_args = pytest_args or []
+    if collect_only:
+        pytest_args.append("--collect-only")
+
+    run_connector_tests(
+        connector_name=connector_name,
+        connector_directory=connector_directory,
+        extra_pytest_args=pytest_args,
+    )
+
+
+def run_connector_tests(
+    connector_name: str,
+    connector_directory: Path,
+    extra_pytest_args: list[str],
+) -> None:
     if pytest is None:
         raise ImportError(
             "pytest is not installed. Please install pytest to run the connector tests."
         )
-    click.echo("Connector test command executed.")
-    connector_name, connector_directory = resolve_connector_name_and_directory(connector)
 
     connector_test_suite = create_connector_test_suite(
         connector_name=connector_name if not connector_directory else None,
         connector_directory=connector_directory,
     )
 
-    pytest_args: list[str] = []
+    pytest_args: list[str] = ["-p", "airbyte_cdk.test.standard_tests.pytest_hooks"]
     if connector_directory:
         pytest_args.append(f"--rootdir={connector_directory}")
         os.chdir(str(connector_directory))
@@ -158,8 +183,8 @@ def test(
     test_file_path.parent.mkdir(parents=True, exist_ok=True)
     test_file_path.write_text(file_text)
 
-    if collect_only:
-        pytest_args.append("--collect-only")
+    if extra_pytest_args:
+        pytest_args.extend(extra_pytest_args)
 
     pytest_args.append(str(test_file_path))
 
@@ -170,7 +195,6 @@ def test(
 
     click.echo(f"Running tests from connector directory: {connector_directory}...")
     click.echo(f"Test file: {test_file_path}")
-    click.echo(f"Collect only: {collect_only}")
     click.echo(f"Pytest args: {pytest_args}")
     click.echo("Invoking Pytest...")
     exit_code = pytest.main(

airbyte_cdk/cli/airbyte_cdk/_image.py

@@ -10,6 +10,7 @@ from pathlib import Path
 
 import rich_click as click
 
+from airbyte_cdk.cli.airbyte_cdk._connector import run_connector_tests
 from airbyte_cdk.models.connector_metadata import MetadataFile
 from airbyte_cdk.utils.connector_paths import resolve_connector_name_and_directory
 from airbyte_cdk.utils.docker import (
@@ -88,6 +89,81 @@ def build(
         sys.exit(1)
 
 
+@image_cli_group.command("test")
+@click.argument(
+    "connector",
+    required=False,
+    type=str,
+    metavar="[CONNECTOR]",
+)
+@click.option(
+    "--image",
+    help="Image to test, instead of building a new one.",
+)
+def image_test(  # "image test" command
+    connector: str | None = None,
+    *,
+    image: str | None = None,
+) -> None:
+    """Test a connector Docker image.
+
+    [CONNECTOR] can be a connector name (e.g. 'source-pokeapi'), a path to a connector directory, or omitted to use the current working directory.
+    If a string containing '/' is provided, it is treated as a path. Otherwise, it is treated as a connector name.
+
+    If an image is provided, it will be used for testing instead of building a new one.
+
+    Note: You should run `airbyte-cdk secrets fetch` before running this command to ensure
+    that the secrets are available for the connector tests.
+    """
+    if not verify_docker_installation():
+        click.echo(
+            "Docker is not installed or not running. Please install Docker and try again.", err=True
+        )
+        sys.exit(1)
+
+    connector_name, connector_directory = resolve_connector_name_and_directory(connector)
+
+    # Select only tests with the 'image_tests' mark
+    pytest_args = ["-m", "image_tests"]
+    if not image:
+        metadata_file_path: Path = connector_directory / "metadata.yaml"
+        try:
+            metadata = MetadataFile.from_file(metadata_file_path)
+        except (FileNotFoundError, ValueError) as e:
+            click.echo(
+                f"Error loading metadata file '{metadata_file_path}': {e!s}",
+                err=True,
+            )
+            sys.exit(1)
+
+        tag = "dev-latest"
+        image = f"{metadata.data.dockerRepository}:{tag}"
+        click.echo(f"Building Image for Connector: {image}")
+        try:
+            image = build_connector_image(
+                connector_directory=connector_directory,
+                connector_name=connector_name,
+                metadata=metadata,
+                tag=tag,
+                no_verify=True,
+            )
+        except ConnectorImageBuildError as e:
+            click.echo(
+                f"Error building connector image: {e!s}",
+                err=True,
+            )
+            sys.exit(1)
+
+    pytest_args.extend(["--connector-image", image])
+
+    click.echo(f"Testing Connector Image: {image}")
+    run_connector_tests(
+        connector_name=connector_name,
+        connector_directory=connector_directory,
+        extra_pytest_args=pytest_args,
+    )
+
+
 __all__ = [
     "image_cli_group",
 ]

airbyte_cdk/cli/airbyte_cdk/_secrets.py

@@ -99,12 +99,12 @@ def secrets_cli_group() -> None:
     help="Print GitHub CI mask for secrets.",
     type=bool,
     is_flag=True,
-    default=False,
+    default=None,
 )
 def fetch(
     connector: str | Path | None = None,
     gcp_project_id: str = GCP_PROJECT_ID,
-    print_ci_secrets_masks: bool = False,
+    print_ci_secrets_masks: bool | None = None,
 ) -> None:
     """Fetch secrets for a connector from Google Secret Manager.
 
@@ -181,22 +181,23 @@ def fetch(
         if secret_count == 0:
             raise exceptions[0]
 
-    if not print_ci_secrets_masks:
-        return
-
-    if not os.environ.get("CI", None):
+    if print_ci_secrets_masks and "CI" not in os.environ:
         click.echo(
             "The `--print-ci-secrets-masks` option is only available in CI environments. "
             "The `CI` env var is either not set or not set to a truthy value. "
             "Skipping printing secret masks.",
             err=True,
         )
-        return
-
-    # Else print the CI mask
-    _print_ci_secrets_masks(
-        secrets_dir=secrets_dir,
-    )
+        print_ci_secrets_masks = False
+    elif print_ci_secrets_masks is None:
+        # If not explicitly set, we check if we are in a CI environment
+        # and set to True if so.
+        print_ci_secrets_masks = os.environ.get("CI", "") != ""
+
+    if print_ci_secrets_masks:
+        _print_ci_secrets_masks(
+            secrets_dir=secrets_dir,
+        )
 
 
 @secrets_cli_group.command("list")

airbyte_cdk/models/airbyte_protocol_serializers.py

@@ -4,9 +4,11 @@ from typing import Any, Dict
 from serpyco_rs import CustomType, Serializer
 
 from .airbyte_protocol import (  # type: ignore[attr-defined] # all classes are imported to airbyte_protocol via *
+    AirbyteCatalog,
     AirbyteMessage,
     AirbyteStateBlob,
     AirbyteStateMessage,
+    AirbyteStream,
     AirbyteStreamState,
     ConfiguredAirbyteCatalog,
     ConfiguredAirbyteStream,
@@ -30,6 +32,8 @@ def custom_type_resolver(t: type) -> CustomType[AirbyteStateBlob, Dict[str, Any]
     return AirbyteStateBlobType() if t is AirbyteStateBlob else None
 
 
+AirbyteCatalogSerializer = Serializer(AirbyteCatalog, omit_none=True)
+AirbyteStreamSerializer = Serializer(AirbyteStream, omit_none=True)
 AirbyteStreamStateSerializer = Serializer(
     AirbyteStreamState, omit_none=True, custom_type_resolver=custom_type_resolver
 )

airbyte_cdk/models/connector_metadata.py

@@ -34,6 +34,15 @@ class ConnectorBuildOptions(BaseModel):
     )
 
 
+class SuggestedStreams(BaseModel):
+    """Suggested streams from metadata.yaml."""
+
+    streams: list[str] = Field(
+        default=[],
+        description="List of suggested streams for the connector",
+    )
+
+
 class ConnectorMetadata(BaseModel):
     """Connector metadata from metadata.yaml."""
 
@@ -47,6 +56,11 @@ class ConnectorMetadata(BaseModel):
         description="List of tags for the connector",
     )
 
+    suggestedStreams: SuggestedStreams | None = Field(
+        default=None,
+        description="Suggested streams for the connector",
+    )
+
     @property
     def language(self) -> ConnectorLanguage:
         """Get the connector language."""

airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -202,7 +202,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
 
         # Combine streams and dynamic_streams. Note: both cannot be empty at the same time,
         # and this is validated during the initialization of the source.
-        streams = self._stream_configs(self._source_config) + self._dynamic_stream_configs(
+        streams = self._stream_configs(self._source_config, config) + self._dynamic_stream_configs(
             self._source_config, config
         )
 

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -25,6 +25,7 @@ properties:
     type: array
     items:
       anyOf:
+        - "$ref": "#/definitions/ConditionalStreams"
         - "$ref": "#/definitions/DeclarativeStream"
         - "$ref": "#/definitions/StateDelegatingStream"
   dynamic_streams:
@@ -424,6 +425,36 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
+  ConditionalStreams:
+    title: Conditional Streams
+    description: Streams that are only available while performing a connector operation when the condition is met.
+    type: object
+    required:
+      - type
+      - streams
+      - condition
+    properties:
+      type:
+        type: string
+        enum: [ConditionalStreams]
+      condition:
+        title: Condition
+        description: Condition that will be evaluated to determine if a set of streams should be available.
+        type: string
+        interpolation_context:
+          - config
+          - parameters
+        examples:
+          - "{{ config['is_sandbox'] }}"
+      streams:
+        title: Streams
+        description: Streams that will be used during an operation based on the condition.
+        type: array
+        items:
+          "$ref": "#/definitions/DeclarativeStream"
+      $parameters:
+        type: object
+        additionalProperties: true
   ConstantBackoffStrategy:
     title: Constant Backoff
     description: Backoff strategy with a constant backoff interval.

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -8,7 +8,7 @@ import pkgutil
 from copy import deepcopy
 from importlib import metadata
 from types import ModuleType
-from typing import Any, Dict, Iterator, List, Mapping, MutableMapping, Optional, Set
+from typing import Any, Dict, Iterator, List, Mapping, Optional, Set
 
 import orjson
 import yaml
@@ -35,6 +35,10 @@ from airbyte_cdk.models.airbyte_protocol_serializers import AirbyteMessageSerial
 from airbyte_cdk.sources.declarative.checks import COMPONENTS_CHECKER_TYPE_MAPPING
 from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
 from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedBoolean
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    ConditionalStreams as ConditionalStreamsModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     DeclarativeStream as DeclarativeStreamModel,
 )
@@ -300,7 +304,9 @@ class ManifestDeclarativeSource(DeclarativeSource):
             }
         )
 
-        stream_configs = self._stream_configs(self._source_config) + self.dynamic_streams
+        stream_configs = (
+            self._stream_configs(self._source_config, config=config) + self.dynamic_streams
+        )
 
         api_budget_model = self._source_config.get("api_budget")
         if api_budget_model:
@@ -319,7 +325,6 @@ class ManifestDeclarativeSource(DeclarativeSource):
             )
             for stream_config in self._initialize_cache_for_parent_streams(deepcopy(stream_configs))
         ]
-
         return source_streams
 
     @staticmethod
@@ -373,7 +378,6 @@ class ManifestDeclarativeSource(DeclarativeSource):
                     )
                 else:
                     stream_config["retriever"]["requester"]["use_cache"] = True
-
         return stream_configs
 
     def spec(self, logger: logging.Logger) -> ConnectorSpecification:
@@ -477,12 +481,27 @@ class ManifestDeclarativeSource(DeclarativeSource):
             # No exception
             return parsed_version
 
-    def _stream_configs(self, manifest: Mapping[str, Any]) -> List[Dict[str, Any]]:
+    def _stream_configs(
+        self, manifest: Mapping[str, Any], config: Mapping[str, Any]
+    ) -> List[Dict[str, Any]]:
         # This has a warning flag for static, but after we finish part 4 we'll replace manifest with self._source_config
-        stream_configs: List[Dict[str, Any]] = manifest.get("streams", [])
-        for s in stream_configs:
-            if "type" not in s:
-                s["type"] = "DeclarativeStream"
+        stream_configs = []
+        for current_stream_config in manifest.get("streams", []):
+            if (
+                "type" in current_stream_config
+                and current_stream_config["type"] == "ConditionalStreams"
+            ):
+                interpolated_boolean = InterpolatedBoolean(
+                    condition=current_stream_config.get("condition"),
+                    parameters={},
+                )
+
+                if interpolated_boolean.eval(config=config):
+                    stream_configs.extend(current_stream_config.get("streams", []))
+            else:
+                if "type" not in current_stream_config:
+                    current_stream_config["type"] = "DeclarativeStream"
+                stream_configs.append(current_stream_config)
         return stream_configs
 
     def _dynamic_stream_configs(

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -1,3 +1,5 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+
 # generated by datamodel-codegen:
 #   filename:  declarative_component_schema.yaml
 
@@ -2168,7 +2170,7 @@ class DeclarativeSource1(BaseModel):
 
     type: Literal["DeclarativeSource"]
     check: Union[CheckStream, CheckDynamicStream]
-    streams: List[Union[DeclarativeStream, StateDelegatingStream]]
+    streams: List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]
     dynamic_streams: Optional[List[DynamicDeclarativeStream]] = None
     version: str = Field(
         ...,
@@ -2201,7 +2203,9 @@ class DeclarativeSource2(BaseModel):
 
     type: Literal["DeclarativeSource"]
     check: Union[CheckStream, CheckDynamicStream]
-    streams: Optional[List[Union[DeclarativeStream, StateDelegatingStream]]] = None
+    streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = (
+        None
+    )
     dynamic_streams: List[DynamicDeclarativeStream]
     version: str = Field(
         ...,
@@ -2280,6 +2284,22 @@ class SelectiveAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class ConditionalStreams(BaseModel):
+    type: Literal["ConditionalStreams"]
+    condition: str = Field(
+        ...,
+        description="Condition that will be evaluated to determine if a set of streams should be available.",
+        examples=["{{ config['is_sandbox'] }}"],
+        title="Condition",
+    )
+    streams: List[DeclarativeStream] = Field(
+        ...,
+        description="Streams that will be used during an operation based on the condition.",
+        title="Streams",
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
+
+
 class FileUploader(BaseModel):
     type: Literal["FileUploader"]
     requester: Union[HttpRequester, CustomRequester] = Field(
@@ -2936,6 +2956,7 @@ CompositeErrorHandler.update_forward_refs()
 DeclarativeSource1.update_forward_refs()
 DeclarativeSource2.update_forward_refs()
 SelectiveAuthenticator.update_forward_refs()
+ConditionalStreams.update_forward_refs()
 FileUploader.update_forward_refs()
 DeclarativeStream.update_forward_refs()
 SessionTokenAuthenticator.update_forward_refs()

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -3150,12 +3150,12 @@ class ModelToComponentFactory:
             This is needed because the URL is not set until the requester is created.
             """
 
-            _url = (
+            _url: str = (
                 model.requester.url
                 if hasattr(model.requester, "url") and model.requester.url is not None
                 else requester.get_url()
             )
-            _url_base = (
+            _url_base: str = (
                 model.requester.url_base
                 if hasattr(model.requester, "url_base") and model.requester.url_base is not None
                 else requester.get_url_base()