airbyte-cdk 6.54.9__py3-none-any.whl → 6.54.11__py3-none-any.whl

airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py

@@ -9,7 +9,7 @@ import time
 from collections import OrderedDict
 from copy import deepcopy
 from datetime import timedelta
-from typing import Any, Callable, Iterable, Mapping, MutableMapping, Optional
+from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional
 
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.declarative.incremental.global_substream_cursor import (
@@ -66,8 +66,8 @@ class ConcurrentPerPartitionCursor(Cursor):
     _GLOBAL_STATE_KEY = "state"
     _PERPARTITION_STATE_KEY = "states"
     _IS_PARTITION_DUPLICATION_LOGGED = False
-    _KEY = 0
-    _VALUE = 1
+    _PARENT_STATE = 0
+    _GENERATION_SEQUENCE = 1
 
     def __init__(
         self,
@@ -99,19 +99,29 @@ class ConcurrentPerPartitionCursor(Cursor):
         self._semaphore_per_partition: OrderedDict[str, threading.Semaphore] = OrderedDict()
 
         # Parent-state tracking: store each partition’s parent state in creation order
-        self._partition_parent_state_map: OrderedDict[str, Mapping[str, Any]] = OrderedDict()
+        self._partition_parent_state_map: OrderedDict[str, tuple[Mapping[str, Any], int]] = (
+            OrderedDict()
+        )
+        self._parent_state: Optional[StreamState] = None
+
+        # Tracks when the last slice for partition is emitted
+        self._partitions_done_generating_stream_slices: set[str] = set()
+        # Used to track the index of partitions that are not closed yet
+        self._processing_partitions_indexes: List[int] = list()
+        self._generated_partitions_count: int = 0
+        # Dictionary to map partition keys to their index
+        self._partition_key_to_index: dict[str, int] = {}
 
-        self._finished_partitions: set[str] = set()
         self._lock = threading.Lock()
-        self._timer = Timer()
-        self._new_global_cursor: Optional[StreamState] = None
         self._lookback_window: int = 0
-        self._parent_state: Optional[StreamState] = None
+        self._new_global_cursor: Optional[StreamState] = None
         self._number_of_partitions: int = 0
         self._use_global_cursor: bool = use_global_cursor
         self._partition_serializer = PerPartitionKeySerializer()
+
         # Track the last time a state message was emitted
         self._last_emission_time: float = 0.0
+        self._timer = Timer()
 
         self._set_initial_state(stream_state)
 
@@ -157,60 +167,37 @@ class ConcurrentPerPartitionCursor(Cursor):
                 self._cursor_per_partition[partition_key].close_partition(partition=partition)
                 cursor = self._cursor_per_partition[partition_key]
                 if (
-                    partition_key in self._finished_partitions
+                    partition_key in self._partitions_done_generating_stream_slices
                     and self._semaphore_per_partition[partition_key]._value == 0
                 ):
                     self._update_global_cursor(cursor.state[self.cursor_field.cursor_field_key])
 
+            # Clean up the partition if it is fully processed
+            self._cleanup_if_done(partition_key)
+
             self._check_and_update_parent_state()
 
             self._emit_state_message()
 
     def _check_and_update_parent_state(self) -> None:
-        """
-        Pop the leftmost partition state from _partition_parent_state_map only if
-        *all partitions* up to (and including) that partition key in _semaphore_per_partition
-        are fully finished (i.e. in _finished_partitions and semaphore._value == 0).
-        Additionally, delete finished semaphores with a value of 0 to free up memory,
-        as they are only needed to track errors and completion status.
-        """
         last_closed_state = None
 
         while self._partition_parent_state_map:
-            # Look at the earliest partition key in creation order
-            earliest_key = next(iter(self._partition_parent_state_map))
-
-            # Verify ALL partitions from the left up to earliest_key are finished
-            all_left_finished = True
-            for p_key, sem in list(
-                self._semaphore_per_partition.items()
-            ):  # Use list to allow modification during iteration
-                # If any earlier partition is still not finished, we must stop
-                if p_key not in self._finished_partitions or sem._value != 0:
-                    all_left_finished = False
-                    break
-                # Once we've reached earliest_key in the semaphore order, we can stop checking
-                if p_key == earliest_key:
-                    break
-
-            # If the partitions up to earliest_key are not all finished, break the while-loop
-            if not all_left_finished:
-                break
+            earliest_key, (candidate_state, candidate_seq) = next(
+                iter(self._partition_parent_state_map.items())
+            )
 
-            # Pop the leftmost entry from parent-state map
-            _, closed_parent_state = self._partition_parent_state_map.popitem(last=False)
-            last_closed_state = closed_parent_state
+            # if any partition that started <= candidate_seq is still open, we must wait
+            if (
+                self._processing_partitions_indexes
+                and self._processing_partitions_indexes[0] <= candidate_seq
+            ):
+                break
 
-            # Clean up finished semaphores with value 0 up to and including earliest_key
-            for p_key in list(self._semaphore_per_partition.keys()):
-                sem = self._semaphore_per_partition[p_key]
-                if p_key in self._finished_partitions and sem._value == 0:
-                    del self._semaphore_per_partition[p_key]
-                    logger.debug(f"Deleted finished semaphore for partition {p_key} with value 0")
-                if p_key == earliest_key:
-                    break
+            # safe to pop
+            self._partition_parent_state_map.popitem(last=False)
+            last_closed_state = candidate_state
 
-        # Update _parent_state if we popped at least one partition
         if last_closed_state is not None:
             self._parent_state = last_closed_state
 
@@ -289,18 +276,24 @@ class ConcurrentPerPartitionCursor(Cursor):
             if not self._IS_PARTITION_DUPLICATION_LOGGED:
                 logger.warning(f"Partition duplication detected for stream {self._stream_name}")
                 self._IS_PARTITION_DUPLICATION_LOGGED = True
+            return
         else:
             self._semaphore_per_partition[partition_key] = threading.Semaphore(0)
 
         with self._lock:
+            seq = self._generated_partitions_count
+            self._generated_partitions_count += 1
+            self._processing_partitions_indexes.append(seq)
+            self._partition_key_to_index[partition_key] = seq
+
             if (
                 len(self._partition_parent_state_map) == 0
                 or self._partition_parent_state_map[
                     next(reversed(self._partition_parent_state_map))
-                ]
+                ][self._PARENT_STATE]
                 != parent_state
             ):
-                self._partition_parent_state_map[partition_key] = deepcopy(parent_state)
+                self._partition_parent_state_map[partition_key] = (deepcopy(parent_state), seq)
 
         for cursor_slice, is_last_slice, _ in iterate_with_last_flag_and_state(
             cursor.stream_slices(),
@@ -308,7 +301,7 @@ class ConcurrentPerPartitionCursor(Cursor):
         ):
             self._semaphore_per_partition[partition_key].release()
             if is_last_slice:
-                self._finished_partitions.add(partition_key)
+                self._partitions_done_generating_stream_slices.add(partition_key)
             yield StreamSlice(
                 partition=partition, cursor_slice=cursor_slice, extra_fields=partition.extra_fields
             )
@@ -338,14 +331,11 @@ class ConcurrentPerPartitionCursor(Cursor):
             while len(self._cursor_per_partition) > self.DEFAULT_MAX_PARTITIONS_NUMBER - 1:
                 # Try removing finished partitions first
                 for partition_key in list(self._cursor_per_partition.keys()):
-                    if partition_key in self._finished_partitions and (
-                        partition_key not in self._semaphore_per_partition
-                        or self._semaphore_per_partition[partition_key]._value == 0
-                    ):
+                    if partition_key not in self._partition_key_to_index:
                         oldest_partition = self._cursor_per_partition.pop(
                             partition_key
                         )  # Remove the oldest partition
-                        logger.warning(
+                        logger.debug(
                             f"The maximum number of partitions has been reached. Dropping the oldest finished partition: {oldest_partition}. Over limit: {self._number_of_partitions - self.DEFAULT_MAX_PARTITIONS_NUMBER}."
                         )
                         break
@@ -474,6 +464,25 @@ class ConcurrentPerPartitionCursor(Cursor):
         ):
             self._new_global_cursor = {self.cursor_field.cursor_field_key: copy.deepcopy(value)}
 
+    def _cleanup_if_done(self, partition_key: str) -> None:
+        """
+        Free every in-memory structure that belonged to a completed partition:
+        cursor, semaphore, flag inside `_finished_partitions`
+        """
+        if not (
+            partition_key in self._partitions_done_generating_stream_slices
+            and self._semaphore_per_partition[partition_key]._value == 0
+        ):
+            return
+
+        self._semaphore_per_partition.pop(partition_key, None)
+        self._partitions_done_generating_stream_slices.discard(partition_key)
+
+        seq = self._partition_key_to_index.pop(partition_key)
+        self._processing_partitions_indexes.remove(seq)
+
+        logger.debug(f"Partition {partition_key} fully processed and cleaned up.")
+
     def _to_partition_key(self, partition: Mapping[str, Any]) -> str:
         return self._partition_serializer.to_partition_key(partition)
 

airbyte_cdk/sources/declarative/interpolation/jinja.py

@@ -147,16 +147,18 @@ class JinjaInterpolation(Interpolation):
             # It can be returned as is
             return s
 
+    @staticmethod
     @cache
-    def _find_undeclared_variables(self, s: Optional[str]) -> Set[str]:
+    def _find_undeclared_variables(s: Optional[str]) -> Set[str]:
         """
         Find undeclared variables and cache them
         """
         ast = _ENVIRONMENT.parse(s)  # type: ignore # parse is able to handle None
         return meta.find_undeclared_variables(ast)
 
+    @staticmethod
     @cache
-    def _compile(self, s: str) -> Template:
+    def _compile(s: str) -> Template:
         """
         We must cache the Jinja Template ourselves because we're using `from_string` instead of a template loader
         """

airbyte_cdk/test/entrypoint_wrapper.py

@@ -44,6 +44,7 @@ from airbyte_cdk.models import (
     Type,
 )
 from airbyte_cdk.sources import Source
+from airbyte_cdk.test.models.scenario import ExpectedOutcome
 
 
 class EntrypointOutput:
@@ -157,8 +158,22 @@ class EntrypointOutput:
 
 
 def _run_command(
-    source: Source, args: List[str], expecting_exception: bool = False
+    source: Source,
+    args: List[str],
+    expecting_exception: bool | None = None,  # Deprecated, use `expected_outcome` instead.
+    *,
+    expected_outcome: ExpectedOutcome | None = None,
 ) -> EntrypointOutput:
+    """Internal function to run a command with the AirbyteEntrypoint.
+
+    Note: Even though this function is private, some connectors do call it directly.
+
+    Note: The `expecting_exception` arg is now deprecated in favor of the tri-state
+    `expected_outcome` arg. The old argument is supported (for now) for backwards compatibility.
+    """
+    expected_outcome = expected_outcome or ExpectedOutcome.from_expecting_exception_bool(
+        expecting_exception,
+    )
     log_capture_buffer = StringIO()
     stream_handler = logging.StreamHandler(log_capture_buffer)
     stream_handler.setLevel(logging.INFO)
@@ -175,27 +190,30 @@ def _run_command(
         for message in source_entrypoint.run(parsed_args):
             messages.append(message)
     except Exception as exception:
-        if not expecting_exception:
+        if expected_outcome.expect_success():
             print("Printing unexpected error from entrypoint_wrapper")
             print("".join(traceback.format_exception(None, exception, exception.__traceback__)))
+
         uncaught_exception = exception
 
     captured_logs = log_capture_buffer.getvalue().split("\n")[:-1]
 
     parent_logger.removeHandler(stream_handler)
 
-    return EntrypointOutput(messages + captured_logs, uncaught_exception)
+    return EntrypointOutput(messages + captured_logs, uncaught_exception=uncaught_exception)
 
 
 def discover(
     source: Source,
     config: Mapping[str, Any],
-    expecting_exception: bool = False,
+    expecting_exception: bool | None = None,  # Deprecated, use `expected_outcome` instead.
+    *,
+    expected_outcome: ExpectedOutcome | None = None,
 ) -> EntrypointOutput:
     """
     config must be json serializable
-    :param expecting_exception: By default if there is an uncaught exception, the exception will be printed out. If this is expected, please
-        provide expecting_exception=True so that the test output logs are cleaner
+    :param expected_outcome: By default if there is an uncaught exception, the exception will be printed out. If this is expected, please
+        provide `expected_outcome=ExpectedOutcome.EXPECT_FAILURE` so that the test output logs are cleaner
     """
 
     with tempfile.TemporaryDirectory() as tmp_directory:
@@ -203,7 +221,10 @@ def discover(
         config_file = make_file(tmp_directory_path / "config.json", config)
 
         return _run_command(
-            source, ["discover", "--config", config_file, "--debug"], expecting_exception
+            source,
+            ["discover", "--config", config_file, "--debug"],
+            expecting_exception=expecting_exception,  # Deprecated, but still supported.
+            expected_outcome=expected_outcome,
         )
 
 
@@ -212,13 +233,15 @@ def read(
     config: Mapping[str, Any],
     catalog: ConfiguredAirbyteCatalog,
     state: Optional[List[AirbyteStateMessage]] = None,
-    expecting_exception: bool = False,
+    expecting_exception: bool | None = None,  # Deprecated, use `expected_outcome` instead.
+    *,
+    expected_outcome: ExpectedOutcome | None = None,
 ) -> EntrypointOutput:
     """
     config and state must be json serializable
 
-    :param expecting_exception: By default if there is an uncaught exception, the exception will be printed out. If this is expected, please
-        provide expecting_exception=True so that the test output logs are cleaner
+    :param expected_outcome: By default if there is an uncaught exception, the exception will be printed out. If this is expected, please
+        provide `expected_outcome=ExpectedOutcome.EXPECT_FAILURE` so that the test output logs are cleaner.
     """
     with tempfile.TemporaryDirectory() as tmp_directory:
         tmp_directory_path = Path(tmp_directory)
@@ -245,7 +268,12 @@ def read(
                 ]
             )
 
-        return _run_command(source, args, expecting_exception)
+        return _run_command(
+            source,
+            args,
+            expecting_exception=expecting_exception,  # Deprecated, but still supported.
+            expected_outcome=expected_outcome,
+        )
 
 
 def make_file(

airbyte_cdk/test/models/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""Models used for standard tests."""
+
+from airbyte_cdk.test.models.outcome import ExpectedOutcome
+from airbyte_cdk.test.models.scenario import ConnectorTestScenario
+
+__all__ = [
+    "ConnectorTestScenario",
+    "ExpectedOutcome",
+]

airbyte_cdk/test/models/outcome.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Run acceptance tests in PyTest.
+
+These tests leverage the same `acceptance-test-config.yml` configuration files as the
+acceptance tests in CAT, but they run in PyTest instead of CAT. This allows us to run
+the acceptance tests in the same local environment as we are developing in, speeding
+up iteration cycles.
+"""
+
+from __future__ import annotations
+
+from enum import Enum, auto
+
+
+class ExpectedOutcome(Enum):
+    """Enum to represent the expected outcome of a test scenario.
+
+    Class supports comparisons to a boolean or None.
+    """
+
+    EXPECT_EXCEPTION = auto()
+    EXPECT_SUCCESS = auto()
+    ALLOW_ANY = auto()
+
+    @classmethod
+    def from_status_str(cls, status: str | None) -> ExpectedOutcome:
+        """Convert a status string to an ExpectedOutcome."""
+        if status is None:
+            return ExpectedOutcome.ALLOW_ANY
+
+        try:
+            return {
+                "succeed": ExpectedOutcome.EXPECT_SUCCESS,
+                "failed": ExpectedOutcome.EXPECT_EXCEPTION,
+            }[status]
+        except KeyError as ex:
+            raise ValueError(f"Invalid status '{status}'. Expected 'succeed' or 'failed'.") from ex
+
+    @classmethod
+    def from_expecting_exception_bool(cls, expecting_exception: bool | None) -> ExpectedOutcome:
+        """Convert a boolean indicating whether an exception is expected to an ExpectedOutcome."""
+        if expecting_exception is None:
+            # Align with legacy behavior where default would be 'False' (no exception expected)
+            return ExpectedOutcome.EXPECT_SUCCESS
+
+        return (
+            ExpectedOutcome.EXPECT_EXCEPTION
+            if expecting_exception
+            else ExpectedOutcome.EXPECT_SUCCESS
+        )
+
+    def expect_exception(self) -> bool:
+        """Return whether the expectation is that an exception should be raised."""
+        return self == ExpectedOutcome.EXPECT_EXCEPTION
+
+    def expect_success(self) -> bool:
+        """Return whether the expectation is that the test should succeed without exceptions."""
+        return self == ExpectedOutcome.EXPECT_SUCCESS

airbyte_cdk/test/{standard_tests/models → models}/scenario.py

@@ -9,11 +9,13 @@ up iteration cycles.
 
 from __future__ import annotations
 
-from pathlib import Path
+from pathlib import Path  # noqa: TC003  # Pydantic needs this (don't move to 'if typing' block)
 from typing import Any, Literal, cast
 
 import yaml
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
+
+from airbyte_cdk.test.models.outcome import ExpectedOutcome
 
 
 class ConnectorTestScenario(BaseModel):
@@ -24,6 +26,10 @@ class ConnectorTestScenario(BaseModel):
     acceptance test configuration file.
     """
 
+    # Allows the class to be hashable, which PyTest will require
+    # when we use to parameterize tests.
+    model_config = ConfigDict(frozen=True)
+
     class AcceptanceTestExpectRecords(BaseModel):
         path: Path
         exact_order: bool = False
@@ -46,6 +52,7 @@ class ConnectorTestScenario(BaseModel):
     def get_config_dict(
         self,
         *,
+        connector_root: Path,
         empty_if_missing: bool,
     ) -> dict[str, Any]:
         """Return the config dictionary.
@@ -61,7 +68,15 @@ class ConnectorTestScenario(BaseModel):
             return self.config_dict
 
         if self.config_path is not None:
-            return cast(dict[str, Any], yaml.safe_load(self.config_path.read_text()))
+            config_path = self.config_path
+            if not config_path.is_absolute():
+                # We usually receive a relative path here. Let's resolve it.
+                config_path = (connector_root / self.config_path).resolve().absolute()
+
+            return cast(
+                dict[str, Any],
+                yaml.safe_load(config_path.read_text()),
+            )
 
         if empty_if_missing:
             return {}
@@ -69,8 +84,13 @@ class ConnectorTestScenario(BaseModel):
         raise ValueError("No config dictionary or path provided.")
 
     @property
-    def expect_exception(self) -> bool:
-        return self.status and self.status == "failed" or False
+    def expected_outcome(self) -> ExpectedOutcome:
+        """Whether the test scenario expects an exception to be raised.
+
+        Returns True if the scenario expects an exception, False if it does not,
+        and None if there is no set expectation.
+        """
+        return ExpectedOutcome.from_status_str(self.status)
 
     @property
     def instance_name(self) -> str:
@@ -83,3 +103,38 @@ class ConnectorTestScenario(BaseModel):
             return f"'{self.config_path.name}' Test Scenario"
 
         return f"'{hash(self)}' Test Scenario"
+
+    def without_expected_outcome(self) -> ConnectorTestScenario:
+        """Return a copy of the scenario that does not expect failure or success.
+
+        This is useful when running multiple steps, to defer the expectations to a later step.
+        """
+        return ConnectorTestScenario(
+            **self.model_dump(exclude={"status"}),
+        )
+
+    def with_expecting_failure(self) -> ConnectorTestScenario:
+        """Return a copy of the scenario that expects failure.
+
+        This is useful when deriving new scenarios from existing ones.
+        """
+        if self.status == "failed":
+            return self
+
+        return ConnectorTestScenario(
+            **self.model_dump(exclude={"status"}),
+            status="failed",
+        )
+
+    def with_expecting_success(self) -> ConnectorTestScenario:
+        """Return a copy of the scenario that expects success.
+
+        This is useful when deriving new scenarios from existing ones.
+        """
+        if self.status == "succeed":
+            return self
+
+        return ConnectorTestScenario(
+            **self.model_dump(exclude={"status"}),
+            status="succeed",
+        )

airbyte_cdk/test/standard_tests/_job_runner.py

@@ -16,7 +16,7 @@ from airbyte_cdk.models import (
     Status,
 )
 from airbyte_cdk.test import entrypoint_wrapper
-from airbyte_cdk.test.standard_tests.models import (
+from airbyte_cdk.test.models import (
     ConnectorTestScenario,
 )
 
@@ -58,6 +58,7 @@ def run_test_job(
     connector: IConnector | type[IConnector] | Callable[[], IConnector],
     verb: Literal["spec", "read", "check", "discover"],
     *,
+    connector_root: Path,
     test_scenario: ConnectorTestScenario | None = None,
     catalog: ConfiguredAirbyteCatalog | dict[str, Any] | None = None,
 ) -> entrypoint_wrapper.EntrypointOutput:
@@ -84,7 +85,10 @@ def run_test_job(
         )
 
     args: list[str] = [verb]
-    config_dict = test_scenario.get_config_dict(empty_if_missing=True)
+    config_dict = test_scenario.get_config_dict(
+        empty_if_missing=True,
+        connector_root=connector_root,
+    )
     if config_dict and verb != "spec":
         # Write the config to a temp json file and pass the path to the file as an argument.
         config_path = (
@@ -118,9 +122,9 @@ def run_test_job(
     result: entrypoint_wrapper.EntrypointOutput = entrypoint_wrapper._run_command(  # noqa: SLF001  # Non-public API
         source=connector_obj,  # type: ignore [arg-type]
         args=args,
-        expecting_exception=test_scenario.expect_exception,
+        expected_outcome=test_scenario.expected_outcome,
     )
-    if result.errors and not test_scenario.expect_exception:
+    if result.errors and test_scenario.expected_outcome.expect_success():
         raise AssertionError(
             f"Expected no errors but got {len(result.errors)}: \n" + _errors_to_str(result)
         )
@@ -135,7 +139,7 @@ def run_test_job(
             + "\n".join([str(msg) for msg in result.connection_status_messages])
             + _errors_to_str(result)
         )
-        if test_scenario.expect_exception:
+        if test_scenario.expected_outcome.expect_exception():
             conn_status = result.connection_status_messages[0].connectionStatus
             assert conn_status, (
                 "Expected CONNECTION_STATUS message to be present. Got: \n"
@@ -149,14 +153,15 @@ def run_test_job(
         return result
 
     # For all other verbs, we assert check that an exception is raised (or not).
-    if test_scenario.expect_exception:
+    if test_scenario.expected_outcome.expect_exception():
         if not result.errors:
             raise AssertionError("Expected exception but got none.")
 
         return result
 
-    assert not result.errors, (
-        f"Expected no errors but got {len(result.errors)}: \n" + _errors_to_str(result)
-    )
+    if test_scenario.expected_outcome.expect_success():
+        assert not result.errors, (
+            f"Expected no errors but got {len(result.errors)}: \n" + _errors_to_str(result)
+        )
 
     return result

airbyte_cdk/test/standard_tests/connector_base.py

@@ -20,10 +20,10 @@ from airbyte_cdk.models import (
     Type,
 )
 from airbyte_cdk.test import entrypoint_wrapper
-from airbyte_cdk.test.standard_tests._job_runner import IConnector, run_test_job
-from airbyte_cdk.test.standard_tests.models import (
+from airbyte_cdk.test.models import (
     ConnectorTestScenario,
 )
+from airbyte_cdk.test.standard_tests._job_runner import IConnector, run_test_job
 from airbyte_cdk.utils.connector_paths import (
     ACCEPTANCE_TEST_CONFIG,
     find_connector_root,
@@ -116,6 +116,7 @@ class ConnectorTestSuiteBase(abc.ABC):
             self.create_connector(scenario),
             "check",
             test_scenario=scenario,
+            connector_root=self.get_connector_root_dir(),
         )
         conn_status_messages: list[AirbyteMessage] = [
             msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS
@@ -163,19 +164,23 @@ class ConnectorTestSuiteBase(abc.ABC):
             ):
                 continue
 
-            test_scenarios.extend(
-                [
-                    ConnectorTestScenario.model_validate(test)
-                    for test in all_tests_config["acceptance_tests"][category]["tests"]
-                    if "config_path" in test and "iam_role" not in test["config_path"]
-                ]
-            )
+            for test in all_tests_config["acceptance_tests"][category]["tests"]:
+                if "config_path" not in test:
+                    # Skip tests without a config_path
+                    continue
+
+                if "iam_role" in test["config_path"]:
+                    # We skip iam_role tests for now, as they are not supported in the test suite.
+                    continue
+
+                scenario = ConnectorTestScenario.model_validate(test)
+
+                if scenario.config_path and scenario.config_path in [
+                    s.config_path for s in test_scenarios
+                ]:
+                    # Skip duplicate scenarios based on config_path
+                    continue
 
-        connector_root = cls.get_connector_root_dir().absolute()
-        for test in test_scenarios:
-            if test.config_path:
-                test.config_path = connector_root / test.config_path
-            if test.configured_catalog_path:
-                test.configured_catalog_path = connector_root / test.configured_catalog_path
+                test_scenarios.append(scenario)
 
         return test_scenarios

airbyte_cdk/test/standard_tests/declarative_sources.py

@@ -9,8 +9,8 @@ from boltons.typeutils import classproperty
 from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
     ConcurrentDeclarativeSource,
 )
+from airbyte_cdk.test.models import ConnectorTestScenario
 from airbyte_cdk.test.standard_tests._job_runner import IConnector
-from airbyte_cdk.test.standard_tests.models import ConnectorTestScenario
 from airbyte_cdk.test.standard_tests.source_base import SourceTestSuiteBase
 from airbyte_cdk.utils.connector_paths import MANIFEST_YAML
 
@@ -78,7 +78,12 @@ class DeclarativeSourceTestSuite(SourceTestSuiteBase):
         config = {
             "__injected_manifest": manifest_dict,
         }
-        config.update(scenario.get_config_dict(empty_if_missing=True))
+        config.update(
+            scenario.get_config_dict(
+                empty_if_missing=True,
+                connector_root=cls.get_connector_root_dir(),
+            ),
+        )
 
         if cls.components_py_path and cls.components_py_path.exists():
             os.environ["AIRBYTE_ENABLE_UNSAFE_CODE"] = "true"

airbyte_cdk/test/standard_tests/source_base.py

@@ -2,6 +2,7 @@
 """Base class for source test suites."""
 
 from dataclasses import asdict
+from typing import TYPE_CHECKING
 
 from airbyte_cdk.models import (
     AirbyteMessage,
@@ -12,14 +13,16 @@ from airbyte_cdk.models import (
     SyncMode,
     Type,
 )
-from airbyte_cdk.test import entrypoint_wrapper
+from airbyte_cdk.test.models import (
+    ConnectorTestScenario,
+)
 from airbyte_cdk.test.standard_tests._job_runner import run_test_job
 from airbyte_cdk.test.standard_tests.connector_base import (
     ConnectorTestSuiteBase,
 )
-from airbyte_cdk.test.standard_tests.models import (
-    ConnectorTestScenario,
-)
+
+if TYPE_CHECKING:
+    from airbyte_cdk.test import entrypoint_wrapper
 
 
 class SourceTestSuiteBase(ConnectorTestSuiteBase):
@@ -43,6 +46,7 @@ class SourceTestSuiteBase(ConnectorTestSuiteBase):
             self.create_connector(scenario),
             "check",
             test_scenario=scenario,
+            connector_root=self.get_connector_root_dir(),
         )
         conn_status_messages: list[AirbyteMessage] = [
             msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS
@@ -61,6 +65,7 @@ class SourceTestSuiteBase(ConnectorTestSuiteBase):
         run_test_job(
             self.create_connector(scenario),
             "discover",
+            connector_root=self.get_connector_root_dir(),
             test_scenario=scenario,
         )
 
@@ -80,6 +85,7 @@ class SourceTestSuiteBase(ConnectorTestSuiteBase):
             verb="spec",
             test_scenario=None,
             connector=self.create_connector(scenario=None),
+            connector_root=self.get_connector_root_dir(),
         )
         # If an error occurs, it will be raised above.
 
@@ -102,10 +108,11 @@ class SourceTestSuiteBase(ConnectorTestSuiteBase):
         discover_result = run_test_job(
             self.create_connector(scenario),
             "discover",
-            test_scenario=scenario,
+            connector_root=self.get_connector_root_dir(),
+            test_scenario=scenario.without_expected_outcome(),
         )
-        if scenario.expect_exception:
-            assert discover_result.errors, "Expected exception but got none."
+        if scenario.expected_outcome.expect_exception() and discover_result.errors:
+            # Failed as expected; we're done.
             return
 
         configured_catalog = ConfiguredAirbyteCatalog(
@@ -122,6 +129,7 @@ class SourceTestSuiteBase(ConnectorTestSuiteBase):
             self.create_connector(scenario),
             "read",
             test_scenario=scenario,
+            connector_root=self.get_connector_root_dir(),
             catalog=configured_catalog,
         )
 
@@ -149,15 +157,14 @@ class SourceTestSuiteBase(ConnectorTestSuiteBase):
                     ),
                     sync_mode="INVALID",  # type: ignore [reportArgumentType]
                     destination_sync_mode="INVALID",  # type: ignore [reportArgumentType]
-                )
-            ]
+                ),
+            ],
         )
-        # Set expected status to "failed" to ensure the test fails if the connector.
-        scenario.status = "failed"
         result: entrypoint_wrapper.EntrypointOutput = run_test_job(
             self.create_connector(scenario),
             "read",
-            test_scenario=scenario,
+            connector_root=self.get_connector_root_dir(),
+            test_scenario=scenario.with_expecting_failure(),  # Expect failure due to bad catalog
             catalog=asdict(invalid_configured_catalog),
         )
         assert result.errors, "Expected errors but got none."

airbyte_cdk/test/standard_tests/util.py

@@ -67,7 +67,7 @@ def create_connector_test_suite(
         )
 
     subclass_overrides: dict[str, Any] = {
-        "get_connector_root_dir": lambda: connector_directory,
+        "get_connector_root_dir": classmethod(lambda cls: connector_directory),
     }
 
     TestSuiteAuto = type(

airbyte_cdk/test/utils/reading.py

@@ -6,6 +6,7 @@ from airbyte_cdk import AbstractSource
 from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, SyncMode
 from airbyte_cdk.test.catalog_builder import CatalogBuilder
 from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read
+from airbyte_cdk.test.models.outcome import ExpectedOutcome
 
 
 def catalog(stream_name: str, sync_mode: SyncMode) -> ConfiguredAirbyteCatalog:
@@ -19,8 +20,17 @@ def read_records(
     stream_name: str,
     sync_mode: SyncMode,
     state: Optional[List[AirbyteStateMessage]] = None,
-    expecting_exception: bool = False,
+    expecting_exception: bool | None = None,  # Deprecated, use expected_outcome instead.
+    *,
+    expected_outcome: ExpectedOutcome | None = None,
 ) -> EntrypointOutput:
     """Read records from a stream."""
     _catalog = catalog(stream_name, sync_mode)
-    return read(source, config, _catalog, state, expecting_exception)
+    return read(
+        source,
+        config,
+        _catalog,
+        state,
+        expecting_exception=expecting_exception,  # Deprecated, for backward compatibility.
+        expected_outcome=expected_outcome,
+    )

{airbyte_cdk-6.54.9.dist-info → airbyte_cdk-6.54.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.54.9
+Version: 6.54.11
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.54.9.dist-info → airbyte_cdk-6.54.11.dist-info}/RECORD

@@ -111,7 +111,7 @@ airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=vCpwX1PVRFP
 airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=WJyA2OYIEgFpVP5Y3o0tIj69AV6IKkn9B16MeXaEItI,6513
 airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
 airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
-airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=8zxfM1aXSwdfTbpL9PHwgg43ZRbNUQRcX038D7YTI_0,22768
+airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=araWk039M89c6lQHEUltfM1VI_xGw9gZIDXRWWF6SkM,22591
 airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py,sha256=Rbe6lJLTtZ5en33MwZiB9-H9-AwDMNHgwBZs8EqhYqk,22172
 airbyte_cdk/sources/declarative/incremental/declarative_cursor.py,sha256=5Bhw9VRPyIuCaD0wmmq_L3DZsa-rJgtKSEUzSd8YYD0,536
 airbyte_cdk/sources/declarative/incremental/global_substream_cursor.py,sha256=2tsE6FgXzemf4fZZ4uGtd8QpRBl9GJ2CRqSNJE5p0EI,16077
@@ -125,7 +125,7 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py,sha256=h36
 airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py,sha256=myVaNtFqxOAwrbp93rgd1dhkqyuvXvET9rsimQ89ktc,1873
 airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=CQkHqGlfa87G6VYMtBAQWin7ECKpfMdrDcg0JO5_rhc,3212
 airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=9IoeuWam3L6GyN10L6U8xNWXmkt9cnahSDNkez1OmFY,982
-airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=wm_W3_usYU_3eUdSpzrj21qsk8-vcK8TtWEHQsH7SY4,7245
+airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=oFGKs3oX0xO6DOL4E9x8rhxwbEoRcgx4HJVIL1RQ9c4,7269
 airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=xRcmjape4_WGmKMJpmBsKY0k4OHJDM46Hv3V-dlSz3w,5640
 airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=ciXtM7Qhus170ZwP8B9Ac4VScX2FPBYvlbZRv_r376U,24692
 airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -378,29 +378,30 @@ airbyte_cdk/sql/shared/sql_processor.py,sha256=1CwfC3fp9dWnHBpKtly7vGduf9ho_Mahi
 airbyte_cdk/sql/types.py,sha256=XEIhRAo_ASd0kVLBkdLf5bHiRhNple-IJrC9TibcDdY,5880
 airbyte_cdk/test/__init__.py,sha256=f_XdkOg4_63QT2k3BbKY34209lppwgw-svzfZstQEq4,199
 airbyte_cdk/test/catalog_builder.py,sha256=-y05Cz1x0Dlk6oE9LSKhCozssV2gYBNtMdV5YYOPOtk,3015
-airbyte_cdk/test/entrypoint_wrapper.py,sha256=UsuvbgXDki00-hgpQgTeoSTr_Sxv6UXYfymHM0jhGwg,10074
+airbyte_cdk/test/entrypoint_wrapper.py,sha256=BOqE_cj33UNNIQuJDxroY9vzmX5tsdGx1YKST8pE9oA,11307
 airbyte_cdk/test/mock_http/__init__.py,sha256=jE5kC6CQ0OXkTqKhciDnNVZHesBFVIA2YvkdFGwva7k,322
 airbyte_cdk/test/mock_http/matcher.py,sha256=4Qj8UnJKZIs-eodshryce3SN1Ayc8GZpBETmP6hTEyc,1446
 airbyte_cdk/test/mock_http/mocker.py,sha256=XgsjMtVoeMpRELPyALgrkHFauH9H5irxrz1Kcxh2yFY,8013
 airbyte_cdk/test/mock_http/request.py,sha256=tdB8cqk2vLgCDTOKffBKsM06llYs4ZecgtH6DKyx6yY,4112
 airbyte_cdk/test/mock_http/response.py,sha256=s4-cQQqTtmeej0pQDWqmG0vUWpHS-93lIWMpW3zSVyU,662
 airbyte_cdk/test/mock_http/response_builder.py,sha256=F-v7ebftqGj7YVIMLKdodmU9U8Dq8aIyllWGo2NGwHc,8331
+airbyte_cdk/test/models/__init__.py,sha256=5f5oFcuUA3dyNTfvvTWav2pTD8WX4nznObKgMTmvdus,290
+airbyte_cdk/test/models/outcome.py,sha256=TdLEnRZv1QjlbI0xdsJqA2rfIvSVRJWXS4TNS9NbKZQ,2135
+airbyte_cdk/test/models/scenario.py,sha256=rlU3ykv6YCQsDxEtHMcDQJS_HQYa8sxfWSM7xiHWSjU,4710
 airbyte_cdk/test/standard_tests/__init__.py,sha256=YS2bghoGmQ-4GNIbe6RuEmvV-V1kpM1OyxTpebrs0Ig,1338
-airbyte_cdk/test/standard_tests/_job_runner.py,sha256=k-gwwXtIJbh0BwH71uTST9LLx246RrnZMchACz-vIS8,6014
-airbyte_cdk/test/standard_tests/connector_base.py,sha256=NMA0x8CA1tQYRhSv1x9lBxkveGfqaRHHt7ggxKAXgfA,6970
-airbyte_cdk/test/standard_tests/declarative_sources.py,sha256=vzpW507HUPtYgP5VEK1MAR_mt1Gq2nk40sjNhKie5yM,3402
+airbyte_cdk/test/standard_tests/_job_runner.py,sha256=vTuLJiJv-LBqXLfcr7RaTm0HBH4y7L8U6tDv3qYdTkg,6195
+airbyte_cdk/test/standard_tests/connector_base.py,sha256=YA_frtCav8ygg9XlHY9Yxi-KOkHBsaiXZGIp4ULa2D4,7107
+airbyte_cdk/test/standard_tests/declarative_sources.py,sha256=4lmXKVJEhYeZAYaaXODwkn-DoJt_V--Thbea0kzOqdc,3502
 airbyte_cdk/test/standard_tests/destination_base.py,sha256=MARZip2mdo_PzGvzf2VBTAfrP4tbjrJYgeJUApnAArA,731
-airbyte_cdk/test/standard_tests/models/__init__.py,sha256=bS25WlzQwPNxpU5DHtUDZo1DuXd0LkEv9qesNhY1jkY,135
-airbyte_cdk/test/standard_tests/models/scenario.py,sha256=kvuc5oIH-TA3TS7xtiz1RQ8tcY_WC0aI6-TsDQAqhUA,2710
 airbyte_cdk/test/standard_tests/pytest_hooks.py,sha256=4OMy2jNQThS8y7Tyj8MiMy2-SWjoefD4lGo-zQmCUfU,1886
-airbyte_cdk/test/standard_tests/source_base.py,sha256=cprlvFIRy0xqsLwAC38I9zWlxC2-PJapdje9kRGjTek,6047
-airbyte_cdk/test/standard_tests/util.py,sha256=JrTXxLpcMvRJhnJDVzY8Y-TmcWhW7GGfNOnbS3v0pR4,2920
+airbyte_cdk/test/standard_tests/source_base.py,sha256=o5N9a1lPgx8IVHgXlZcRdKmE4hhgQtTgYoAOA79MgS4,6421
+airbyte_cdk/test/standard_tests/util.py,sha256=340vihLJ_2rEnq91dRHutbPM4ssm2ze1uq01cOI5vF4,2937
 airbyte_cdk/test/state_builder.py,sha256=kLPql9lNzUJaBg5YYRLJlY_Hy5JLHJDVyKPMZMoYM44,946
 airbyte_cdk/test/utils/__init__.py,sha256=Hu-1XT2KDoYjDF7-_ziDwv5bY3PueGjANOCbzeOegDg,57
 airbyte_cdk/test/utils/data.py,sha256=CkCR1_-rujWNmPXFR1IXTMwx1rAl06wAyIKWpDcN02w,820
 airbyte_cdk/test/utils/http_mocking.py,sha256=F2hpm2q4ijojQN5u2XtgTAp8aNgHgJ64eZNkZ9BW0ig,550
 airbyte_cdk/test/utils/manifest_only_fixtures.py,sha256=7HqCmsfNaAIjq2o9V9f-rgQdksncDZFfMifQpFzlLXo,2104
-airbyte_cdk/test/utils/reading.py,sha256=SOTDYlps6Te9KumfTJ3vVDSm9EUXhvKtE8aD7gvdPlg,965
+airbyte_cdk/test/utils/reading.py,sha256=9ReW2uoITE7NCpVBKn6EfM9yi9_SvqhsNLb-5LODka8,1289
 airbyte_cdk/utils/__init__.py,sha256=qhnC02DbS35OY8oB_tkYHwZzHed2FZeBM__G8IOgckY,347
 airbyte_cdk/utils/airbyte_secrets_utils.py,sha256=wEtRnl5KRhN6eLJwrDrC4FJjyqt_4vkA1F65mdl8c24,3142
 airbyte_cdk/utils/analytics_message.py,sha256=bi3uugQ2NjecnwTnz63iD5D1M8ZR8mXPbdtt6w5cC4s,653
@@ -420,9 +421,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.54.9.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.54.9.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.54.9.dist-info/METADATA,sha256=g20X2L94686rOWrD-xHQY8rlWv-t8o3ZGqVZuGf-Hik,6343
-airbyte_cdk-6.54.9.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.54.9.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
-airbyte_cdk-6.54.9.dist-info/RECORD,,
+airbyte_cdk-6.54.11.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.54.11.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.54.11.dist-info/METADATA,sha256=GAc-NcvDVzTVP4sUrNphva1hP0kyZt-ANEM5Qz5CgPY,6344
+airbyte_cdk-6.54.11.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.54.11.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
+airbyte_cdk-6.54.11.dist-info/RECORD,,

airbyte_cdk/test/standard_tests/models/__init__.py

@@ -1,7 +0,0 @@
-from airbyte_cdk.test.standard_tests.models.scenario import (
-    ConnectorTestScenario,
-)
-
-__all__ = [
-    "ConnectorTestScenario",
-]