airbyte-cdk 6.45.0.dev4107__tar.gz → 6.45.0.post6.dev14369631849__tar.gz

{airbyte_cdk-6.45.0.dev4107 → airbyte_cdk-6.45.0.post6.dev14369631849}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.45.0.dev4107
+Version: 6.45.0.post6.dev14369631849
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -22,7 +22,7 @@ Provides-Extra: sql
 Provides-Extra: vector-db-based
 Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
-Requires-Dist: airbyte-protocol-models-dataclasses (>=0.15,<0.16)
+Requires-Dist: airbyte-protocol-models-dataclasses (>=0.14,<0.15)
 Requires-Dist: anyascii (>=0.3.2,<0.4.0)
 Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
 Requires-Dist: backoff

{airbyte_cdk-6.45.0.dev4107 → airbyte_cdk-6.45.0.post6.dev14369631849}/airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -4,7 +4,7 @@
 
 
 from dataclasses import asdict, dataclass, field
-from typing import Any, List, Mapping
+from typing import Any, Dict, List, Mapping
 
 from airbyte_cdk.connector_builder.test_reader import TestReader
 from airbyte_cdk.models import (
@@ -27,30 +27,34 @@ from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE = 5
 DEFAULT_MAXIMUM_NUMBER_OF_SLICES = 5
 DEFAULT_MAXIMUM_RECORDS = 100
+DEFAULT_MAXIMUM_STREAMS = 100
 
 MAX_PAGES_PER_SLICE_KEY = "max_pages_per_slice"
 MAX_SLICES_KEY = "max_slices"
 MAX_RECORDS_KEY = "max_records"
+MAX_STREAMS_KEY = "max_streams"
 
 
 @dataclass
-class TestReadLimits:
+class TestLimits:
     max_records: int = field(default=DEFAULT_MAXIMUM_RECORDS)
     max_pages_per_slice: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE)
     max_slices: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_SLICES)
+    max_streams: int = field(default=DEFAULT_MAXIMUM_STREAMS)
 
 
-def get_limits(config: Mapping[str, Any]) -> TestReadLimits:
+def get_limits(config: Mapping[str, Any]) -> TestLimits:
     command_config = config.get("__test_read_config", {})
     max_pages_per_slice = (
         command_config.get(MAX_PAGES_PER_SLICE_KEY) or DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE
     )
     max_slices = command_config.get(MAX_SLICES_KEY) or DEFAULT_MAXIMUM_NUMBER_OF_SLICES
     max_records = command_config.get(MAX_RECORDS_KEY) or DEFAULT_MAXIMUM_RECORDS
-    return TestReadLimits(max_records, max_pages_per_slice, max_slices)
+    max_streams = command_config.get(MAX_STREAMS_KEY) or DEFAULT_MAXIMUM_STREAMS
+    return TestLimits(max_records, max_pages_per_slice, max_slices, max_streams)
 
 
-def create_source(config: Mapping[str, Any], limits: TestReadLimits) -> ManifestDeclarativeSource:
+def create_source(config: Mapping[str, Any], limits: TestLimits) -> ManifestDeclarativeSource:
     manifest = config["__injected_declarative_manifest"]
     return ManifestDeclarativeSource(
         config=config,
@@ -71,7 +75,7 @@ def read_stream(
     config: Mapping[str, Any],
     configured_catalog: ConfiguredAirbyteCatalog,
     state: List[AirbyteStateMessage],
-    limits: TestReadLimits,
+    limits: TestLimits,
 ) -> AirbyteMessage:
     try:
         test_read_handler = TestReader(
@@ -117,5 +121,40 @@ def resolve_manifest(source: ManifestDeclarativeSource) -> AirbyteMessage:
         return error.as_airbyte_message()
 
 
+def full_resolve_manifest(source: ManifestDeclarativeSource, limits: TestLimits) -> AirbyteMessage:
+    try:
+        manifest = {**source.resolved_manifest}
+        streams = manifest.get("streams", [])
+        for stream in streams:
+            stream["dynamic_stream_name"] = None
+
+        mapped_streams: Dict[str, List[Dict[str, Any]]] = {}
+        for stream in source.dynamic_streams:
+            generated_streams = mapped_streams.setdefault(stream["dynamic_stream_name"], [])
+
+            if len(generated_streams) < limits.max_streams:
+                generated_streams += [stream]
+
+        for generated_streams_list in mapped_streams.values():
+            streams.extend(generated_streams_list)
+
+        manifest["streams"] = streams
+        return AirbyteMessage(
+            type=Type.RECORD,
+            record=AirbyteRecordMessage(
+                data={"manifest": manifest},
+                emitted_at=_emitted_at(),
+                stream="full_resolve_manifest",
+            ),
+        )
+    except AirbyteTracedException as exc:
+        return exc.as_airbyte_message()
+    except Exception as exc:
+        error = AirbyteTracedException.from_exception(
+            exc, message=f"Error full resolving manifest: {str(exc)}"
+        )
+        return error.as_airbyte_message()
+
+
 def _emitted_at() -> int:
     return ab_datetime_now().to_epoch_millis()

{airbyte_cdk-6.45.0.dev4107 → airbyte_cdk-6.45.0.post6.dev14369631849}/airbyte_cdk/connector_builder/main.py

@@ -10,8 +10,9 @@ import orjson
 
 from airbyte_cdk.connector import BaseConnector
 from airbyte_cdk.connector_builder.connector_builder_handler import (
-    TestReadLimits,
+    TestLimits,
     create_source,
+    full_resolve_manifest,
     get_limits,
     read_stream,
     resolve_manifest,
@@ -72,7 +73,7 @@ def handle_connector_builder_request(
     config: Mapping[str, Any],
     catalog: Optional[ConfiguredAirbyteCatalog],
     state: List[AirbyteStateMessage],
-    limits: TestReadLimits,
+    limits: TestLimits,
 ) -> AirbyteMessage:
     if command == "resolve_manifest":
         return resolve_manifest(source)
@@ -81,6 +82,8 @@ def handle_connector_builder_request(
             catalog is not None
         ), "`test_read` requires a valid `ConfiguredAirbyteCatalog`, got None."
         return read_stream(source, config, catalog, state, limits)
+    elif command == "full_resolve_manifest":
+        return full_resolve_manifest(source, limits)
     else:
         raise ValueError(f"Unrecognized command {command}.")
 

{airbyte_cdk-6.45.0.dev4107 → airbyte_cdk-6.45.0.post6.dev14369631849}/airbyte_cdk/models/__init__.py

@@ -19,7 +19,6 @@ from .airbyte_protocol import (
     AirbyteMessage,
     AirbyteProtocol,
     AirbyteRecordMessage,
-    AirbyteRecordMessageFileReference,
     AirbyteStateBlob,
     AirbyteStateMessage,
     AirbyteStateStats,

{airbyte_cdk-6.45.0.dev4107 → airbyte_cdk-6.45.0.post6.dev14369631849}/airbyte_cdk/models/airbyte_protocol.py

@@ -8,6 +8,8 @@ from typing import Annotated, Any, Dict, List, Mapping, Optional, Union
 from airbyte_protocol_dataclasses.models import *  # noqa: F403  # Allow '*'
 from serpyco_rs.metadata import Alias
 
+from airbyte_cdk.models.file_transfer_record_message import AirbyteFileTransferRecordMessage
+
 # ruff: noqa: F405  # ignore fuzzy import issues with 'import *'
 
 
@@ -82,7 +84,7 @@ class AirbyteMessage:
     spec: Optional[ConnectorSpecification] = None  # type: ignore [name-defined]
     connectionStatus: Optional[AirbyteConnectionStatus] = None  # type: ignore [name-defined]
     catalog: Optional[AirbyteCatalog] = None  # type: ignore [name-defined]
-    record: Optional[AirbyteRecordMessage] = None  # type: ignore [name-defined]
+    record: Optional[Union[AirbyteFileTransferRecordMessage, AirbyteRecordMessage]] = None  # type: ignore [name-defined]
     state: Optional[AirbyteStateMessage] = None
     trace: Optional[AirbyteTraceMessage] = None  # type: ignore [name-defined]
     control: Optional[AirbyteControlMessage] = None  # type: ignore [name-defined]

airbyte_cdk-6.45.0.post6.dev14369631849/airbyte_cdk/models/file_transfer_record_message.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+
+@dataclass
+class AirbyteFileTransferRecordMessage:
+    stream: str
+    file: Dict[str, Any]
+    emitted_at: int
+    namespace: Optional[str] = None
+    data: Optional[Dict[str, Any]] = None

{airbyte_cdk-6.45.0.dev4107 → airbyte_cdk-6.45.0.post6.dev14369631849}/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py

@@ -149,7 +149,7 @@ class ConcurrentReadProcessor:
         message = stream_data_to_airbyte_message(
             stream_name=record.stream_name,
             data_or_message=record.data,
-            file_reference=record.file_reference,
+            is_file_transfer_message=record.is_file_transfer_message,
         )
         stream = self._stream_name_to_instance[record.stream_name]
 

{airbyte_cdk-6.45.0.dev4107 → airbyte_cdk-6.45.0.post6.dev14369631849}/airbyte_cdk/sources/declarative/async_job/job.py

@@ -34,6 +34,12 @@ class AsyncJob:
 
     def status(self) -> AsyncJobStatus:
         if self._timer.has_timed_out():
+            # TODO: we should account the fact that,
+            # certain APIs could send the `Timeout` status,
+            # thus we should not return `Timeout` in that case,
+            # but act based on the scenario.
+
+            # the default behavior is to return `Timeout` status and retry.
             return AsyncJobStatus.TIMED_OUT
         return self._status
 

{airbyte_cdk-6.45.0.dev4107 → airbyte_cdk-6.45.0.post6.dev14369631849}/airbyte_cdk/sources/declarative/async_job/job_orchestrator.py

@@ -44,16 +44,21 @@ class AsyncPartition:
     This bucket of api_jobs is a bit useless for this iteration but should become interesting when we will be able to split jobs
     """
 
-    _MAX_NUMBER_OF_ATTEMPTS = 3
+    _DEFAULT_MAX_JOB_RETRY = 3
 
-    def __init__(self, jobs: List[AsyncJob], stream_slice: StreamSlice) -> None:
+    def __init__(
+        self, jobs: List[AsyncJob], stream_slice: StreamSlice, job_max_retry: Optional[int] = None
+    ) -> None:
         self._attempts_per_job = {job: 1 for job in jobs}
         self._stream_slice = stream_slice
+        self._job_max_retry = (
+            job_max_retry if job_max_retry is not None else self._DEFAULT_MAX_JOB_RETRY
+        )
 
     def has_reached_max_attempt(self) -> bool:
         return any(
             map(
-                lambda attempt_count: attempt_count >= self._MAX_NUMBER_OF_ATTEMPTS,
+                lambda attempt_count: attempt_count >= self._job_max_retry,
                 self._attempts_per_job.values(),
             )
         )
@@ -62,7 +67,7 @@ class AsyncPartition:
         current_attempt_count = self._attempts_per_job.pop(job_to_replace, None)
         if current_attempt_count is None:
             raise ValueError("Could not find job to replace")
-        elif current_attempt_count >= self._MAX_NUMBER_OF_ATTEMPTS:
+        elif current_attempt_count >= self._job_max_retry:
             raise ValueError(f"Max attempt reached for job in partition {self._stream_slice}")
 
         new_attempt_count = current_attempt_count + 1
@@ -155,6 +160,7 @@ class AsyncJobOrchestrator:
         message_repository: MessageRepository,
         exceptions_to_break_on: Iterable[Type[Exception]] = tuple(),
         has_bulk_parent: bool = False,
+        job_max_retry: Optional[int] = None,
     ) -> None:
         """
         If the stream slices provided as a parameters relies on a async job streams that relies on the same JobTracker, `has_bulk_parent`
@@ -175,11 +181,12 @@ class AsyncJobOrchestrator:
         self._message_repository = message_repository
         self._exceptions_to_break_on: Tuple[Type[Exception], ...] = tuple(exceptions_to_break_on)
         self._has_bulk_parent = has_bulk_parent
+        self._job_max_retry = job_max_retry
 
         self._non_breaking_exceptions: List[Exception] = []
 
     def _replace_failed_jobs(self, partition: AsyncPartition) -> None:
-        failed_status_jobs = (AsyncJobStatus.FAILED,)
+        failed_status_jobs = (AsyncJobStatus.FAILED, AsyncJobStatus.TIMED_OUT)
         jobs_to_replace = [job for job in partition.jobs if job.status() in failed_status_jobs]
         for job in jobs_to_replace:
             new_job = self._start_job(job.job_parameters(), job.api_job_id())
@@ -214,7 +221,7 @@ class AsyncJobOrchestrator:
             for _slice in self._slice_iterator:
                 at_least_one_slice_consumed_from_slice_iterator_during_current_iteration = True
                 job = self._start_job(_slice)
-                self._running_partitions.append(AsyncPartition([job], _slice))
+                self._running_partitions.append(AsyncPartition([job], _slice, self._job_max_retry))
                 if self._has_bulk_parent and self._slice_iterator.has_next():
                     break
         except ConcurrentJobLimitReached:
@@ -363,7 +370,7 @@ class AsyncJobOrchestrator:
                     self._reallocate_partition(current_running_partitions, partition)
 
             # We only remove completed / timeout jobs jobs as we want failed jobs to be re-allocated in priority
-            self._remove_completed_or_timed_out_jobs(partition)
+            self._remove_completed_jobs(partition)
 
         # update the referenced list with running partitions
         self._running_partitions = current_running_partitions
@@ -378,11 +385,7 @@ class AsyncJobOrchestrator:
     def _stop_timed_out_jobs(self, partition: AsyncPartition) -> None:
         for job in partition.jobs:
             if job.status() == AsyncJobStatus.TIMED_OUT:
-                self._abort_job(job, free_job_allocation=True)
-                raise AirbyteTracedException(
-                    internal_message=f"Job {job.api_job_id()} has timed out. Try increasing the `polling job timeout`.",
-                    failure_type=FailureType.config_error,
-                )
+                self._abort_job(job, free_job_allocation=False)
 
     def _abort_job(self, job: AsyncJob, free_job_allocation: bool = True) -> None:
         try:
@@ -392,7 +395,7 @@ class AsyncJobOrchestrator:
         except Exception as exception:
             LOGGER.warning(f"Could not free budget for job {job.api_job_id()}: {exception}")
 
-    def _remove_completed_or_timed_out_jobs(self, partition: AsyncPartition) -> None:
+    def _remove_completed_jobs(self, partition: AsyncPartition) -> None:
         """
         Remove completed or timed out jobs from the partition.
 
@@ -400,7 +403,7 @@ class AsyncJobOrchestrator:
             partition (AsyncPartition): The partition to process.
         """
         for job in partition.jobs:
-            if job.status() in [AsyncJobStatus.COMPLETED, AsyncJobStatus.TIMED_OUT]:
+            if job.status() == AsyncJobStatus.COMPLETED:
                 self._job_tracker.remove_job(job.api_job_id())
 
     def _reallocate_partition(
@@ -415,10 +418,7 @@ class AsyncJobOrchestrator:
             current_running_partitions (list): The list of currently running partitions.
             partition (AsyncPartition): The partition to reallocate.
         """
-        for job in partition.jobs:
-            if job.status() != AsyncJobStatus.TIMED_OUT:
-                # allow the FAILED jobs to be re-allocated for partition
-                current_running_partitions.insert(0, partition)
+        current_running_partitions.insert(0, partition)
 
     def _process_partitions_with_errors(self, partition: AsyncPartition) -> None:
         """

{airbyte_cdk-6.45.0.dev4107 → airbyte_cdk-6.45.0.post6.dev14369631849}/airbyte_cdk/sources/declarative/async_job/job_tracker.py

@@ -3,9 +3,11 @@
 import logging
 import threading
 import uuid
-from typing import Set
+from dataclasses import dataclass, field
+from typing import Any, Mapping, Set, Union
 
 from airbyte_cdk.logger import lazy_log
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 
 LOGGER = logging.getLogger("airbyte")
 
@@ -14,15 +16,29 @@ class ConcurrentJobLimitReached(Exception):
     pass
 
 
+@dataclass
 class JobTracker:
-    def __init__(self, limit: int):
+    limit: Union[int, str]
+    config: Mapping[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
         self._jobs: Set[str] = set()
-        if limit < 1:
+        self._lock = threading.Lock()
+        if isinstance(self.limit, str):
+            try:
+                self.limit = int(
+                    InterpolatedString(self.limit, parameters={}).eval(config=self.config)
+                )
+            except Exception as e:
+                LOGGER.warning(
+                    f"Error interpolating max job count: {self.limit}. Setting to 1. {e}"
+                )
+                self.limit = 1
+        if self.limit < 1:
             LOGGER.warning(
-                f"The `max_concurrent_async_job_count` property is less than 1: {limit}. Setting to 1. Please update the source manifest to set a valid value."
+                f"The `max_concurrent_async_job_count` property is less than 1: {self.limit}. Setting to 1. Please update the source manifest to set a valid value."
             )
-        self._limit = 1 if limit < 1 else limit
-        self._lock = threading.Lock()
+        self._limit = self.limit if self.limit >= 1 else 1
 
     def try_to_get_intent(self) -> str:
         lazy_log(

{airbyte_cdk-6.45.0.dev4107 → airbyte_cdk-6.45.0.post6.dev14369631849}/airbyte_cdk/sources/declarative/checks/__init__.py

@@ -7,7 +7,10 @@ from typing import Mapping
 from pydantic.v1 import BaseModel
 
 from airbyte_cdk.sources.declarative.checks.check_dynamic_stream import CheckDynamicStream
-from airbyte_cdk.sources.declarative.checks.check_stream import CheckStream
+from airbyte_cdk.sources.declarative.checks.check_stream import (
+    CheckStream,
+    DynamicStreamCheckConfig,
+)
 from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
 from airbyte_cdk.sources.declarative.models import (
     CheckDynamicStream as CheckDynamicStreamModel,
@@ -21,4 +24,4 @@ COMPONENTS_CHECKER_TYPE_MAPPING: Mapping[str, type[BaseModel]] = {
     "CheckDynamicStream": CheckDynamicStreamModel,
 }
 
-__all__ = ["CheckStream", "CheckDynamicStream", "ConnectionChecker"]
+__all__ = ["CheckStream", "CheckDynamicStream", "ConnectionChecker", "DynamicStreamCheckConfig"]

airbyte_cdk-6.45.0.post6.dev14369631849/airbyte_cdk/sources/declarative/checks/check_stream.py

@@ -0,0 +1,158 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+import traceback
+from dataclasses import InitVar, dataclass
+from typing import Any, Dict, List, Mapping, Optional, Tuple
+
+from airbyte_cdk import AbstractSource
+from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
+from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy
+
+
+@dataclass(frozen=True)
+class DynamicStreamCheckConfig:
+    """Defines the configuration for dynamic stream during connection checking. This class specifies
+    what dynamic streams  in the stream template should be updated with value, supporting dynamic interpolation
+    and type enforcement."""
+
+    dynamic_stream_name: str
+    stream_count: int = 0
+
+
+@dataclass
+class CheckStream(ConnectionChecker):
+    """
+    Checks the connections by checking availability of one or many streams selected by the developer
+
+    Attributes:
+        stream_name (List[str]): names of streams to check
+    """
+
+    stream_names: List[str]
+    parameters: InitVar[Mapping[str, Any]]
+    dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._parameters = parameters
+        if self.dynamic_streams_check_configs is None:
+            self.dynamic_streams_check_configs = []
+
+    def _log_error(self, logger: logging.Logger, action: str, error: Exception) -> Tuple[bool, str]:
+        """Logs an error and returns a formatted error message."""
+        error_message = f"Encountered an error while {action}. Error: {error}"
+        logger.error(error_message + f"Error traceback: \n {traceback.format_exc()}", exc_info=True)
+        return False, error_message
+
+    def check_connection(
+        self, source: AbstractSource, logger: logging.Logger, config: Mapping[str, Any]
+    ) -> Tuple[bool, Any]:
+        """Checks the connection to the source and its streams."""
+        try:
+            streams = source.streams(config=config)
+            if not streams:
+                return False, f"No streams to connect to from source {source}"
+        except Exception as error:
+            return self._log_error(logger, "discovering streams", error)
+
+        stream_name_to_stream = {s.name: s for s in streams}
+        for stream_name in self.stream_names:
+            if stream_name not in stream_name_to_stream:
+                raise ValueError(
+                    f"{stream_name} is not part of the catalog. Expected one of {list(stream_name_to_stream.keys())}."
+                )
+
+            stream_availability, message = self._check_stream_availability(
+                stream_name_to_stream, stream_name, logger
+            )
+            if not stream_availability:
+                return stream_availability, message
+
+        should_check_dynamic_streams = (
+            hasattr(source, "resolved_manifest")
+            and hasattr(source, "dynamic_streams")
+            and self.dynamic_streams_check_configs
+        )
+
+        if should_check_dynamic_streams:
+            return self._check_dynamic_streams_availability(source, stream_name_to_stream, logger)
+
+        return True, None
+
+    def _check_stream_availability(
+        self, stream_name_to_stream: Dict[str, Any], stream_name: str, logger: logging.Logger
+    ) -> Tuple[bool, Any]:
+        """Checks if streams are available."""
+        availability_strategy = HttpAvailabilityStrategy()
+        try:
+            stream = stream_name_to_stream[stream_name]
+            stream_is_available, reason = availability_strategy.check_availability(stream, logger)
+            if not stream_is_available:
+                message = f"Stream {stream_name} is not available: {reason}"
+                logger.warning(message)
+                return stream_is_available, message
+        except Exception as error:
+            return self._log_error(logger, f"checking availability of stream {stream_name}", error)
+        return True, None
+
+    def _check_dynamic_streams_availability(
+        self, source: AbstractSource, stream_name_to_stream: Dict[str, Any], logger: logging.Logger
+    ) -> Tuple[bool, Any]:
+        """Checks the availability of dynamic streams."""
+        dynamic_streams = source.resolved_manifest.get("dynamic_streams", [])  # type: ignore[attr-defined] # The source's resolved_manifest manifest is checked before calling this method
+        dynamic_stream_name_to_dynamic_stream = {
+            ds.get("name", f"dynamic_stream_{i}"): ds for i, ds in enumerate(dynamic_streams)
+        }
+        generated_streams = self._map_generated_streams(source.dynamic_streams)  # type: ignore[attr-defined] # The source's dynamic_streams manifest is checked before calling this method
+
+        for check_config in self.dynamic_streams_check_configs:  # type: ignore[union-attr] # None value for self.dynamic_streams_check_configs handled in __post_init__
+            if check_config.dynamic_stream_name not in dynamic_stream_name_to_dynamic_stream:
+                return (
+                    False,
+                    f"Dynamic stream {check_config.dynamic_stream_name} is not found in manifest.",
+                )
+
+            generated = generated_streams.get(check_config.dynamic_stream_name, [])
+            stream_availability, message = self._check_generated_streams_availability(
+                generated, stream_name_to_stream, logger, check_config.stream_count
+            )
+            if not stream_availability:
+                return stream_availability, message
+
+        return True, None
+
+    def _map_generated_streams(
+        self, dynamic_streams: List[Dict[str, Any]]
+    ) -> Dict[str, List[Dict[str, Any]]]:
+        """Maps dynamic stream names to their corresponding generated streams."""
+        mapped_streams: Dict[str, List[Dict[str, Any]]] = {}
+        for stream in dynamic_streams:
+            mapped_streams.setdefault(stream["dynamic_stream_name"], []).append(stream)
+        return mapped_streams
+
+    def _check_generated_streams_availability(
+        self,
+        generated_streams: List[Dict[str, Any]],
+        stream_name_to_stream: Dict[str, Any],
+        logger: logging.Logger,
+        max_count: int,
+    ) -> Tuple[bool, Any]:
+        """Checks availability of generated dynamic streams."""
+        availability_strategy = HttpAvailabilityStrategy()
+        for declarative_stream in generated_streams[: min(max_count, len(generated_streams))]:
+            stream = stream_name_to_stream[declarative_stream["name"]]
+            try:
+                stream_is_available, reason = availability_strategy.check_availability(
+                    stream, logger
+                )
+                if not stream_is_available:
+                    message = f"Dynamic Stream {stream.name} is not available: {reason}"
+                    logger.warning(message)
+                    return False, message
+            except Exception as error:
+                return self._log_error(
+                    logger, f"checking availability of dynamic stream {stream.name}", error
+                )
+        return True, None

{airbyte_cdk-6.45.0.dev4107 → airbyte_cdk-6.45.0.post6.dev14369631849}/airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -25,7 +25,6 @@ from airbyte_cdk.sources.declarative.incremental.per_partition_with_global impor
     PerPartitionWithGlobalCursor,
 )
 from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
-from airbyte_cdk.sources.declarative.models import FileUploader
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
     ConcurrencyLevel as ConcurrencyLevelModel,
 )
@@ -207,10 +206,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             # these legacy Python streams the way we do low-code streams to determine if they are concurrent compatible,
             # so we need to treat them as synchronous
 
-            supports_file_transfer = (
-                "file_uploader" in name_to_stream_mapping[declarative_stream.name]
-            )
-
             if (
                 isinstance(declarative_stream, DeclarativeStream)
                 and name_to_stream_mapping[declarative_stream.name]["type"]
@@ -327,7 +322,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                             else None,
                             logger=self.logger,
                             cursor=cursor,
-                            supports_file_transfer=supports_file_transfer,
                         )
                     )
                 elif (
@@ -359,7 +353,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                             cursor_field=None,
                             logger=self.logger,
                             cursor=final_state_cursor,
-                            supports_file_transfer=supports_file_transfer,
                         )
                     )
                 elif (
@@ -413,7 +406,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
                             cursor_field=perpartition_cursor.cursor_field.cursor_field_key,
                             logger=self.logger,
                             cursor=perpartition_cursor,
-                            supports_file_transfer=supports_file_transfer,
                         )
                     )
                 else: