airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py

@@ -0,0 +1,112 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+from copy import deepcopy
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Dict, Iterable, List, Mapping
+
+import dpath
+from typing_extensions import deprecated
+
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
+from airbyte_cdk.sources.declarative.resolvers.components_resolver import (
+    ComponentMappingDefinition,
+    ComponentsResolver,
+    ResolvedComponentMappingDefinition,
+)
+from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
+from airbyte_cdk.sources.source import ExperimentalClassWarning
+from airbyte_cdk.sources.types import Config
+
+
+@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
+@dataclass
+class HttpComponentsResolver(ComponentsResolver):
+    """
+    Resolves and populates stream templates with components fetched via an HTTP retriever.
+
+    Attributes:
+        retriever (Retriever): The retriever used to fetch data from an API.
+        config (Config): Configuration object for the resolver.
+        components_mapping (List[ComponentMappingDefinition]): List of mappings to resolve.
+        parameters (InitVar[Mapping[str, Any]]): Additional parameters for interpolation.
+    """
+
+    retriever: Retriever
+    config: Config
+    components_mapping: List[ComponentMappingDefinition]
+    parameters: InitVar[Mapping[str, Any]]
+    _resolved_components: List[ResolvedComponentMappingDefinition] = field(
+        init=False, repr=False, default_factory=list
+    )
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        """
+        Initializes and parses component mappings, converting them to resolved definitions.
+
+        Args:
+            parameters (Mapping[str, Any]): Parameters for interpolation.
+        """
+        for component_mapping in self.components_mapping:
+            if isinstance(component_mapping.value, (str, InterpolatedString)):
+                interpolated_value = (
+                    InterpolatedString.create(component_mapping.value, parameters=parameters)
+                    if isinstance(component_mapping.value, str)
+                    else component_mapping.value
+                )
+
+                field_path = [
+                    InterpolatedString.create(path, parameters=parameters)
+                    for path in component_mapping.field_path
+                ]
+
+                self._resolved_components.append(
+                    ResolvedComponentMappingDefinition(
+                        field_path=field_path,
+                        value=interpolated_value,
+                        value_type=component_mapping.value_type,
+                        parameters=parameters,
+                    )
+                )
+            else:
+                raise ValueError(
+                    f"Expected a string or InterpolatedString for value in mapping: {component_mapping}"
+                )
+
+    def resolve_components(
+        self, stream_template_config: Dict[str, Any]
+    ) -> Iterable[Dict[str, Any]]:
+        """
+        Resolves components in the stream template configuration by populating values.
+
+        Args:
+            stream_template_config (Dict[str, Any]): Stream template to populate.
+
+        Yields:
+            Dict[str, Any]: Updated configurations with resolved components.
+        """
+        kwargs = {"stream_template_config": stream_template_config}
+
+        for stream_slice in self.retriever.stream_slices():
+            for components_values in self.retriever.read_records(
+                records_schema={}, stream_slice=stream_slice
+            ):
+                updated_config = deepcopy(stream_template_config)
+                kwargs["components_values"] = components_values  # type: ignore[assignment] # component_values will always be of type Mapping[str, Any]
+                kwargs["stream_slice"] = stream_slice  # type: ignore[assignment] # stream_slice will always be of type Mapping[str, Any]
+
+                for resolved_component in self._resolved_components:
+                    valid_types = (
+                        (resolved_component.value_type,) if resolved_component.value_type else None
+                    )
+                    value = resolved_component.value.eval(
+                        self.config, valid_types=valid_types, **kwargs
+                    )
+
+                    path = [
+                        path.eval(self.config, **kwargs) for path in resolved_component.field_path
+                    ]
+                    dpath.set(updated_config, path, value)
+
+                yield updated_config

airbyte_cdk/sources/declarative/retrievers/__init__.py

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from airbyte_cdk.sources.declarative.retrievers.async_retriever import AsyncRetriever
+from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
+from airbyte_cdk.sources.declarative.retrievers.simple_retriever import (
+    LazySimpleRetriever,
+    SimpleRetriever,
+    SimpleRetrieverTestReadDecorator,
+)
+
+__all__ = [
+    "Retriever",
+    "SimpleRetriever",
+    "SimpleRetrieverTestReadDecorator",
+    "AsyncRetriever",
+    "LazySimpleRetriever",
+]

airbyte_cdk/sources/declarative/retrievers/async_retriever.py

@@ -0,0 +1,124 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Iterable, Mapping, Optional
+
+from airbyte_cdk.sources.declarative.async_job.job import AsyncJob
+from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector
+from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import (
+    AsyncJobPartitionRouter,
+)
+from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
+from airbyte_cdk.sources.streams.core import StreamData
+from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+from airbyte_cdk.sources.utils.slice_logger import AlwaysLogSliceLogger
+
+
+@dataclass
+class AsyncRetriever(Retriever):
+    config: Config
+    parameters: InitVar[Mapping[str, Any]]
+    record_selector: RecordSelector
+    stream_slicer: AsyncJobPartitionRouter
+    slice_logger: AlwaysLogSliceLogger = field(
+        init=False,
+        default_factory=lambda: AlwaysLogSliceLogger(),
+    )
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._parameters = parameters
+
+    @property
+    def exit_on_rate_limit(self) -> bool:
+        """
+        Whether to exit on rate limit. This is a property of the job repository
+        and not the stream slicer. The stream slicer is responsible for creating
+        the jobs, but the job repository is responsible for managing the rate
+        limits and other job-related properties.
+
+        Note:
+         - If the `creation_requester` cannot place / create the job - it might be the case of the RateLimits
+         - If the `creation_requester` can place / create the job - it means all other requesters should successfully manage
+           to complete the results.
+        """
+        job_orchestrator = self.stream_slicer._job_orchestrator
+        if job_orchestrator is None:
+            # Default value when orchestrator is not available
+            return False
+        return job_orchestrator._job_repository.creation_requester.exit_on_rate_limit  # type: ignore
+
+    @exit_on_rate_limit.setter
+    def exit_on_rate_limit(self, value: bool) -> None:
+        """
+        Sets the `exit_on_rate_limit` property of the job repository > creation_requester,
+        meaning that the Job cannot be placed / created if the rate limit is reached.
+        Thus no further work on managing jobs is expected to be done.
+        """
+        job_orchestrator = self.stream_slicer._job_orchestrator
+        if job_orchestrator is not None:
+            job_orchestrator._job_repository.creation_requester.exit_on_rate_limit = value  # type: ignore[attr-defined, assignment]
+
+    @property
+    def state(self) -> StreamState:
+        """
+        As a first iteration for sendgrid, there is no state to be managed
+        """
+        return {}
+
+    @state.setter
+    def state(self, value: StreamState) -> None:
+        """
+        As a first iteration for sendgrid, there is no state to be managed
+        """
+        pass
+
+    def _get_stream_state(self) -> StreamState:
+        """
+        Gets the current state of the stream.
+
+        Returns:
+            StreamState: Mapping[str, Any]
+        """
+
+        return self.state
+
+    def _validate_and_get_stream_slice_jobs(
+        self, stream_slice: Optional[StreamSlice] = None
+    ) -> Iterable[AsyncJob]:
+        """
+        Validates the stream_slice argument and returns the partition from it.
+
+        Args:
+            stream_slice (Optional[StreamSlice]): The stream slice to validate and extract the partition from.
+
+        Returns:
+            AsyncPartition: The partition extracted from the stream_slice.
+
+        Raises:
+            AirbyteTracedException: If the stream_slice is not an instance of StreamSlice or if the partition is not present in the stream_slice.
+
+        """
+        return stream_slice.extra_fields.get("jobs", []) if stream_slice else []
+
+    def stream_slices(self) -> Iterable[Optional[StreamSlice]]:
+        yield from self.stream_slicer.stream_slices()
+
+    def read_records(
+        self,
+        records_schema: Mapping[str, Any],
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Iterable[StreamData]:
+        # emit the slice_descriptor log message, for connector builder TestRead
+        yield self.slice_logger.create_slice_log_message(stream_slice.cursor_slice)  # type: ignore
+
+        stream_state: StreamState = self._get_stream_state()
+        jobs: Iterable[AsyncJob] = self._validate_and_get_stream_slice_jobs(stream_slice)
+        records: Iterable[Mapping[str, Any]] = self.stream_slicer.fetch_records(jobs)
+
+        yield from self.record_selector.filter_and_transform(
+            all_data=records,
+            stream_state=stream_state,
+            records_schema=records_schema,
+            stream_slice=stream_slice,
+        )

airbyte_cdk/sources/declarative/retrievers/file_uploader.py

@@ -0,0 +1,89 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+import json
+import logging
+import uuid
+from dataclasses import InitVar, dataclass, field
+from pathlib import Path
+from typing import Any, Mapping, Optional, Union
+
+from airbyte_cdk.models import AirbyteRecordMessageFileReference
+from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import (
+    InterpolatedString,
+)
+from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import (
+    SafeResponse,
+)
+from airbyte_cdk.sources.declarative.requesters import Requester
+from airbyte_cdk.sources.declarative.types import Record, StreamSlice
+from airbyte_cdk.sources.types import Config
+from airbyte_cdk.sources.utils.files_directory import get_files_directory
+
+logger = logging.getLogger("airbyte")
+
+
+@dataclass
+class FileUploader:
+    requester: Requester
+    download_target_extractor: RecordExtractor
+    config: Config
+    parameters: InitVar[Mapping[str, Any]]
+
+    filename_extractor: Optional[Union[InterpolatedString, str]] = None
+    content_extractor: Optional[RecordExtractor] = None
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        if self.filename_extractor:
+            self.filename_extractor = InterpolatedString.create(
+                self.filename_extractor,
+                parameters=parameters,
+            )
+
+    def upload(self, record: Record) -> None:
+        mocked_response = SafeResponse()
+        mocked_response.content = json.dumps(record.data).encode()
+        download_target = list(self.download_target_extractor.extract_records(mocked_response))[0]
+        if not isinstance(download_target, str):
+            raise ValueError(
+                f"download_target is expected to be a str but was {type(download_target)}: {download_target}"
+            )
+
+        response = self.requester.send_request(
+            stream_slice=StreamSlice(
+                partition={}, cursor_slice={}, extra_fields={"download_target": download_target}
+            ),
+        )
+
+        if self.content_extractor:
+            raise NotImplementedError("TODO")
+        else:
+            files_directory = Path(get_files_directory())
+
+            file_name = (
+                self.filename_extractor.eval(self.config, record=record)
+                if self.filename_extractor
+                else str(uuid.uuid4())
+            )
+            file_name = file_name.lstrip("/")
+            file_relative_path = Path(record.stream_name) / Path(file_name)
+
+            full_path = files_directory / file_relative_path
+            full_path.parent.mkdir(parents=True, exist_ok=True)
+
+            with open(str(full_path), "wb") as f:
+                f.write(response.content)
+            file_size_bytes = full_path.stat().st_size
+
+            logger.info("File uploaded successfully")
+            logger.info(f"File url: {str(full_path)}")
+            logger.info(f"File size: {file_size_bytes / 1024} KB")
+            logger.info(f"File relative path: {str(file_relative_path)}")
+
+            record.file_reference = AirbyteRecordMessageFileReference(
+                staging_file_url=str(full_path),
+                source_file_relative_path=str(file_relative_path),
+                file_size_bytes=file_size_bytes,
+            )

airbyte_cdk/sources/declarative/retrievers/retriever.py

@@ -0,0 +1,54 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import abstractmethod
+from typing import Any, Iterable, Mapping, Optional
+
+from airbyte_cdk.sources.declarative.incremental.per_partition_cursor import StreamSlice
+from airbyte_cdk.sources.streams.core import StreamData
+from airbyte_cdk.sources.types import StreamState
+
+
+class Retriever:
+    """
+    Responsible for fetching a stream's records from an HTTP API source.
+    """
+
+    @abstractmethod
+    def read_records(
+        self,
+        records_schema: Mapping[str, Any],
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Iterable[StreamData]:
+        """
+        Fetch a stream's records from an HTTP API source
+
+        :param records_schema: json schema to describe record
+        :param stream_slice: The stream slice to read data for
+        :return: The records read from the API source
+        """
+
+    @abstractmethod
+    def stream_slices(self) -> Iterable[Optional[StreamSlice]]:
+        """Returns the stream slices"""
+
+    @property
+    @abstractmethod
+    def state(self) -> StreamState:
+        """State getter, should return state in form that can serialized to a string and send to the output
+        as a STATE AirbyteMessage.
+
+        A good example of a state is a cursor_value:
+            {
+                self.cursor_field: "cursor_value"
+            }
+
+         State should try to be as small as possible but at the same time descriptive enough to restore
+         syncing process from the point where it stopped.
+        """
+
+    @state.setter
+    @abstractmethod
+    def state(self, value: StreamState) -> None:
+        """State setter, accept state serialized by state getter."""