airbyte-cdk 6.37.0.dev0__py3-none-any.whl → 6.37.2.dev1__py3-none-any.whl

airbyte_cdk/entrypoint.py

@@ -37,8 +37,8 @@ from airbyte_cdk.sources import Source
 from airbyte_cdk.sources.connector_state_manager import HashableStreamDescriptor
 from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config
 
-from airbyte_cdk.utils import PrintBuffer, is_cloud_environment, message_utils  # add PrintBuffer back once fixed
-# from airbyte_cdk.utils import is_cloud_environment, message_utils
+# from airbyte_cdk.utils import PrintBuffer, is_cloud_environment, message_utils  # add PrintBuffer back once fixed
+from airbyte_cdk.utils import is_cloud_environment, message_utils
 from airbyte_cdk.utils.airbyte_secrets_utils import get_secrets, update_secrets
 from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
@@ -337,11 +337,11 @@ def launch(source: Source, args: List[str]) -> None:
     parsed_args = source_entrypoint.parse_args(args)
     # temporarily removes the PrintBuffer because we're seeing weird print behavior for concurrent syncs
     # Refer to: https://github.com/airbytehq/oncall/issues/6235
-    with PrintBuffer():
-        for message in source_entrypoint.run(parsed_args):
-            # simply printing is creating issues for concurrent CDK as Python uses different two instructions to print: one for the message and
-            # the other for the break line. Adding `\n` to the message ensure that both are printed at the same time
-            print(f"{message}\n", end="", flush=True)
+    # with PrintBuffer():
+    for message in source_entrypoint.run(parsed_args):
+        # simply printing is creating issues for concurrent CDK as Python uses different two instructions to print: one for the message and
+        # the other for the break line. Adding `\n` to the message ensure that both are printed at the same time
+        print(f"{message}\n", end="", flush=True)
 
 
 def _init_internal_request_filter() -> None:

airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py

@@ -5,7 +5,7 @@ import json
 import logging
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from io import BufferedIOBase, TextIOWrapper
+from io import BufferedIOBase, BytesIO, TextIOWrapper
 from typing import Any, Generator, MutableMapping, Optional
 
 import orjson
@@ -124,7 +124,8 @@ class CsvParser(Parser):
         """
         Parse CSV data from decompressed bytes.
         """
-        text_data = TextIOWrapper(data, encoding=self.encoding)  # type: ignore
+        bytes_data = BytesIO(data.read())
+        text_data = TextIOWrapper(bytes_data, encoding=self.encoding)  # type: ignore
         reader = csv.DictReader(text_data, delimiter=self._get_delimiter() or ",")
         for row in reader:
             yield row

airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py

@@ -136,6 +136,7 @@ class ResponseToFileExtractor(RecordExtractor):
         """
 
         try:
+            # TODO: Add support for other file types, like `json`, with `pd.read_json()`
             with open(path, "r", encoding=file_encoding) as data:
                 chunks = pd.read_csv(
                     data, chunksize=chunk_size, iterator=True, dialect="unix", dtype=object

airbyte_cdk/sources/declarative/requesters/http_job_repository.py

@@ -23,6 +23,7 @@ from airbyte_cdk.sources.declarative.extractors.response_to_file_extractor impor
 )
 from airbyte_cdk.sources.declarative.requesters.requester import Requester
 from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever
+from airbyte_cdk.sources.http_logger import format_http_message
 from airbyte_cdk.sources.types import Record, StreamSlice
 from airbyte_cdk.utils import AirbyteTracedException
 
@@ -71,7 +72,15 @@ class AsyncHttpJobRepository(AsyncJobRepository):
         """
 
         polling_response: Optional[requests.Response] = self.polling_requester.send_request(
-            stream_slice=stream_slice
+            stream_slice=stream_slice,
+            log_formatter=lambda polling_response: format_http_message(
+                response=polling_response,
+                title="Async Job -- Polling",
+                description="Poll the status of the server-side async job.",
+                stream_name=None,
+                is_auxiliary=True,
+                type="ASYNC_POLL",
+            ),
         )
         if polling_response is None:
             raise AirbyteTracedException(
@@ -118,8 +127,17 @@ class AsyncHttpJobRepository(AsyncJobRepository):
         """
 
         response: Optional[requests.Response] = self.creation_requester.send_request(
-            stream_slice=stream_slice
+            stream_slice=stream_slice,
+            log_formatter=lambda response: format_http_message(
+                response=response,
+                title="Async Job -- Create",
+                description="Create the server-side async job.",
+                stream_name=None,
+                is_auxiliary=True,
+                type="ASYNC_CREATE",
+            ),
         )
+
         if not response:
             raise AirbyteTracedException(
                 internal_message="Always expect a response or an exception from creation_requester",
@@ -217,13 +235,33 @@ class AsyncHttpJobRepository(AsyncJobRepository):
         if not self.abort_requester:
             return
 
-        self.abort_requester.send_request(stream_slice=self._get_create_job_stream_slice(job))
+        abort_response = self.abort_requester.send_request(
+            stream_slice=self._get_create_job_stream_slice(job),
+            log_formatter=lambda abort_response: format_http_message(
+                response=abort_response,
+                title="Async Job -- Abort",
+                description="Abort the running server-side async job.",
+                stream_name=None,
+                is_auxiliary=True,
+                type="ASYNC_ABORT",
+            ),
+        )
 
     def delete(self, job: AsyncJob) -> None:
         if not self.delete_requester:
             return
 
-        self.delete_requester.send_request(stream_slice=self._get_create_job_stream_slice(job))
+        delete_job_reponse = self.delete_requester.send_request(
+            stream_slice=self._get_create_job_stream_slice(job),
+            log_formatter=lambda delete_job_reponse: format_http_message(
+                response=delete_job_reponse,
+                title="Async Job -- Delete",
+                description="Delete the specified job from the list of Jobs.",
+                stream_name=None,
+                is_auxiliary=True,
+                type="ASYNC_DELETE",
+            ),
+        )
         self._clean_up_job(job.api_job_id())
 
     def _clean_up_job(self, job_id: str) -> None:

airbyte_cdk/sources/declarative/retrievers/async_retriever.py

@@ -1,13 +1,12 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 
 
-from dataclasses import InitVar, dataclass
+from dataclasses import InitVar, dataclass, field
 from typing import Any, Iterable, Mapping, Optional
 
 from typing_extensions import deprecated
 
 from airbyte_cdk.sources.declarative.async_job.job import AsyncJob
-from airbyte_cdk.sources.declarative.async_job.job_orchestrator import AsyncPartition
 from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector
 from airbyte_cdk.sources.declarative.partition_routers.async_job_partition_router import (
     AsyncJobPartitionRouter,
@@ -16,6 +15,7 @@ from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
 from airbyte_cdk.sources.source import ExperimentalClassWarning
 from airbyte_cdk.sources.streams.core import StreamData
 from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+from airbyte_cdk.sources.utils.slice_logger import AlwaysLogSliceLogger
 
 
 @deprecated(
@@ -28,6 +28,10 @@ class AsyncRetriever(Retriever):
     parameters: InitVar[Mapping[str, Any]]
     record_selector: RecordSelector
     stream_slicer: AsyncJobPartitionRouter
+    slice_logger: AlwaysLogSliceLogger = field(
+        init=False,
+        default_factory=lambda: AlwaysLogSliceLogger(),
+    )
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._parameters = parameters
@@ -75,13 +79,16 @@ class AsyncRetriever(Retriever):
         return stream_slice.extra_fields.get("jobs", []) if stream_slice else []
 
     def stream_slices(self) -> Iterable[Optional[StreamSlice]]:
-        return self.stream_slicer.stream_slices()
+        yield from self.stream_slicer.stream_slices()
 
     def read_records(
         self,
         records_schema: Mapping[str, Any],
         stream_slice: Optional[StreamSlice] = None,
     ) -> Iterable[StreamData]:
+        # emit the slice_descriptor log message, for connector builder TestRead
+        yield self.slice_logger.create_slice_log_message(stream_slice.cursor_slice)  # type: ignore
+
         stream_state: StreamState = self._get_stream_state()
         jobs: Iterable[AsyncJob] = self._validate_and_get_stream_slice_jobs(stream_slice)
         records: Iterable[Mapping[str, Any]] = self.stream_slicer.fetch_records(jobs)

{airbyte_cdk-6.37.0.dev0.dist-info → airbyte_cdk-6.37.2.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.37.0.dev0
+Version: 6.37.2.dev1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.37.0.dev0.dist-info → airbyte_cdk-6.37.2.dev1.dist-info}/RECORD

@@ -26,7 +26,7 @@ airbyte_cdk/destinations/vector_db_based/indexer.py,sha256=beiSi2Uu67EoTr7yQSaCJ
 airbyte_cdk/destinations/vector_db_based/test_utils.py,sha256=MkqLiOJ5QyKbV4rNiJhe-BHM7FD-ADHQ4bQGf4c5lRY,1932
 airbyte_cdk/destinations/vector_db_based/utils.py,sha256=FOyEo8Lc-fY8UyhpCivhZtIqBRyxf3cUt6anmK03fUY,1127
 airbyte_cdk/destinations/vector_db_based/writer.py,sha256=nZ00xPiohElJmYktEZZIhr0m5EDETCHGhg0Lb2S7A20,5095
-airbyte_cdk/entrypoint.py,sha256=2b8lsoJkMIVtQE0vKkpsjHYnIUJm9G6HDuEN_lc4SP0,18569
+airbyte_cdk/entrypoint.py,sha256=xFLY2PV8mKXUaeBAknczbK6plrs4_B1WdWA6K3iaRJI,18555
 airbyte_cdk/exception_handler.py,sha256=D_doVl3Dt60ASXlJsfviOCswxGyKF2q0RL6rif3fNks,2013
 airbyte_cdk/logger.py,sha256=qi4UGuSYQQGaFaTVJlMD9lLppwqLXt1XBhwSXo-Q5IA,3660
 airbyte_cdk/models/__init__.py,sha256=MOTiuML2wShBaMSIwikdjyye2uUWBjo4J1QFSbnoiM4,2075
@@ -75,7 +75,7 @@ airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=5o5Gslt
 airbyte_cdk/sources/declarative/declarative_source.py,sha256=nF7wBqFd3AQmEKAm4CnIo29CJoQL562cJGSCeL8U8bA,1531
 airbyte_cdk/sources/declarative/declarative_stream.py,sha256=venZjfpvtqr3oFSuvMBWtn4h9ayLhD4L65ACuXCDZ64,10445
 airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
-airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=DJbWaaJ5LHCBpyWz-4bEw8rqtJYqabEYZtxnfRtWFE0,4946
+airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py,sha256=dQtXYK2ngM8JnVsQi_UXQugWYjotMrzRJfc6kNz7Sx4,5003
 airbyte_cdk/sources/declarative/decoders/decoder.py,sha256=sl-Gt8lXi7yD2Q-sD8je5QS2PbgrgsYjxRLWsay7DMc,826
 airbyte_cdk/sources/declarative/decoders/json_decoder.py,sha256=BdWpXXPhEGf_zknggJmhojLosmxuw51RBVTS0jvdCPc,2080
 airbyte_cdk/sources/declarative/decoders/noop_decoder.py,sha256=iZh0yKY_JzgBnJWiubEusf5c0o6Khd-8EWFWT-8EgFo,542
@@ -89,7 +89,7 @@ airbyte_cdk/sources/declarative/extractors/http_selector.py,sha256=2zWZ4ewTqQC8V
 airbyte_cdk/sources/declarative/extractors/record_extractor.py,sha256=XJELMjahAsaomlvQgN2zrNO0DJX0G0fr9r682gUz7Pg,691
 airbyte_cdk/sources/declarative/extractors/record_filter.py,sha256=yTdEkyDUSW2KbFkEwJJMlS963C955LgCCOVfTmmScpQ,3367
 airbyte_cdk/sources/declarative/extractors/record_selector.py,sha256=HCqx7IyENM_aRF4it2zJN26_vDu6WeP8XgCxQWHUvcY,6934
-airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=LhqGDfX06_dDYLKsIVnwQ_nAWCln-v8PV7Wgt_QVeTI,6533
+airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py,sha256=L6hQV7bdwEp8y-TBMeQY-xmrNyRggL14lKXdWnzYFfA,6622
 airbyte_cdk/sources/declarative/extractors/type_transformer.py,sha256=d6Y2Rfg8pMVEEnHllfVksWZdNVOU55yk34O03dP9muY,1626
 airbyte_cdk/sources/declarative/incremental/__init__.py,sha256=U1oZKtBaEC6IACmvziY9Wzg7Z8EgF4ZuR7NwvjlB_Sk,1255
 airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py,sha256=MT5JbdEbnPzk3VWZGGvThe4opoX5dHhSXFrnTRYC6dg,22210
@@ -142,7 +142,7 @@ airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.
 airbyte_cdk/sources/declarative/requesters/error_handlers/default_http_response_filter.py,sha256=q0YkeYUUWO6iErUy0vjqiOkhg8_9d5YcCmtlpXAJJ9E,1314
 airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py,sha256=Tan66odx8VHzfdyyXMQkXz2pJYksllGqvxmpoajgcK4,669
 airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py,sha256=E-fQbt4ShfxZVoqfnmOx69C6FUPWZz8BIqI3DN9Kcjs,7935
-airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=3GtOefPH08evlSUxaILkiKLTHbIspFY4qd5B3ZqNE60,10063
+airbyte_cdk/sources/declarative/requesters/http_job_repository.py,sha256=Wjh5ARW1kcuRfihfFEloQ_l1N6yY1NN_4qFZJiqlH4o,11612
 airbyte_cdk/sources/declarative/requesters/http_requester.py,sha256=pR2uR5b9eGyvYIOYwus3mz3OaqRu1ozwja_ys1SE7hc,14952
 airbyte_cdk/sources/declarative/requesters/paginators/__init__.py,sha256=uArbKs9JKNCt7t9tZoeWwjDpyI1HoPp29FNW0JzvaEM,644
 airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py,sha256=ZW4lwWNAzb4zL0jKc-HjowP5-y0Zg9xi0YlK6tkx_XY,12057
@@ -169,7 +169,7 @@ airbyte_cdk/sources/declarative/resolvers/components_resolver.py,sha256=KPjKc0yb
 airbyte_cdk/sources/declarative/resolvers/config_components_resolver.py,sha256=dz4iJV9liD_LzY_Mn4XmAStoUll60R3MIGWV4aN3pgg,5223
 airbyte_cdk/sources/declarative/resolvers/http_components_resolver.py,sha256=AiojNs8wItJFrENZBFUaDvau3sgwudO6Wkra36upSPo,4639
 airbyte_cdk/sources/declarative/retrievers/__init__.py,sha256=ix9m1dkR69DcXCXUKC5RK_ZZM7ojTLBQ4IkWQTfmfCk,456
-airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=2oQn_vo7uJKp4pdMnsF5CG5Iwc9rkPeEOLoAm_9bcus,3222
+airbyte_cdk/sources/declarative/retrievers/async_retriever.py,sha256=dwYZ70eg9DKHEqZydHhMFPkEILbNcXu7E-djOCikNgI,3530
 airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc37XzUAYmzlXd1a7eSsspM-CMuWA,1696
 airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=bOAKQLgMv1Vca-ozMPRVAg1V5nkyUoPwqC02lKpnLiM,24575
 airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
@@ -360,9 +360,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.37.0.dev0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.37.0.dev0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.37.0.dev0.dist-info/METADATA,sha256=kcKo6BoaqSRqsOMLq2fmnSa6iVkhtMy4YMEL4Dk1WYU,6015
-airbyte_cdk-6.37.0.dev0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.37.0.dev0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
-airbyte_cdk-6.37.0.dev0.dist-info/RECORD,,
+airbyte_cdk-6.37.2.dev1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.37.2.dev1.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.37.2.dev1.dist-info/METADATA,sha256=-nhNtzFqhzx4jLmK3XuDXywoS33ZGSuy9PM0PsRC97g,6015
+airbyte_cdk-6.37.2.dev1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.37.2.dev1.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
+airbyte_cdk-6.37.2.dev1.dist-info/RECORD,,