airbyte-cdk 6.36.3.dev0__tar.gz → 6.36.5.dev0__tar.gz

{airbyte_cdk-6.36.3.dev0 → airbyte_cdk-6.36.5.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.36.3.dev0
+Version: 6.36.5.dev0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.36.3.dev0 → airbyte_cdk-6.36.5.dev0}/airbyte_cdk/sources/declarative/datetime/datetime_parser.py

@@ -31,7 +31,8 @@ class DatetimeParser:
             return datetime.datetime.fromtimestamp(float(date), tz=datetime.timezone.utc)
         elif format == "%ms":
             return self._UNIX_EPOCH + datetime.timedelta(milliseconds=int(date))
-
+        elif "%_ms" in format:
+            format = format.replace("%_ms", "%f")
         parsed_datetime = datetime.datetime.strptime(str(date), format)
         if self._is_naive(parsed_datetime):
             return parsed_datetime.replace(tzinfo=datetime.timezone.utc)
@@ -48,6 +49,11 @@ class DatetimeParser:
         if format == "%ms":
             # timstamp() returns a float representing the number of seconds since the unix epoch
             return str(int(dt.timestamp() * 1000))
+        if "%_ms" in format:
+            _format = format.replace("%_ms", "%f")
+            milliseconds = int(dt.microsecond / 1000)
+            formatted_dt = dt.strftime(_format).replace(dt.strftime("%f"), "%03d" % milliseconds)
+            return formatted_dt
         else:
             return dt.strftime(format)
 

{airbyte_cdk-6.36.3.dev0 → airbyte_cdk-6.36.5.dev0}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -844,6 +844,7 @@ definitions:
             * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`
             * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`
             * **%f**: Microsecond (zero-padded to 6 digits) - `000000`
+            * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`
             * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`
             * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`
             * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`
@@ -1779,6 +1780,9 @@ definitions:
           - stream_interval
           - stream_partition
           - stream_slice
+          - creation_response
+          - polling_response
+          - download_target
         examples:
           - "/products"
           - "/quotes/{{ stream_partition['id'] }}/quote_line_groups"
@@ -2398,6 +2402,7 @@ definitions:
             * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`
             * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`
             * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`
+            * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`
             * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`
             * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`
             * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`
@@ -3223,7 +3228,7 @@ definitions:
       - polling_requester
       - download_requester
       - status_extractor
-      - urls_extractor
+      - download_target_extractor
     properties:
       type:
         type: string
@@ -3240,7 +3245,7 @@ definitions:
         anyOf:
           - "$ref": "#/definitions/CustomRecordExtractor"
           - "$ref": "#/definitions/DpathExtractor"
-      urls_extractor:
+      download_target_extractor:
         description: Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.
         anyOf:
           - "$ref": "#/definitions/CustomRecordExtractor"
@@ -3261,7 +3266,7 @@ definitions:
         anyOf:
           - "$ref": "#/definitions/CustomRequester"
           - "$ref": "#/definitions/HttpRequester"
-      url_requester:
+      download_target_requester:
         description: Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.
         anyOf:
           - "$ref": "#/definitions/CustomRequester"
@@ -3667,6 +3672,21 @@ interpolation:
             self: https://api.sendgrid.com/v3/marketing/lists?page_size=1&page_token=
             next: https://api.sendgrid.com/v3/marketing/lists?page_size=1&page_token=0236d6d2
             count: 82
+    - title: creation_response
+      description: The response received from the creation_requester in the AsyncRetriever component.
+      type: object
+      examples:
+        - id: "1234"
+    - title: polling_response
+      description: The response received from the polling_requester in the AsyncRetriever component.
+      type: object
+      examples:
+        - id: "1234"
+    - title: download_target
+      description: The `URL` received from the polling_requester in the AsyncRetriever with jobStatus as `COMPLETED`.
+      type: string
+      examples:
+        - "https://api.sendgrid.com/v3/marketing/lists?page_size=1&page_token=0236d6d2&filename=xxx_yyy_zzz.csv"
     - title: stream_interval
       description: The current stream interval being processed. The keys are defined by the incremental sync component. Default keys are `start_time` and `end_time`.
       type: object

{airbyte_cdk-6.36.3.dev0 → airbyte_cdk-6.36.5.dev0}/airbyte_cdk/sources/declarative/decoders/composite_raw_decoder.py

@@ -5,7 +5,7 @@ import json
 import logging
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from io import BufferedIOBase, BytesIO, TextIOWrapper
+from io import BufferedIOBase, TextIOWrapper
 from typing import Any, Generator, MutableMapping, Optional
 
 import orjson
@@ -124,8 +124,7 @@ class CsvParser(Parser):
         """
         Parse CSV data from decompressed bytes.
         """
-        bytes_data = BytesIO(data.read())
-        text_data = TextIOWrapper(bytes_data, encoding=self.encoding)  # type: ignore
+        text_data = TextIOWrapper(data, encoding=self.encoding)  # type: ignore
         reader = csv.DictReader(text_data, delimiter=self._get_delimiter() or ",")
         for row in reader:
             yield row
@@ -152,8 +151,6 @@ class CompositeRawDecoder(Decoder):
         self, response: requests.Response
     ) -> Generator[MutableMapping[str, Any], None, None]:
         if self.is_stream_response():
-            response.raw.auto_close = False
             yield from self.parser.parse(data=response.raw)  # type: ignore[arg-type]
-            response.raw.close()
         else:
             yield from self.parser.parse(data=io.BytesIO(response.content))

{airbyte_cdk-6.36.3.dev0 → airbyte_cdk-6.36.5.dev0}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -2263,7 +2263,7 @@ class AsyncRetriever(BaseModel):
     status_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
         ..., description="Responsible for fetching the actual status of the async job."
     )
-    urls_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
+    download_target_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
         ...,
         description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.",
     )
@@ -2278,7 +2278,7 @@ class AsyncRetriever(BaseModel):
         ...,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to fetch the status of the running async job.",
     )
-    url_requester: Optional[Union[CustomRequester, HttpRequester]] = Field(
+    download_target_requester: Optional[Union[CustomRequester, HttpRequester]] = Field(
         None,
         description="Requester component that describes how to prepare HTTP requests to send to the source API to extract the url from polling response by the completed async job.",
     )

{airbyte_cdk-6.36.3.dev0 → airbyte_cdk-6.36.5.dev0}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -1405,7 +1405,19 @@ class ModelToComponentFactory:
         try:
             module_ref = importlib.import_module(module_name_full)
         except ModuleNotFoundError as e:
-            raise ValueError(f"Could not load module `{module_name_full}`.") from e
+            if split[0] == "source_declarative_manifest":
+                # During testing, the modules containing the custom components are not moved to source_declarative_manifest. In order to run the test, add the source folder to your PYTHONPATH or add it runtime using sys.path.append
+                try:
+                    import os
+
+                    module_name_with_source_declarative_manifest = ".".join(split[1:-1])
+                    module_ref = importlib.import_module(
+                        module_name_with_source_declarative_manifest
+                    )
+                except ModuleNotFoundError:
+                    raise ValueError(f"Could not load module `{module_name_full}`.") from e
+            else:
+                raise ValueError(f"Could not load module `{module_name_full}`.") from e
 
         try:
             return getattr(module_ref, class_name)
@@ -2744,32 +2756,32 @@ class ModelToComponentFactory:
             if model.delete_requester
             else None
         )
-        url_requester = (
+        download_target_requester = (
             self._create_component_from_model(
-                model=model.url_requester,
+                model=model.download_target_requester,
                 decoder=decoder,
                 config=config,
                 name=f"job extract_url - {name}",
             )
-            if model.url_requester
+            if model.download_target_requester
             else None
         )
         status_extractor = self._create_component_from_model(
             model=model.status_extractor, decoder=decoder, config=config, name=name
         )
-        urls_extractor = self._create_component_from_model(
-            model=model.urls_extractor, decoder=decoder, config=config, name=name
+        download_target_extractor = self._create_component_from_model(
+            model=model.download_target_extractor, decoder=decoder, config=config, name=name
         )
         job_repository: AsyncJobRepository = AsyncHttpJobRepository(
             creation_requester=creation_requester,
             polling_requester=polling_requester,
             download_retriever=download_retriever,
-            url_requester=url_requester,
+            download_target_requester=download_target_requester,
             abort_requester=abort_requester,
             delete_requester=delete_requester,
             status_extractor=status_extractor,
             status_mapping=self._create_async_job_status_mapping(model.status_mapping, config),
-            urls_extractor=urls_extractor,
+            download_target_extractor=download_target_extractor,
         )
 
         async_job_partition_router = AsyncJobPartitionRouter(

{airbyte_cdk-6.36.3.dev0 → airbyte_cdk-6.36.5.dev0}/airbyte_cdk/sources/declarative/requesters/README.md

@@ -1,8 +1,8 @@
 # AsyncHttpJobRepository sequence diagram
 
 - Components marked as optional are not required and can be ignored.
-- if `url_requester` is not provided, `urls_extractor` will get urls from the `polling_job_response`
-- interpolation_context, e.g. `create_job_response` or `polling_job_response` can be obtained from stream_slice
+- if `download_target_requester` is not provided, `download_target_extractor` will get urls from the `polling_response`
+- interpolation_context, e.g. `creation_response` or `polling_response` can be obtained from stream_slice
 
 ```mermaid
 ---
@@ -12,7 +12,7 @@ sequenceDiagram
     participant AsyncHttpJobRepository as AsyncOrchestrator
     participant CreationRequester as creation_requester
     participant PollingRequester as polling_requester
-    participant UrlRequester as url_requester (Optional)
+    participant UrlRequester as download_target_requester (Optional)
     participant DownloadRetriever as download_retriever
     participant AbortRequester as abort_requester (Optional)
     participant DeleteRequester as delete_requester (Optional)
@@ -25,14 +25,14 @@ sequenceDiagram
 
     loop Poll for job status
         AsyncHttpJobRepository ->> PollingRequester: Check job status
-        PollingRequester ->> Reporting Server: Status request (interpolation_context: `create_job_response`)
+        PollingRequester ->> Reporting Server: Status request (interpolation_context: `creation_response`)
         Reporting Server -->> PollingRequester: Status response
         PollingRequester -->> AsyncHttpJobRepository: Job status
     end
 
     alt Status: Ready
         AsyncHttpJobRepository ->> UrlRequester: Request download URLs (if applicable)
-        UrlRequester ->> Reporting Server: URL request (interpolation_context: `polling_job_response`)
+        UrlRequester ->> Reporting Server: URL request (interpolation_context: `polling_response`)
         Reporting Server -->> UrlRequester: Download URLs
         UrlRequester -->> AsyncHttpJobRepository: Download URLs
 

{airbyte_cdk-6.36.3.dev0 → airbyte_cdk-6.36.5.dev0}/airbyte_cdk/sources/declarative/requesters/http_job_repository.py

@@ -43,13 +43,13 @@ class AsyncHttpJobRepository(AsyncJobRepository):
     delete_requester: Optional[Requester]
     status_extractor: DpathExtractor
     status_mapping: Mapping[str, AsyncJobStatus]
-    urls_extractor: DpathExtractor
+    download_target_extractor: DpathExtractor
 
     job_timeout: Optional[timedelta] = None
     record_extractor: RecordExtractor = field(
         init=False, repr=False, default_factory=lambda: ResponseToFileExtractor({})
     )
-    url_requester: Optional[Requester] = (
+    download_target_requester: Optional[Requester] = (
         None  # use it in case polling_requester provides some <id> and extra request is needed to obtain list of urls to download from
     )
 
@@ -211,12 +211,15 @@ class AsyncHttpJobRepository(AsyncJobRepository):
 
         """
 
-        for url in self._get_download_url(job):
+        for target_url in self._get_download_targets(job):
             job_slice = job.job_parameters()
             stream_slice = StreamSlice(
                 partition=job_slice.partition,
                 cursor_slice=job_slice.cursor_slice,
-                extra_fields={**job_slice.extra_fields, "url": url},
+                extra_fields={
+                    **job_slice.extra_fields,
+                    "download_target": target_url,
+                },
             )
             for message in self.download_retriever.read_records({}, stream_slice):
                 if isinstance(message, Record):
@@ -269,27 +272,29 @@ class AsyncHttpJobRepository(AsyncJobRepository):
         del self._polling_job_response_by_id[job_id]
 
     def _get_create_job_stream_slice(self, job: AsyncJob) -> StreamSlice:
+        creation_response = self._create_job_response_by_id[job.api_job_id()].json()
         stream_slice = StreamSlice(
-            partition={"create_job_response": self._create_job_response_by_id[job.api_job_id()]},
+            partition={},
             cursor_slice={},
+            extra_fields={"creation_response": creation_response},
         )
         return stream_slice
 
-    def _get_download_url(self, job: AsyncJob) -> Iterable[str]:
-        if not self.url_requester:
+    def _get_download_targets(self, job: AsyncJob) -> Iterable[str]:
+        if not self.download_target_requester:
             url_response = self._polling_job_response_by_id[job.api_job_id()]
         else:
+            polling_response = self._polling_job_response_by_id[job.api_job_id()].json()
             stream_slice: StreamSlice = StreamSlice(
-                partition={
-                    "polling_job_response": self._polling_job_response_by_id[job.api_job_id()]
-                },
+                partition={},
                 cursor_slice={},
+                extra_fields={"polling_response": polling_response},
             )
-            url_response = self.url_requester.send_request(stream_slice=stream_slice)  # type: ignore # we expect url_requester to always be presented, otherwise raise an exception as we cannot proceed with the report
+            url_response = self.download_target_requester.send_request(stream_slice=stream_slice)  # type: ignore # we expect download_target_requester to always be presented, otherwise raise an exception as we cannot proceed with the report
             if not url_response:
                 raise AirbyteTracedException(
-                    internal_message="Always expect a response or an exception from url_requester",
+                    internal_message="Always expect a response or an exception from download_target_requester",
                     failure_type=FailureType.system_error,
                 )
 
-        yield from self.urls_extractor.extract_records(url_response)  # type: ignore # we expect urls_extractor to always return list of strings
+        yield from self.download_target_extractor.extract_records(url_response)  # type: ignore # we expect download_target_extractor to always return list of strings

{airbyte_cdk-6.36.3.dev0 → airbyte_cdk-6.36.5.dev0}/airbyte_cdk/sources/declarative/requesters/http_requester.py

@@ -85,7 +85,7 @@ class HttpRequester(Requester):
         self._parameters = parameters
 
         if self.error_handler is not None and hasattr(self.error_handler, "backoff_strategies"):
-            backoff_strategies = self.error_handler.backoff_strategies
+            backoff_strategies = self.error_handler.backoff_strategies  # type: ignore
         else:
             backoff_strategies = None
 
@@ -125,6 +125,12 @@ class HttpRequester(Requester):
         kwargs = {
             "stream_slice": stream_slice,
             "next_page_token": next_page_token,
+            # update the interpolation context with extra fields, if passed.
+            **(
+                stream_slice.extra_fields
+                if stream_slice is not None and hasattr(stream_slice, "extra_fields")
+                else {}
+            ),
         }
         path = str(self._path.eval(self.config, **kwargs))
         return path.lstrip("/")

{airbyte_cdk-6.36.3.dev0 → airbyte_cdk-6.36.5.dev0}/airbyte_cdk/sources/declarative/yaml_declarative_source.py

@@ -39,13 +39,18 @@ class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage
         )
 
     def _read_and_parse_yaml_file(self, path_to_yaml_file: str) -> ConnectionDefinition:
-        package = self.__class__.__module__.split(".")[0]
+        try:
+            # For testing purposes, we want to allow to just pass a file. However, this
+            with open(path_to_yaml_file, "r") as f:
+                return yaml.safe_load(f)  # type: ignore  # we assume the yaml represents a ConnectionDefinition
+        except FileNotFoundError:
+            # Running inside the container, the working directory during an operation is not structured the same as the static files
+            package = self.__class__.__module__.split(".")[0]
 
-        yaml_config = pkgutil.get_data(package, path_to_yaml_file)
-        if yaml_config:
-            decoded_yaml = yaml_config.decode()
-            return self._parse(decoded_yaml)
-        else:
+            yaml_config = pkgutil.get_data(package, path_to_yaml_file)
+            if yaml_config:
+                decoded_yaml = yaml_config.decode()
+                return self._parse(decoded_yaml)
             return {}
 
     def _emit_manifest_debug_message(self, extra_args: dict[str, Any]) -> None:

{airbyte_cdk-6.36.3.dev0 → airbyte_cdk-6.36.5.dev0}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.36.3dev0"  # Version will be calculated dynamically.
+version = "6.36.5dev"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false