airbyte-cdk 6.48.6__py3-none-any.whl → 6.48.7.dev0__py3-none-any.whl

airbyte_cdk/destinations/destination.py

@@ -18,6 +18,7 @@ from airbyte_cdk.models import (
     AirbyteMessageSerializer,
     ConfiguredAirbyteCatalog,
     ConfiguredAirbyteCatalogSerializer,
+    DestinationCatalog,
     Type,
 )
 from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit
@@ -26,8 +27,74 @@ from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 logger = logging.getLogger("airbyte")
 
 
+def parse_args(args: List[str]) -> argparse.Namespace:
+    """
+    :param args: commandline arguments
+    :return:
+    """
+
+    parent_parser = argparse.ArgumentParser(add_help=False)
+    parent_parser.add_argument(
+        "--debug", action="store_true", help="enables detailed debug logs related to the sync"
+    )
+    main_parser = argparse.ArgumentParser()
+    subparsers = main_parser.add_subparsers(title="commands", dest="command")
+
+    # spec
+    subparsers.add_parser(
+        "spec", help="outputs the json configuration specification", parents=[parent_parser]
+    )
+
+    # check
+    check_parser = subparsers.add_parser(
+        "check", help="checks the config can be used to connect", parents=[parent_parser]
+    )
+    required_check_parser = check_parser.add_argument_group("required named arguments")
+    required_check_parser.add_argument(
+        "--config", type=str, required=True, help="path to the json configuration file"
+    )
+
+    # discover
+    discover_parser = subparsers.add_parser(
+        "discover",
+        help="discover the objects available in the destination",
+        parents=[parent_parser],
+    )
+    required_discover_parser = discover_parser.add_argument_group("required named arguments")
+    required_discover_parser.add_argument(
+        "--config", type=str, required=True, help="path to the json configuration file"
+    )
+
+    # write
+    write_parser = subparsers.add_parser(
+        "write", help="Writes data to the destination", parents=[parent_parser]
+    )
+    write_required = write_parser.add_argument_group("required named arguments")
+    write_required.add_argument(
+        "--config", type=str, required=True, help="path to the JSON configuration file"
+    )
+    write_required.add_argument(
+        "--catalog", type=str, required=True, help="path to the configured catalog JSON file"
+    )
+
+    parsed_args = main_parser.parse_args(args)
+    cmd = parsed_args.command
+    if not cmd:
+        raise Exception("No command entered. ")
+    elif cmd not in ["spec", "check", "discover", "write"]:
+        # This is technically dead code since parse_args() would fail if this was the case
+        # But it's non-obvious enough to warrant placing it here anyways
+        raise Exception(f"Unknown command entered: {cmd}")
+
+    return parsed_args
+
+
 class Destination(Connector, ABC):
-    VALID_CMDS = {"spec", "check", "write"}
+    VALID_CMDS = {"spec", "check", "discover", "write"}
+
+    def discover(self) -> DestinationCatalog:
+        """Implement to define what objects are available in the destination"""
+        raise NotImplementedError("Discover method is not implemented")
 
     @abstractmethod
     def write(
@@ -68,52 +135,9 @@ class Destination(Connector, ABC):
         )
         logger.info("Writing complete.")
 
-    def parse_args(self, args: List[str]) -> argparse.Namespace:
-        """
-        :param args: commandline arguments
-        :return:
-        """
-
-        parent_parser = argparse.ArgumentParser(add_help=False)
-        main_parser = argparse.ArgumentParser()
-        subparsers = main_parser.add_subparsers(title="commands", dest="command")
-
-        # spec
-        subparsers.add_parser(
-            "spec", help="outputs the json configuration specification", parents=[parent_parser]
-        )
-
-        # check
-        check_parser = subparsers.add_parser(
-            "check", help="checks the config can be used to connect", parents=[parent_parser]
-        )
-        required_check_parser = check_parser.add_argument_group("required named arguments")
-        required_check_parser.add_argument(
-            "--config", type=str, required=True, help="path to the json configuration file"
-        )
-
-        # write
-        write_parser = subparsers.add_parser(
-            "write", help="Writes data to the destination", parents=[parent_parser]
-        )
-        write_required = write_parser.add_argument_group("required named arguments")
-        write_required.add_argument(
-            "--config", type=str, required=True, help="path to the JSON configuration file"
-        )
-        write_required.add_argument(
-            "--catalog", type=str, required=True, help="path to the configured catalog JSON file"
-        )
-
-        parsed_args = main_parser.parse_args(args)
-        cmd = parsed_args.command
-        if not cmd:
-            raise Exception("No command entered. ")
-        elif cmd not in ["spec", "check", "write"]:
-            # This is technically dead code since parse_args() would fail if this was the case
-            # But it's non-obvious enough to warrant placing it here anyways
-            raise Exception(f"Unknown command entered: {cmd}")
-
-        return parsed_args
+    @staticmethod
+    def parse_args(args: List[str]) -> argparse.Namespace:
+        return parse_args(args)
 
     def run_cmd(self, parsed_args: argparse.Namespace) -> Iterable[AirbyteMessage]:
         cmd = parsed_args.command
@@ -137,6 +161,8 @@ class Destination(Connector, ABC):
 
         if cmd == "check":
             yield self._run_check(config=config)
+        elif cmd == "discover":
+            yield AirbyteMessage(type=Type.DESTINATION_CATALOG, destination_catalog=self.discover())
         elif cmd == "write":
             # Wrap in UTF-8 to override any other input encodings
             wrapped_stdin = io.TextIOWrapper(sys.stdin.buffer, encoding="utf-8")

airbyte_cdk/models/__init__.py

@@ -35,6 +35,10 @@ from .airbyte_protocol import (
     ConfiguredAirbyteCatalog,
     ConfiguredAirbyteStream,
     ConnectorSpecification,
+    DestinationCatalog,
+    DestinationObject,
+    DestinationObjectProperty,
+    DestinationOperation,
     DestinationSyncMode,
     EstimateType,
     FailureType,

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -299,8 +299,8 @@ definitions:
             - "$ref": "#/definitions/CustomAuthenticator"
             - "$ref": "#/definitions/OAuthAuthenticator"
             - "$ref": "#/definitions/JwtAuthenticator"
-            - "$ref": "#/definitions/NoAuth"
             - "$ref": "#/definitions/SessionTokenAuthenticator"
+            - "$ref": "#/definitions/NoAuth"
             - "$ref": "#/definitions/LegacySessionTokenAuthenticator"
         examples:
           - authenticators:
@@ -1417,7 +1417,6 @@ definitions:
           - "$ref": "#/definitions/CustomIncrementalSync"
       primary_key:
         title: Primary Key
-        description: The primary key of the stream.
         "$ref": "#/definitions/PrimaryKey"
         default: ""
       schema_loader:
@@ -1468,18 +1467,18 @@ definitions:
           requester:
             description: Requester component that describes how to prepare HTTP requests to send to the source API.
             anyOf:
-              - "$ref": "#/definitions/CustomRequester"
               - "$ref": "#/definitions/HttpRequester"
+              - "$ref": "#/definitions/CustomRequester"
           download_target_extractor:
             description: Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response
             anyOf:
-              - "$ref": "#/definitions/CustomRecordExtractor"
               - "$ref": "#/definitions/DpathExtractor"
+              - "$ref": "#/definitions/CustomRecordExtractor"
           file_extractor:
             description: Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content
             anyOf:
-              - "$ref": "#/definitions/CustomRecordExtractor"
               - "$ref": "#/definitions/DpathExtractor"
+              - "$ref": "#/definitions/CustomRecordExtractor"
           filename_extractor:
             description: Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided.
             type: string
@@ -1919,7 +1918,7 @@ definitions:
         deprecated: true
         deprecation_message: "Use `url` field instead."
         title: API Base URL
-        description: Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
+        description: Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.
         linkable: true
         type: string
         interpolation_context:
@@ -1937,8 +1936,8 @@ definitions:
           - "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups"
           - "https://example.com/api/v1/resource/{{ next_page_token['id'] }}"
       url:
-        title: The URL of an API endpoint
-        description: The URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
+        title: API Endpoint URL
+        description: The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.
         type: string
         interpolation_context:
           - config
@@ -1958,7 +1957,7 @@ definitions:
         deprecated: true
         deprecation_message: "Use `url` field instead."
         title: URL Path
-        description: Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
+        description: Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.
         type: string
         interpolation_context:
           - config
@@ -1989,7 +1988,6 @@ definitions:
         linkable: true
         description: Authentication method to use for requests sent to the API.
         anyOf:
-          - "$ref": "#/definitions/NoAuth"
           - "$ref": "#/definitions/ApiKeyAuthenticator"
           - "$ref": "#/definitions/BasicHttpAuthenticator"
           - "$ref": "#/definitions/BearerAuthenticator"
@@ -1998,11 +1996,52 @@ definitions:
           - "$ref": "#/definitions/SessionTokenAuthenticator"
           - "$ref": "#/definitions/SelectiveAuthenticator"
           - "$ref": "#/definitions/CustomAuthenticator"
+          - "$ref": "#/definitions/NoAuth"
           - "$ref": "#/definitions/LegacySessionTokenAuthenticator"
       fetch_properties_from_endpoint:
         title: Fetch Properties from Endpoint
         description: Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.
         "$ref": "#/definitions/PropertiesFromEndpoint"
+      request_parameters:
+        title: Query Parameters
+        description: Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.
+        anyOf:
+          - type: object
+            title: Key/Value Pairs
+            additionalProperties:
+              anyOf:
+                - type: string
+                - $ref: "#/definitions/QueryProperties"
+          - type: string
+            title: Interpolated Value
+        interpolation_context:
+          - next_page_token
+          - stream_interval
+          - stream_partition
+          - stream_slice
+        examples:
+          - unit: "day"
+          - query: 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
+          - searchIn: "{{ ','.join(config.get('search_in', [])) }}"
+          - sort_by[asc]: updated_at
+      request_headers:
+        title: Request Headers
+        description: Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.
+        anyOf:
+          - type: object
+            title: Key/Value Pairs
+            additionalProperties:
+              type: string
+          - type: string
+            title: Interpolated Value
+        interpolation_context:
+          - next_page_token
+          - stream_interval
+          - stream_partition
+          - stream_slice
+        examples:
+          - Output-Format: JSON
+          - Version: "{{ config['version'] }}"
       request_body_data:
         deprecated: true
         deprecation_message: "Use `request_body` field instead."
@@ -2045,7 +2084,7 @@ definitions:
               field: "updated_at"
               order: "ascending"
       request_body:
-        title: Request Body Payload to be send as a part of the API request.
+        title: Request Body
         description: Specifies how to populate the body of the request with a payload. Can contain nested objects.
         anyOf:
           - "$ref": "#/definitions/RequestBodyPlainText"
@@ -2081,46 +2120,6 @@ definitions:
               query:
                 param1: "value1"
                 param2: "{{ config['param2_value'] }}"
-      request_headers:
-        title: Request Headers
-        description: Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.
-        anyOf:
-          - type: object
-            title: Key/Value Pairs
-            additionalProperties:
-              type: string
-          - type: string
-            title: Interpolated Value
-        interpolation_context:
-          - next_page_token
-          - stream_interval
-          - stream_partition
-          - stream_slice
-        examples:
-          - Output-Format: JSON
-          - Version: "{{ config['version'] }}"
-      request_parameters:
-        title: Query Parameters
-        description: Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.
-        anyOf:
-          - type: object
-            title: Key/Value Pairs
-            additionalProperties:
-              anyOf:
-                - type: string
-                - $ref: "#/definitions/QueryProperties"
-          - type: string
-            title: Interpolated Value
-        interpolation_context:
-          - next_page_token
-          - stream_interval
-          - stream_partition
-          - stream_slice
-        examples:
-          - unit: "day"
-          - query: 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
-          - searchIn: "{{ ','.join(config.get('search_in', [])) }}"
-          - sort_by[asc]: updated_at
       error_handler:
         title: Error Handler
         description: Error handler component that defines how to handle errors.
@@ -3100,7 +3099,7 @@ definitions:
         type: array
         default: []
         items:
-          - type: string
+          type: string
         interpolation_context:
           - config
       parent_key:
@@ -3155,14 +3154,19 @@ definitions:
     anyOf:
       - type: string
         title: Single Key
+        description: The single top-level field to use as the primary key.
       - type: array
         title: Composite Key
+        description: An array of top-level fields representing a composite primary key.
         items:
           type: string
       - type: array
         title: Composite Key of Nested Fields
+        description: An array of arrays representing a composite primary key where the fields are nested fields.
         items:
           type: array
+          title: Nested Field Path
+          description: Path to the nested field in the record.
           items:
             type: string
     default: ""
@@ -3194,8 +3198,8 @@ definitions:
       retriever:
         description: Requester component that describes how to fetch the properties to query from a remote API endpoint.
         anyOf:
-          - "$ref": "#/definitions/CustomRetriever"
           - "$ref": "#/definitions/SimpleRetriever"
+          - "$ref": "#/definitions/CustomRetriever"
       $parameters:
         type: object
         additionalProperties: true
@@ -3426,6 +3430,7 @@ definitions:
     additionalProperties: true
   LegacySessionTokenAuthenticator:
     title: Session Token Authenticator
+    deprecated: true
     description: Deprecated - use SessionTokenAuthenticator instead. Authenticator for requests authenticated using session tokens. A session token is a random value generated by a server to identify a specific user for the duration of one interaction session.
     type: object
     required:
@@ -3534,15 +3539,15 @@ definitions:
           - "$ref": "#/definitions/HttpRequester"
           - "$ref": "#/definitions/CustomRequester"
       decoder:
-        title: Decoder
+        title: HTTP Response Format
         description: Component decoding the response so records can be extracted.
         anyOf:
-          - "$ref": "#/definitions/CsvDecoder"
-          - "$ref": "#/definitions/GzipDecoder"
           - "$ref": "#/definitions/JsonDecoder"
+          - "$ref": "#/definitions/XmlDecoder"
+          - "$ref": "#/definitions/CsvDecoder"
           - "$ref": "#/definitions/JsonlDecoder"
+          - "$ref": "#/definitions/GzipDecoder"
           - "$ref": "#/definitions/IterableDecoder"
-          - "$ref": "#/definitions/XmlDecoder"
           - "$ref": "#/definitions/ZipfileDecoder"
           - "$ref": "#/definitions/CustomDecoder"
       record_selector:

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -2080,8 +2080,8 @@ class SelectiveAuthenticator(BaseModel):
             CustomAuthenticator,
             OAuthAuthenticator,
             JwtAuthenticator,
-            NoAuth,
             SessionTokenAuthenticator,
+            NoAuth,
             LegacySessionTokenAuthenticator,
         ],
     ] = Field(
@@ -2103,15 +2103,15 @@ class SelectiveAuthenticator(BaseModel):
 
 class FileUploader(BaseModel):
     type: Literal["FileUploader"]
-    requester: Union[CustomRequester, HttpRequester] = Field(
+    requester: Union[HttpRequester, CustomRequester] = Field(
         ...,
         description="Requester component that describes how to prepare HTTP requests to send to the source API.",
     )
-    download_target_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field(
+    download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field(
         ...,
         description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response",
     )
-    file_extractor: Optional[Union[CustomRecordExtractor, DpathExtractor]] = Field(
+    file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field(
         None,
         description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content",
     )
@@ -2144,9 +2144,7 @@ class DeclarativeStream(BaseModel):
         description="Component used to fetch data incrementally based on a time field in the data.",
         title="Incremental Sync",
     )
-    primary_key: Optional[PrimaryKey] = Field(
-        "", description="The primary key of the stream.", title="Primary Key"
-    )
+    primary_key: Optional[PrimaryKey] = Field("", title="Primary Key")
     schema_loader: Optional[
         Union[
             InlineSchemaLoader,
@@ -2242,7 +2240,7 @@ class HttpRequester(BaseModelWithDeprecations):
         None,
         deprecated=True,
         deprecation_message="Use `url` field instead.",
-        description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
+        description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.",
         examples=[
             "https://connect.squareup.com/v2",
             "{{ config['base_url'] or 'https://app.posthog.com'}}/api",
@@ -2253,20 +2251,20 @@ class HttpRequester(BaseModelWithDeprecations):
     )
     url: Optional[str] = Field(
         None,
-        description="The URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
+        description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.",
         examples=[
             "https://connect.squareup.com/v2",
             "{{ config['url'] or 'https://app.posthog.com'}}/api",
             "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups",
             "https://example.com/api/v1/resource/{{ next_page_token['id'] }}",
         ],
-        title="The URL of an API endpoint",
+        title="API Endpoint URL",
     )
     path: Optional[str] = Field(
         None,
         deprecated=True,
         deprecation_message="Use `url` field instead.",
-        description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
+        description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.",
         examples=[
             "/products",
             "/quotes/{{ stream_partition['id'] }}/quote_line_groups",
@@ -2282,7 +2280,6 @@ class HttpRequester(BaseModelWithDeprecations):
     )
     authenticator: Optional[
         Union[
-            NoAuth,
             ApiKeyAuthenticator,
             BasicHttpAuthenticator,
             BearerAuthenticator,
@@ -2291,6 +2288,7 @@ class HttpRequester(BaseModelWithDeprecations):
             SessionTokenAuthenticator,
             SelectiveAuthenticator,
             CustomAuthenticator,
+            NoAuth,
             LegacySessionTokenAuthenticator,
         ]
     ] = Field(
@@ -2303,6 +2301,25 @@ class HttpRequester(BaseModelWithDeprecations):
         description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.",
         title="Fetch Properties from Endpoint",
     )
+    request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field(
+        None,
+        description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.",
+        examples=[
+            {"unit": "day"},
+            {
+                "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
+            },
+            {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"},
+            {"sort_by[asc]": "updated_at"},
+        ],
+        title="Query Parameters",
+    )
+    request_headers: Optional[Union[Dict[str, str], str]] = Field(
+        None,
+        description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.",
+        examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}],
+        title="Request Headers",
+    )
     request_body_data: Optional[Union[Dict[str, str], str]] = Field(
         None,
         deprecated=True,
@@ -2363,26 +2380,7 @@ class HttpRequester(BaseModelWithDeprecations):
                 },
             },
         ],
-        title="Request Body Payload to be send as a part of the API request.",
-    )
-    request_headers: Optional[Union[Dict[str, str], str]] = Field(
-        None,
-        description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.",
-        examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}],
-        title="Request Headers",
-    )
-    request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field(
-        None,
-        description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.",
-        examples=[
-            {"unit": "day"},
-            {
-                "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"'
-            },
-            {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"},
-            {"sort_by[asc]": "updated_at"},
-        ],
-        title="Query Parameters",
+        title="Request Body",
     )
     error_handler: Optional[
         Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler]
@@ -2475,7 +2473,7 @@ class PropertiesFromEndpoint(BaseModel):
         description="Describes the path to the field that should be extracted",
         examples=[["name"]],
     )
-    retriever: Union[CustomRetriever, SimpleRetriever] = Field(
+    retriever: Union[SimpleRetriever, CustomRetriever] = Field(
         ...,
         description="Requester component that describes how to fetch the properties to query from a remote API endpoint.",
     )
@@ -2526,19 +2524,19 @@ class SimpleRetriever(BaseModel):
     )
     decoder: Optional[
         Union[
-            CsvDecoder,
-            GzipDecoder,
             JsonDecoder,
+            XmlDecoder,
+            CsvDecoder,
             JsonlDecoder,
+            GzipDecoder,
             IterableDecoder,
-            XmlDecoder,
             ZipfileDecoder,
             CustomDecoder,
         ]
     ] = Field(
         None,
         description="Component decoding the response so records can be extracted.",
-        title="Decoder",
+        title="HTTP Response Format",
     )
     record_selector: RecordSelector = Field(
         ...,

airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py

@@ -1,11 +1,10 @@
 #
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 #
-
-
+from abc import ABC, abstractmethod
 from copy import deepcopy
 from dataclasses import InitVar, dataclass, field
-from typing import Any, List, Mapping, MutableMapping, Optional, Union
+from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Union
 
 import dpath
 from typing_extensions import deprecated
@@ -16,7 +15,7 @@ from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
 from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
 from airbyte_cdk.sources.declarative.transformations import RecordTransformation
 from airbyte_cdk.sources.source import ExperimentalClassWarning
-from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
+from airbyte_cdk.sources.types import Config
 
 AIRBYTE_DATA_TYPES: Mapping[str, MutableMapping[str, Any]] = {
     "string": {"type": ["null", "string"]},
@@ -114,6 +113,38 @@ class SchemaTypeIdentifier:
         )
 
 
+@deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
+class AdditionalPropertyFieldsInferrer(ABC):
+    """
+    Infers additional fields to be added to each property. For example, if this inferrer returns {"toto": "tata"}, a property that would have looked like this:
+    ```
+        "properties": {
+            "Id": {
+                "type": ["null", "string"],
+            },
+            <...>
+        }
+    ```
+    ... will look like this:
+        ```
+        "properties": {
+            "Id": {
+                "type": ["null", "string"],
+                "toto": "tata"
+            },
+            <...>
+        }
+    ```
+    """
+
+    @abstractmethod
+    def infer(self, property_definition: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
+        """
+        Infers additional property fields from the given property definition.
+        """
+        pass
+
+
 @deprecated("This class is experimental. Use at your own risk.", category=ExperimentalClassWarning)
 @dataclass
 class DynamicSchemaLoader(SchemaLoader):
@@ -126,6 +157,8 @@ class DynamicSchemaLoader(SchemaLoader):
     parameters: InitVar[Mapping[str, Any]]
     schema_type_identifier: SchemaTypeIdentifier
     schema_transformations: List[RecordTransformation] = field(default_factory=lambda: [])
+    additional_property_fields_inferrer: Optional[AdditionalPropertyFieldsInferrer] = None
+    allow_additional_properties: bool = True
 
     def get_json_schema(self) -> Mapping[str, Any]:
         """
@@ -149,22 +182,26 @@ class DynamicSchemaLoader(SchemaLoader):
                 property_definition,
                 self.schema_type_identifier.type_pointer,
             )
+
+            value.update(
+                self.additional_property_fields_inferrer.infer(property_definition)
+                if self.additional_property_fields_inferrer
+                else {}
+            )
             properties[key] = value
 
-        transformed_properties = self._transform(properties, {})
+        transformed_properties = self._transform(properties)
 
         return {
             "$schema": "https://json-schema.org/draft-07/schema#",
             "type": "object",
-            "additionalProperties": True,
+            "additionalProperties": self.allow_additional_properties,
             "properties": transformed_properties,
         }
 
     def _transform(
         self,
         properties: Mapping[str, Any],
-        stream_state: StreamState,
-        stream_slice: Optional[StreamSlice] = None,
     ) -> Mapping[str, Any]:
         for transformation in self.schema_transformations:
             transformation.transform(
@@ -190,7 +227,7 @@ class DynamicSchemaLoader(SchemaLoader):
         self,
         raw_schema: MutableMapping[str, Any],
         field_type_path: Optional[List[Union[InterpolatedString, str]]],
-    ) -> Union[Mapping[str, Any], List[Mapping[str, Any]]]:
+    ) -> Dict[str, Any]:
         """
         Determines the JSON Schema type for a field, supporting nullable and combined types.
         """
@@ -220,7 +257,7 @@ class DynamicSchemaLoader(SchemaLoader):
                 f"Invalid data type. Available string or two items list of string. Got {mapped_field_type}."
             )
 
-    def _resolve_complex_type(self, complex_type: ComplexFieldType) -> Mapping[str, Any]:
+    def _resolve_complex_type(self, complex_type: ComplexFieldType) -> Dict[str, Any]:
         if not complex_type.items:
             return self._get_airbyte_type(complex_type.field_type)
 
@@ -255,14 +292,14 @@ class DynamicSchemaLoader(SchemaLoader):
         return field_type
 
     @staticmethod
-    def _get_airbyte_type(field_type: str) -> MutableMapping[str, Any]:
+    def _get_airbyte_type(field_type: str) -> Dict[str, Any]:
         """
         Maps a field type to its corresponding Airbyte type definition.
         """
         if field_type not in AIRBYTE_DATA_TYPES:
             raise ValueError(f"Invalid Airbyte data type: {field_type}")
 
-        return deepcopy(AIRBYTE_DATA_TYPES[field_type])
+        return deepcopy(AIRBYTE_DATA_TYPES[field_type])  # type: ignore  # a copy of a dict should be a dict, not a MutableMapping
 
     def _extract_data(
         self,

airbyte_cdk/test/catalog_builder.py

@@ -2,6 +2,8 @@
 
 from typing import Any, Dict, List, Union, overload
 
+from airbyte_protocol_dataclasses.models import DestinationSyncMode
+
 from airbyte_cdk.models import (
     ConfiguredAirbyteCatalog,
     ConfiguredAirbyteStream,
@@ -32,6 +34,12 @@ class ConfiguredAirbyteStreamBuilder:
         self._stream["sync_mode"] = sync_mode.name
         return self
 
+    def with_destination_sync_mode(
+        self, sync_mode: DestinationSyncMode
+    ) -> "ConfiguredAirbyteStreamBuilder":
+        self._stream["destination_sync_mode"] = sync_mode.name
+        return self
+
     def with_primary_key(self, pk: List[List[str]]) -> "ConfiguredAirbyteStreamBuilder":
         self._stream["primary_key"] = pk
         self._stream["stream"]["source_defined_primary_key"] = pk  # type: ignore  # we assume that self._stream["stream"] is a Dict[str, Any]
@@ -58,7 +66,7 @@ class CatalogBuilder:
     def with_stream(
         self,
         name: Union[str, ConfiguredAirbyteStreamBuilder],
-        sync_mode: Union[SyncMode, None] = None,
+        sync_mode: SyncMode = SyncMode.full_refresh,
     ) -> "CatalogBuilder":
         # As we are introducing a fully fledge ConfiguredAirbyteStreamBuilder, we would like to deprecate the previous interface
         # with_stream(str, SyncMode)

airbyte_cdk/test/mock_http/request.py

@@ -72,7 +72,11 @@ class HttpRequest:
         elif isinstance(body, bytes):
             return json.loads(body.decode())  # type: ignore  # assumes return type of Mapping[str, Any]
         elif isinstance(body, str):
-            return json.loads(body)  # type: ignore  # assumes return type of Mapping[str, Any]
+            try:
+                return json.loads(body)  # type: ignore  # assumes return type of Mapping[str, Any]
+            except json.JSONDecodeError:
+                # one of the body is a mapping while the other isn't so comparison should fail anyway
+                return None
         return None
 
     @staticmethod

{airbyte_cdk-6.48.6.dist-info → airbyte_cdk-6.48.7.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.48.6
+Version: 6.48.7.dev0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -23,7 +23,7 @@ Provides-Extra: sql
 Provides-Extra: vector-db-based
 Requires-Dist: Jinja2 (>=3.1.2,<3.2.0)
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
-Requires-Dist: airbyte-protocol-models-dataclasses (>=0.15,<0.16)
+Requires-Dist: airbyte-protocol-models-dataclasses (==0.15.0.dev1746621859)
 Requires-Dist: anyascii (>=0.3.2,<0.4.0)
 Requires-Dist: avro (>=1.11.2,<1.13.0) ; extra == "file-based"
 Requires-Dist: backoff

{airbyte_cdk-6.48.6.dist-info → airbyte_cdk-6.48.7.dev0.dist-info}/RECORD

@@ -23,7 +23,7 @@ airbyte_cdk/connector_builder/test_reader/message_grouper.py,sha256=84BAEPIBHMq3
 airbyte_cdk/connector_builder/test_reader/reader.py,sha256=mP1yHK5vG38KxoKoT2QQ7ZNbkdLA1rMAU3EKpucjHls,21098
 airbyte_cdk/connector_builder/test_reader/types.py,sha256=hPZG3jO03kBaPyW94NI3JHRS1jxXGSNBcN1HFzOxo5Y,2528
 airbyte_cdk/destinations/__init__.py,sha256=FyDp28PT_YceJD5HDFhA-mrGfX9AONIyMQ4d68CHNxQ,213
-airbyte_cdk/destinations/destination.py,sha256=CIq-yb8C_0QvcKCtmStaHfiqn53GEfRAIGGCkJhKP1Q,5880
+airbyte_cdk/destinations/destination.py,sha256=VLuzaVniWDKtXqJzCE1yLppLGm9SvwucI3Pz82vOzgU,6755
 airbyte_cdk/destinations/vector_db_based/README.md,sha256=QAe8c_1Afme4r2TCE10cTSaxUE3zgCBuArSuRQqK8tA,2115
 airbyte_cdk/destinations/vector_db_based/__init__.py,sha256=eAkzwTjBbXBhJ5GfPO5I53Zgpv5xQFLRQS8n4nuyPt0,1006
 airbyte_cdk/destinations/vector_db_based/config.py,sha256=1u87eibIWLZ_wuaCvE3yp5ayguM9dGhGXbT8agmkUBg,12468
@@ -47,7 +47,7 @@ airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_json_data
 airbyte_cdk/manifest_migrations/migrations/http_requester_url_base_to_url.py,sha256=EX1MVYVpoWypA28qoH48wA0SYZjGdlR8bcSixTDzfgo,1346
 airbyte_cdk/manifest_migrations/migrations/registry.yaml,sha256=K5KBQ2C1T_dWExEJFuEAe1VO_QqOijOCh90rnUOCEyc,960
 airbyte_cdk/manifest_migrations/migrations_registry.py,sha256=zly2fwaOxDukqC7eowzrDlvhA2v71FjW74kDzvRXhSY,2619
-airbyte_cdk/models/__init__.py,sha256=Et9wJWs5VOWynGbb-3aJRhsdAHAiLkNNLxdwqJAuqkw,2114
+airbyte_cdk/models/__init__.py,sha256=2_fM0Vms2athSn3MDsISG9JXlfKqaeKRuLR0rudQvoc,2218
 airbyte_cdk/models/airbyte_protocol.py,sha256=oZdKsZ7yPjUt9hvxdWNpxCtgjSV2RWhf4R9Np03sqyY,3613
 airbyte_cdk/models/airbyte_protocol_serializers.py,sha256=s6SaFB2CMrG_7jTQGn_fhFbQ1FUxhCxf5kq2RWGHMVI,1749
 airbyte_cdk/models/connector_metadata.py,sha256=Lwb0JKiWvnqHduXjHHgBBBhTsGoLiNjxbG93W_OjcKQ,2875
@@ -89,7 +89,7 @@ airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=GoZJ8Oxb
 airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
 airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
-airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=AK65U441O1kjmTnURofXyxDVyLkwCI-mVNflOdVpQM8,166443
+airbyte_cdk/sources/declarative/declarative_component_schema.yaml,sha256=iWOtZ5staQtx0FZ4CSruEqyxdSO3-4SfwIQ1nw_bY_w,166744
 airbyte_cdk/sources/declarative/declarative_source.py,sha256=qmyMnnet92eGc3C22yBtpvD5UZjqdhsAafP_zxI5wp8,1814
 airbyte_cdk/sources/declarative/declarative_stream.py,sha256=dCRlddBUSaJmBNBz1pSO1r2rTw8AP5d2_vlmIeGs2gg,10767
 airbyte_cdk/sources/declarative/decoders/__init__.py,sha256=JHb_0d3SE6kNY10mxA5YBEKPeSbsWYjByq1gUQxepoE,953
@@ -133,7 +133,7 @@ airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migrati
 airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
 airbyte_cdk/sources/declarative/models/__init__.py,sha256=nUFxNCiKeYRVXuZEKA7GD-lTHxsiKcQ8FitZjKhPIvE,100
 airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py,sha256=Rq5kzR5bflqBf6td2ZAgw6lP3iN_mNi4tjntn_R01_o,5851
-airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=CXk0TwSGrpEwtbrfa8PSt6JZxpov0NqjTpEEu9Pv0OM,118040
+airbyte_cdk/sources/declarative/models/declarative_component_schema.py,sha256=P0l-NiKm873uBZ0x6vRJK2u4P-XPTGeTWMnhafKryRw,117940
 airbyte_cdk/sources/declarative/parsers/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 airbyte_cdk/sources/declarative/parsers/custom_code_compiler.py,sha256=nlVvHC511NUyDEEIRBkoeDTAvLqKNp-hRy8D19z8tdk,5941
 airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=wnRUP0Xeru9Rbu5OexXSDN9QWDo8YU4tT9M2LDVOgGA,802
@@ -210,7 +210,7 @@ airbyte_cdk/sources/declarative/retrievers/retriever.py,sha256=XPLs593Xv8c5cKMc3
 airbyte_cdk/sources/declarative/retrievers/simple_retriever.py,sha256=O7qpM71L1_ATIbEKa8y658jdiSJSPw0KmuGKgnaruQU,31008
 airbyte_cdk/sources/declarative/schema/__init__.py,sha256=xU45UvM5O4c1PSM13UHpCdh5hpW3HXy9vRRGEiAC1rg,795
 airbyte_cdk/sources/declarative/schema/default_schema_loader.py,sha256=UnbzlExmwoQiVV8zDg4lhAEaqA_0pRfwbMRe8yqOuWk,1834
-airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=J8Q_iJYhcSQLWyt0bTZCbDAGpxt9G8FCc6Q9jtGsNzw,10703
+airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py,sha256=pdQUtQXhT8VDm1wBlLF9bqYyx_wMSKOai7qyDVGUgXI,11939
 airbyte_cdk/sources/declarative/schema/inline_schema_loader.py,sha256=bVETE10hRsatRJq3R3BeyRR0wIoK3gcP1gcpVRQ_P5U,464
 airbyte_cdk/sources/declarative/schema/json_file_schema_loader.py,sha256=5Wl-fqW-pVf_dxJ4yGHMAFfC4JjKHYJhqFJT1xA57F4,4177
 airbyte_cdk/sources/declarative/schema/schema_loader.py,sha256=kjt8v0N5wWKA5zyLnrDLxf1PJKdUqvQq2RVnAOAzNSY,379
@@ -364,12 +364,12 @@ airbyte_cdk/sql/shared/catalog_providers.py,sha256=qiahORhtN6qBUGHhSKmzE00uC4i6W
 airbyte_cdk/sql/shared/sql_processor.py,sha256=1CwfC3fp9dWnHBpKtly7vGduf9ho_MahiwxGFcULG3Y,27687
 airbyte_cdk/sql/types.py,sha256=XEIhRAo_ASd0kVLBkdLf5bHiRhNple-IJrC9TibcDdY,5880
 airbyte_cdk/test/__init__.py,sha256=f_XdkOg4_63QT2k3BbKY34209lppwgw-svzfZstQEq4,199
-airbyte_cdk/test/catalog_builder.py,sha256=-y05Cz1x0Dlk6oE9LSKhCozssV2gYBNtMdV5YYOPOtk,3015
+airbyte_cdk/test/catalog_builder.py,sha256=b8sg1smzqSTNkwD0KUTZZin7tRFrjScco9v_RxKFYSU,3296
 airbyte_cdk/test/entrypoint_wrapper.py,sha256=TyUmVJyIuGelAv6y8Wy_BnwqIRw_drjfZWKlroljCuQ,9951
 airbyte_cdk/test/mock_http/__init__.py,sha256=jE5kC6CQ0OXkTqKhciDnNVZHesBFVIA2YvkdFGwva7k,322
 airbyte_cdk/test/mock_http/matcher.py,sha256=4Qj8UnJKZIs-eodshryce3SN1Ayc8GZpBETmP6hTEyc,1446
 airbyte_cdk/test/mock_http/mocker.py,sha256=XgsjMtVoeMpRELPyALgrkHFauH9H5irxrz1Kcxh2yFY,8013
-airbyte_cdk/test/mock_http/request.py,sha256=tdB8cqk2vLgCDTOKffBKsM06llYs4ZecgtH6DKyx6yY,4112
+airbyte_cdk/test/mock_http/request.py,sha256=nct71fBILqy-PP4TZAkRBor--w4CJWWZZ83w9PLklok,4304
 airbyte_cdk/test/mock_http/response.py,sha256=s4-cQQqTtmeej0pQDWqmG0vUWpHS-93lIWMpW3zSVyU,662
 airbyte_cdk/test/mock_http/response_builder.py,sha256=F-v7ebftqGj7YVIMLKdodmU9U8Dq8aIyllWGo2NGwHc,8331
 airbyte_cdk/test/standard_tests/__init__.py,sha256=YS2bghoGmQ-4GNIbe6RuEmvV-V1kpM1OyxTpebrs0Ig,1338
@@ -408,9 +408,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.48.6.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.48.6.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.48.6.dist-info/METADATA,sha256=ukLCWdp4GYARrBEW7wQubufsruhncIeEhAvMFweXs9Y,6343
-airbyte_cdk-6.48.6.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.48.6.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
-airbyte_cdk-6.48.6.dist-info/RECORD,,
+airbyte_cdk-6.48.7.dev0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.48.7.dev0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.48.7.dev0.dist-info/METADATA,sha256=KaGGArfmNfNBkCp4AEJtH66h_vzlO5e7jrKZz2Az_Tg,6358
+airbyte_cdk-6.48.7.dev0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.48.7.dev0.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
+airbyte_cdk-6.48.7.dev0.dist-info/RECORD,,