airbyte-cdk 6.48.17.dev0__py3-none-any.whl → 6.49.1__py3-none-any.whl

airbyte_cdk/cli/source_declarative_manifest/_run.py

@@ -16,15 +16,17 @@ source-declarative-manifest spec
 
 from __future__ import annotations
 
+import argparse
 import json
 import pkgutil
 import sys
 import traceback
-from collections.abc import Mapping
+from collections.abc import MutableMapping
 from pathlib import Path
 from typing import Any, cast
 
 import orjson
+import yaml
 
 from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
 from airbyte_cdk.models import (
@@ -54,7 +56,7 @@ class SourceLocalYaml(YamlDeclarativeSource):
     def __init__(
         self,
         catalog: ConfiguredAirbyteCatalog | None,
-        config: Mapping[str, Any] | None,
+        config: MutableMapping[str, Any] | None,
         state: TState,
         **kwargs: Any,
     ) -> None:
@@ -91,7 +93,8 @@ def handle_command(args: list[str]) -> None:
 
 def _get_local_yaml_source(args: list[str]) -> SourceLocalYaml:
     try:
-        config, catalog, state = _parse_inputs_into_config_catalog_state(args)
+        parsed_args = AirbyteEntrypoint.parse_args(args)
+        config, catalog, state = _parse_inputs_into_config_catalog_state(parsed_args)
         return SourceLocalYaml(config=config, catalog=catalog, state=state)
     except Exception as error:
         print(
@@ -162,14 +165,30 @@ def create_declarative_source(
     connector builder.
     """
     try:
-        config: Mapping[str, Any] | None
+        config: MutableMapping[str, Any] | None
         catalog: ConfiguredAirbyteCatalog | None
         state: list[AirbyteStateMessage]
-        config, catalog, state = _parse_inputs_into_config_catalog_state(args)
-        if config is None or "__injected_declarative_manifest" not in config:
+
+        parsed_args = AirbyteEntrypoint.parse_args(args)
+        config, catalog, state = _parse_inputs_into_config_catalog_state(parsed_args)
+
+        if config is None:
+            raise ValueError(
+                "Invalid config: `__injected_declarative_manifest` should be provided at the root "
+                "of the config or using the --manifest-path argument."
+            )
+
+        # If a manifest_path is provided in the args, inject it into the config
+        if hasattr(parsed_args, "manifest_path") and parsed_args.manifest_path:
+            injected_manifest = _parse_manifest_from_file(parsed_args.manifest_path)
+            if injected_manifest:
+                config["__injected_declarative_manifest"] = injected_manifest
+
+        if "__injected_declarative_manifest" not in config:
             raise ValueError(
                 "Invalid config: `__injected_declarative_manifest` should be provided at the root "
-                f"of the config but config only has keys: {list(config.keys() if config else [])}"
+                "of the config or using the --manifest-path argument. "
+                f"Config only has keys: {list(config.keys() if config else [])}"
             )
         if not isinstance(config["__injected_declarative_manifest"], dict):
             raise ValueError(
@@ -177,6 +196,9 @@ def create_declarative_source(
                 f"but got type: {type(config['__injected_declarative_manifest'])}"
             )
 
+        if hasattr(parsed_args, "components_path") and parsed_args.components_path:
+            _register_components_from_file(parsed_args.components_path)
+
         return ConcurrentDeclarativeSource(
             config=config,
             catalog=catalog,
@@ -205,13 +227,12 @@ def create_declarative_source(
 
 
 def _parse_inputs_into_config_catalog_state(
-    args: list[str],
+    parsed_args: argparse.Namespace,
 ) -> tuple[
-    Mapping[str, Any] | None,
+    MutableMapping[str, Any] | None,
     ConfiguredAirbyteCatalog | None,
     list[AirbyteStateMessage],
 ]:
-    parsed_args = AirbyteEntrypoint.parse_args(args)
     config = (
         ConcurrentDeclarativeSource.read_config(parsed_args.config)
         if hasattr(parsed_args, "config")
@@ -231,6 +252,44 @@ def _parse_inputs_into_config_catalog_state(
     return config, catalog, state
 
 
+def _parse_manifest_from_file(filepath: str) -> dict[str, Any] | None:
+    """Extract and parse a manifest file specified in the args."""
+    try:
+        with open(filepath, "r", encoding="utf-8") as manifest_file:
+            manifest_content = yaml.safe_load(manifest_file)
+            if manifest_content is None:
+                raise ValueError(f"Manifest file at {filepath} is empty")
+            if not isinstance(manifest_content, dict):
+                raise ValueError(f"Manifest must be a dictionary, got {type(manifest_content)}")
+            return manifest_content
+    except Exception as error:
+        raise ValueError(f"Failed to load manifest file from {filepath}: {error}")
+
+
+def _register_components_from_file(filepath: str) -> None:
+    """Load and register components from a Python file specified in the args."""
+    import importlib.util
+    import sys
+
+    components_path = Path(filepath)
+
+    module_name = "components"
+    sdm_module_name = "source_declarative_manifest.components"
+
+    # Create module spec
+    spec = importlib.util.spec_from_file_location(module_name, components_path)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Could not load module from {components_path}")
+
+    # Create module and execute code, registering the module before executing its code
+    # To avoid issues with dataclasses that look up the module
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    sys.modules[sdm_module_name] = module
+
+    spec.loader.exec_module(module)
+
+
 def run() -> None:
     args: list[str] = sys.argv[1:]
     handle_command(args)

airbyte_cdk/connector.py

@@ -8,7 +8,7 @@ import logging
 import os
 import pkgutil
 from abc import ABC, abstractmethod
-from typing import Any, Generic, Mapping, Optional, Protocol, TypeVar
+from typing import Any, Generic, Mapping, MutableMapping, Optional, Protocol, TypeVar
 
 import yaml
 
@@ -41,9 +41,9 @@ class BaseConnector(ABC, Generic[TConfig]):
         """
 
     @staticmethod
-    def read_config(config_path: str) -> Mapping[str, Any]:
+    def read_config(config_path: str) -> MutableMapping[str, Any]:
         config = BaseConnector._read_json_file(config_path)
-        if isinstance(config, Mapping):
+        if isinstance(config, MutableMapping):
             return config
         else:
             raise ValueError(

airbyte_cdk/entrypoint.py

@@ -84,6 +84,18 @@ class AirbyteEntrypoint(object):
         required_check_parser.add_argument(
             "--config", type=str, required=True, help="path to the json configuration file"
         )
+        check_parser.add_argument(
+            "--manifest-path",
+            type=str,
+            required=False,
+            help="path to the YAML manifest file to inject into the config",
+        )
+        check_parser.add_argument(
+            "--components-path",
+            type=str,
+            required=False,
+            help="path to the custom components file, if it exists",
+        )
 
         # discover
         discover_parser = subparsers.add_parser(
@@ -95,6 +107,18 @@ class AirbyteEntrypoint(object):
         required_discover_parser.add_argument(
             "--config", type=str, required=True, help="path to the json configuration file"
         )
+        discover_parser.add_argument(
+            "--manifest-path",
+            type=str,
+            required=False,
+            help="path to the YAML manifest file to inject into the config",
+        )
+        discover_parser.add_argument(
+            "--components-path",
+            type=str,
+            required=False,
+            help="path to the custom components file, if it exists",
+        )
 
         # read
         read_parser = subparsers.add_parser(
@@ -114,6 +138,18 @@ class AirbyteEntrypoint(object):
             required=True,
             help="path to the catalog used to determine which data to read",
         )
+        read_parser.add_argument(
+            "--manifest-path",
+            type=str,
+            required=False,
+            help="path to the YAML manifest file to inject into the config",
+        )
+        read_parser.add_argument(
+            "--components-path",
+            type=str,
+            required=False,
+            help="path to the custom components file, if it exists",
+        )
 
         return main_parser.parse_args(args)
 

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1741,54 +1741,6 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
-  KeyValueExtractor:
-    title: Key Value Extractor
-    description: Record extractor that extract with .
-    type: object
-    required:
-      - type
-      - keys_extractor
-      - values_extractor
-    properties:
-      type:
-        type: string
-        enum: [ KeyValueExtractor ]
-      keys_extractor:
-        description: placeholder
-        anyOf:
-          - "$ref": "#/definitions/DpathExtractor"
-          - "$ref": "#/definitions/CustomRecordExtractor"
-      values_extractor:
-        description: placeholder
-        anyOf:
-          - "$ref": "#/definitions/DpathExtractor"
-          - "$ref": "#/definitions/CustomRecordExtractor"
-      $parameters:
-        type: object
-        additionalProperties: true
-  CombinedExtractor:
-    title: Combined Extractor
-    description: Record extractor that extract with .
-    type: object
-    required:
-      - type
-      - extractors
-    properties:
-      type:
-        type: string
-        enum: [ CombinedExtractor ]
-      extractors:
-        description: placeholder
-        type: array
-        items:
-          anyOf:
-            - "$ref": "#/definitions/DpathExtractor"
-            - "$ref": "#/definitions/CombinedExtractor"
-            - "$ref": "#/definitions/KeyValueExtractor"
-            - "$ref": "#/definitions/CustomRecordExtractor"
-      $parameters:
-        type: object
-        additionalProperties: true
   DpathExtractor:
     title: Dpath Extractor
     description: Record extractor that searches a decoded response over a path defined as an array of fields.
@@ -2366,12 +2318,6 @@ definitions:
           - "$ref": "#/definitions/AsyncRetriever"
           - "$ref": "#/definitions/CustomRetriever"
           - "$ref": "#/definitions/SimpleRetriever"
-      schema_filter:
-        title: Schema Filter
-        description: placeholder
-        anyOf:
-          - "$ref": "#/definitions/RecordFilter"
-          - "$ref": "#/definitions/CustomRecordFilter"
       schema_transformations:
         title: Schema Transformations
         description: A list of transformations to be applied to the schema.
@@ -3369,8 +3315,6 @@ definitions:
       extractor:
         anyOf:
           - "$ref": "#/definitions/DpathExtractor"
-          - "$ref": "#/definitions/CombinedExtractor"
-          - "$ref": "#/definitions/KeyValueExtractor"
           - "$ref": "#/definitions/CustomRecordExtractor"
       record_filter:
         title: Record Filter
@@ -3859,6 +3803,61 @@ definitions:
         title: Advanced Auth
         description: Advanced specification for configuring the authentication flow.
         "$ref": "#/definitions/AuthFlow"
+      config_normalization_rules:
+        title: Config Normalization Rules
+        type: object
+        additionalProperties: false
+        properties:
+          config_migrations:
+            title: Config Migrations
+            description: The discrete migrations that will be applied on the incoming config. Each migration will be applied in the order they are defined.
+            type: array
+            items:
+              "$ref": "#/definitions/ConfigMigration"
+            default: []
+          transformations:
+            title: Transformations
+            description: The list of transformations that will be applied on the incoming config at the start of each sync. The transformations will be applied in the order they are defined.
+            type: array
+            items:
+              anyOf:
+                - "$ref": "#/definitions/ConfigRemapField"
+                - "$ref": "#/definitions/ConfigAddFields"
+                - "$ref": "#/definitions/ConfigRemoveFields"
+            default: []
+          validations:
+            title: Validations
+            description: The list of validations that will be performed on the incoming config at the start of each sync.
+            type: array
+            items:
+              anyOf:
+                - "$ref": "#/definitions/DpathValidator"
+                - "$ref": "#/definitions/PredicateValidator"
+            default: []
+  ConfigMigration:
+    title: Config Migration
+    description: A config migration that will be applied on the incoming config at the start of a sync.
+    type: object
+    required:
+      - type
+      - transformations
+    properties:
+      type:
+        type: string
+        enum: [ConfigMigration]
+      description:
+        type: string
+        description: The description/purpose of the config migration.
+      transformations:
+        title: Transformations
+        description: The list of transformations that will attempt to be applied on an incoming unmigrated config. The transformations will be applied in the order they are defined.
+        type: array
+        items:
+          anyOf:
+            - "$ref": "#/definitions/ConfigRemapField"
+            - "$ref": "#/definitions/ConfigAddFields"
+            - "$ref": "#/definitions/ConfigRemoveFields"
+        default: []
   SubstreamPartitionRouter:
     title: Substream Partition Router
     description: Partition router that is used to retrieve records that have been partitioned according to records from the specified parent streams. An example of a parent stream is automobile brands and the substream would be the various car models associated with each branch.
@@ -4050,9 +4049,6 @@ definitions:
         title: Value Type
         description: The expected data type of the value. If omitted, the type will be inferred from the value provided.
         "$ref": "#/definitions/ValueType"
-      create_or_update:
-        type: boolean
-        default: false
       $parameters:
         type: object
         additionalProperties: true
@@ -4104,10 +4100,6 @@ definitions:
           - ["data"]
           - ["data", "streams"]
           - ["data", "{{ parameters.name }}"]
-      default_values:
-        title: Default Values
-        description: placeholder
-        type: array
       $parameters:
         type: object
         additionalProperties: true
@@ -4119,11 +4111,7 @@ definitions:
         type: string
         enum: [ConfigComponentsResolver]
       stream_config:
-        anyOf:
-          - type: array
-            items:
-              "$ref": "#/definitions/StreamConfig"
-          - "$ref": "#/definitions/StreamConfig"
+        "$ref": "#/definitions/StreamConfig"
       components_mapping:
         type: array
         items:
@@ -4152,7 +4140,9 @@ definitions:
       stream_template:
         title: Stream Template
         description: Reference to the stream template.
-        "$ref": "#/definitions/DeclarativeStream"
+        anyOf:
+          - "$ref": "#/definitions/DeclarativeStream"
+          - "$ref": "#/definitions/StateDelegatingStream"
       components_resolver:
         title: Components Resolver
         description: Component resolve and populates stream templates with components values.
@@ -4231,6 +4221,224 @@ definitions:
         description: The GraphQL query to be executed
         default: {}
     additionalProperties: true
+  DpathValidator:
+    title: Dpath Validator
+    description: Validator that extracts the value located at a given field path.
+    type: object
+    required:
+      - type
+      - field_path
+      - validation_strategy
+    properties:
+      type:
+        type: string
+        enum: [DpathValidator]
+      field_path:
+        title: Field Path
+        description: List of potentially nested fields describing the full path of the field to validate. Use "*" to validate all values from an array.
+        type: array
+        items:
+          type: string
+        interpolation_context:
+          - config
+        examples:
+          - ["data"]
+          - ["data", "records"]
+          - ["data", "{{ parameters.name }}"]
+          - ["data", "*", "record"]
+      validation_strategy:
+        title: Validation Strategy
+        description: The condition that the specified config value will be evaluated against
+        anyOf:
+          - "$ref": "#/definitions/ValidateAdheresToSchema"
+  PredicateValidator:
+    title: Predicate Validator
+    description: Validator that applies a validation strategy to a specified value.
+    type: object
+    required:
+      - type
+      - value
+      - validation_strategy
+    properties:
+      type:
+        type: string
+        enum: [PredicateValidator]
+      value:
+        title: Value
+        description: The value to be validated. Can be a literal value or interpolated from configuration.
+        type:
+          - string
+          - number
+          - object
+          - array
+          - boolean
+          - "null"
+        interpolation_context:
+          - config
+        examples:
+          - "test-value"
+          - "{{ config['api_version'] }}"
+          - "{{ config['tenant_id'] }}"
+          - 123
+      validation_strategy:
+        title: Validation Strategy
+        description: The validation strategy to apply to the value.
+        anyOf:
+          - "$ref": "#/definitions/ValidateAdheresToSchema"
+  ValidateAdheresToSchema:
+    title: Validate Adheres To Schema
+    description: Validates that a user-provided schema adheres to a specified JSON schema.
+    type: object
+    required:
+      - type
+      - base_schema
+    properties:
+      type:
+        type: string
+        enum: [ValidateAdheresToSchema]
+      base_schema:
+        title: Base JSON Schema
+        description: The base JSON schema against which the user-provided schema will be validated.
+        type:
+          - string
+          - object
+        interpolation_context:
+          - config
+        examples:
+          - "{{ config['report_validation_schema'] }}"
+          - |
+            '{
+              "$schema": "http://json-schema.org/draft-07/schema#",
+              "title": "Person",
+              "type": "object",
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "description": "The person's name"
+                },
+                "age": {
+                  "type": "integer",
+                  "minimum": 0,
+                  "description": "The person's age"
+                }
+              },
+              "required": ["name", "age"]
+            }'
+          - $schema: "http://json-schema.org/draft-07/schema#"
+            title: Person
+            type: object
+            properties:
+              name:
+                type: string
+                description: "The person's name"
+              age:
+                type: integer
+                minimum: 0
+                description: "The person's age"
+            required:
+              - name
+              - age
+  ConfigRemapField:
+    title: Remap Field
+    description: Transformation that remaps a field's value to another value based on a static map.
+    type: object
+    required:
+      - type
+      - map
+      - field_path
+    properties:
+      type:
+        type: string
+        enum: [ConfigRemapField]
+      map:
+        title: Value Mapping
+        description: A mapping of original values to new values. When a field value matches a key in this map, it will be replaced with the corresponding value.
+        interpolation_context:
+          - config
+        type:
+          - object
+          - string
+        additionalProperties: true
+        examples:
+          - pending: "in_progress"
+            done: "completed"
+            cancelled: "terminated"
+          - "{{ config['status_mapping'] }}"
+      field_path:
+        title: Field Path
+        description: The path to the field whose value should be remapped. Specified as a list of path components to navigate through nested objects.
+        interpolation_context:
+          - config
+        type: array
+        items:
+          type: string
+        examples:
+          - ["status"]
+          - ["data", "status"]
+          - ["data", "{{ config.name }}", "status"]
+          - ["data", "*", "status"]
+  ConfigAddFields:
+    title: Config Add Fields
+    description: Transformation that adds fields to a config. The path of the added field can be nested.
+    type: object
+    required:
+      - type
+      - fields
+    properties:
+      type:
+        type: string
+        enum: [ConfigAddFields]
+      fields:
+        title: Fields
+        description: A list of transformations (path and corresponding value) that will be added to the config.
+        type: array
+        items:
+          "$ref": "#/definitions/AddedFieldDefinition"
+      condition:
+        description: Fields will be added if expression is evaluated to True.
+        type: string
+        default: ""
+        interpolation_context:
+          - config
+          - property
+        examples:
+          - "{{ config['environemnt'] == 'sandbox' }}"
+          - "{{ property is integer }}"
+          - "{{ property|length > 5 }}"
+          - "{{ property == 'some_string_to_match' }}"
+  ConfigRemoveFields:
+    title: Config Remove Fields
+    description: Transformation that removes a field from the config.
+    type: object
+    required:
+      - type
+      - field_pointers
+    properties:
+      type:
+        type: string
+        enum: [ConfigRemoveFields]
+      field_pointers:
+        title: Field Pointers
+        description: A list of field pointers to be removed from the config.
+        type: array
+        items:
+          items:
+            type: string
+        examples:
+          - ["tags"]
+          - [["content", "html"], ["content", "plain_text"]]
+      condition:
+        description: Fields will be removed if expression is evaluated to True.
+        type: string
+        default: ""
+        interpolation_context:
+          - config
+          - property
+        examples:
+          - "{{ config['environemnt'] == 'sandbox' }}"
+          - "{{ property is integer }}"
+          - "{{ property|length > 5 }}"
+          - "{{ property == 'some_string_to_match' }}"
 interpolation:
   variables:
     - title: config

airbyte_cdk/sources/declarative/extractors/__init__.py

@@ -2,10 +2,8 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from airbyte_cdk.sources.declarative.extractors.combined_extractor import CombinedExtractor
 from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor
 from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
-from airbyte_cdk.sources.declarative.extractors.key_value_extractor import KeyValueExtractor
 from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
 from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector
 from airbyte_cdk.sources.declarative.extractors.response_to_file_extractor import (
@@ -20,6 +18,4 @@ __all__ = [
     "RecordFilter",
     "RecordSelector",
     "ResponseToFileExtractor",
-    "KeyValueExtractor",
-    "CombinedExtractor",
 ]

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -262,7 +262,9 @@ class ManifestDeclarativeSource(DeclarativeSource):
             }
         )
 
-        stream_configs = self._stream_configs(self._source_config) + self.dynamic_streams
+        stream_configs = self._stream_configs(self._source_config) + self._dynamic_stream_configs(
+            self._source_config, config
+        )
 
         api_budget_model = self._source_config.get("api_budget")
         if api_budget_model: