airbyte-cdk 6.49.1__tar.gz → 6.51.0__tar.gz

{airbyte_cdk-6.49.1 → airbyte_cdk-6.51.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.49.1
+Version: 6.51.0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.49.1 → airbyte_cdk-6.51.0}/airbyte_cdk/cli/source_declarative_manifest/_run.py

@@ -58,6 +58,7 @@ class SourceLocalYaml(YamlDeclarativeSource):
         catalog: ConfiguredAirbyteCatalog | None,
         config: MutableMapping[str, Any] | None,
         state: TState,
+        config_path: str | None = None,
         **kwargs: Any,
     ) -> None:
         """
@@ -76,6 +77,7 @@ class SourceLocalYaml(YamlDeclarativeSource):
             config=config,
             state=state,  # type: ignore [arg-type]
             path_to_yaml="manifest.yaml",
+            config_path=config_path,
         )
 
 
@@ -95,7 +97,12 @@ def _get_local_yaml_source(args: list[str]) -> SourceLocalYaml:
     try:
         parsed_args = AirbyteEntrypoint.parse_args(args)
         config, catalog, state = _parse_inputs_into_config_catalog_state(parsed_args)
-        return SourceLocalYaml(config=config, catalog=catalog, state=state)
+        return SourceLocalYaml(
+            config=config,
+            catalog=catalog,
+            state=state,
+            config_path=parsed_args.config if hasattr(parsed_args, "config") else None,
+        )
     except Exception as error:
         print(
             orjson.dumps(
@@ -204,6 +211,7 @@ def create_declarative_source(
             catalog=catalog,
             state=state,
             source_config=cast(dict[str, Any], config["__injected_declarative_manifest"]),
+            config_path=parsed_args.config if hasattr(parsed_args, "config") else None,
         )
     except Exception as error:
         print(

{airbyte_cdk-6.49.1 → airbyte_cdk-6.51.0}/airbyte_cdk/sources/declarative/concurrent_declarative_source.py

@@ -74,6 +74,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
         debug: bool = False,
         emit_connector_builder_messages: bool = False,
         component_factory: Optional[ModelToComponentFactory] = None,
+        config_path: Optional[str] = None,
         **kwargs: Any,
     ) -> None:
         # todo: We could remove state from initialization. Now that streams are grouped during the read(), a source
@@ -96,6 +97,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
             debug=debug,
             emit_connector_builder_messages=emit_connector_builder_messages,
             component_factory=component_factory,
+            config_path=config_path,
         )
 
         concurrency_level_from_manifest = self._source_config.get("concurrency_level")

{airbyte_cdk-6.49.1 → airbyte_cdk-6.51.0}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -2318,6 +2318,12 @@ definitions:
           - "$ref": "#/definitions/AsyncRetriever"
           - "$ref": "#/definitions/CustomRetriever"
           - "$ref": "#/definitions/SimpleRetriever"
+      schema_filter:
+        title: Schema Filter
+        description: Responsible for filtering fields to be added to json schema.
+        anyOf:
+          - "$ref": "#/definitions/RecordFilter"
+          - "$ref": "#/definitions/CustomRecordFilter"
       schema_transformations:
         title: Schema Transformations
         description: A list of transformations to be applied to the schema.

{airbyte_cdk-6.49.1 → airbyte_cdk-6.51.0}/airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
 #
 
 import json
@@ -8,13 +8,15 @@ import pkgutil
 from copy import deepcopy
 from importlib import metadata
 from types import ModuleType
-from typing import Any, Dict, Iterator, List, Mapping, Optional, Set
+from typing import Any, Dict, Iterator, List, Mapping, MutableMapping, Optional, Set
 
+import orjson
 import yaml
 from jsonschema.exceptions import ValidationError
 from jsonschema.validators import validate
 from packaging.version import InvalidVersion, Version
 
+from airbyte_cdk.config_observation import create_connector_config_control_message
 from airbyte_cdk.connector_builder.models import (
     LogMessage as ConnectorBuilderLogMessage,
 )
@@ -29,6 +31,7 @@ from airbyte_cdk.models import (
     ConnectorSpecification,
     FailureType,
 )
+from airbyte_cdk.models.airbyte_protocol_serializers import AirbyteMessageSerializer
 from airbyte_cdk.sources.declarative.checks import COMPONENTS_CHECKER_TYPE_MAPPING
 from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
 from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
@@ -57,9 +60,10 @@ from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
     ModelToComponentFactory,
 )
 from airbyte_cdk.sources.declarative.resolvers import COMPONENTS_RESOLVER_TYPE_MAPPING
+from airbyte_cdk.sources.declarative.spec.spec import Spec
 from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.streams.core import Stream
-from airbyte_cdk.sources.types import ConnectionDefinition
+from airbyte_cdk.sources.types import Config, ConnectionDefinition
 from airbyte_cdk.sources.utils.slice_logger import (
     AlwaysLogSliceLogger,
     DebugSliceLogger,
@@ -99,6 +103,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
         component_factory: Optional[ModelToComponentFactory] = None,
         migrate_manifest: Optional[bool] = False,
         normalize_manifest: Optional[bool] = False,
+        config_path: Optional[str] = None,
     ) -> None:
         """
         Args:
@@ -108,6 +113,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
             emit_connector_builder_messages: True if messages should be emitted to the connector builder.
             component_factory: optional factory if ModelToComponentFactory's default behavior needs to be tweaked.
             normalize_manifest: Optional flag to indicate if the manifest should be normalized.
+            config_path: Optional path to the config file.
         """
         self.logger = logging.getLogger(f"airbyte.{self.name}")
         self._should_normalize = normalize_manifest
@@ -130,7 +136,6 @@ class ManifestDeclarativeSource(DeclarativeSource):
         self._slice_logger: SliceLogger = (
             AlwaysLogSliceLogger() if emit_connector_builder_messages else DebugSliceLogger()
         )
-        self._config = config or {}
 
         # resolve all components in the manifest
         self._source_config = self._pre_process_manifest(dict(source_config))
@@ -139,6 +144,12 @@ class ManifestDeclarativeSource(DeclarativeSource):
         # apply additional post-processing to the manifest
         self._post_process_manifest()
 
+        spec: Optional[Mapping[str, Any]] = self._source_config.get("spec")
+        self._spec_component: Optional[Spec] = (
+            self._constructor.create_component(SpecModel, spec, dict()) if spec else None
+        )
+        self._config = self._migrate_and_transform_config(config_path, config) or {}
+
     @property
     def resolved_manifest(self) -> Mapping[str, Any]:
         """
@@ -199,6 +210,30 @@ class ManifestDeclarativeSource(DeclarativeSource):
             normalizer = ManifestNormalizer(self._source_config, self._declarative_component_schema)
             self._source_config = normalizer.normalize()
 
+    def _migrate_and_transform_config(
+        self,
+        config_path: Optional[str],
+        config: Optional[Config],
+    ) -> Optional[Config]:
+        if not config:
+            return None
+        if not self._spec_component:
+            return config
+        mutable_config = dict(config)
+        self._spec_component.migrate_config(mutable_config)
+        if mutable_config != config:
+            if config_path:
+                with open(config_path, "w") as f:
+                    json.dump(mutable_config, f)
+            self.message_repository.emit_message(
+                create_connector_config_control_message(mutable_config)
+            )
+            # We have no mechanism for consuming the queue, so we print the messages to stdout
+            for message in self.message_repository.consume_queue():
+                print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode())
+        self._spec_component.transform_config(mutable_config)
+        return mutable_config
+
     def _migrate_manifest(self) -> None:
         """
         This method is used to migrate the manifest. It should be called after the manifest has been validated.
@@ -255,6 +290,9 @@ class ManifestDeclarativeSource(DeclarativeSource):
             )
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
+        if self._spec_component:
+            self._spec_component.validate_config(config)
+
         self._emit_manifest_debug_message(
             extra_args={
                 "source_name": self.name,
@@ -355,14 +393,9 @@ class ManifestDeclarativeSource(DeclarativeSource):
             }
         )
 
-        spec = self._source_config.get("spec")
-        if spec:
-            if "type" not in spec:
-                spec["type"] = "Spec"
-            spec_component = self._constructor.create_component(SpecModel, spec, dict())
-            return spec_component.generate_spec()
-        else:
-            return super().spec(logger)
+        return (
+            self._spec_component.generate_spec() if self._spec_component else super().spec(logger)
+        )
 
     def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus:
         self._configure_logger_level(logger)

{airbyte_cdk-6.49.1 → airbyte_cdk-6.51.0}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -1,5 +1,3 @@
-# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
-
 # generated by datamodel-codegen:
 #   filename:  declarative_component_schema.yaml
 
@@ -2582,6 +2580,11 @@ class DynamicSchemaLoader(BaseModel):
         description="Component used to coordinate how records are extracted across stream slices and request pages.",
         title="Retriever",
     )
+    schema_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field(
+        None,
+        description="Responsible for filtering fields to be added to json schema.",
+        title="Schema Filter",
+    )
     schema_transformations: Optional[
         List[
             Union[

{airbyte_cdk-6.49.1 → airbyte_cdk-6.51.0}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -2544,10 +2544,19 @@ class ModelToComponentFactory:
         schema_type_identifier = self._create_component_from_model(
             model.schema_type_identifier, config=config, parameters=model.parameters or {}
         )
+        schema_filter = (
+            self._create_component_from_model(
+                model.schema_filter, config=config, parameters=model.parameters or {}
+            )
+            if model.schema_filter is not None
+            else None
+        )
+
         return DynamicSchemaLoader(
             retriever=retriever,
             config=config,
             schema_transformations=schema_transformations,
+            schema_filter=schema_filter,
             schema_type_identifier=schema_type_identifier,
             parameters=model.parameters or {},
         )

{airbyte_cdk-6.49.1 → airbyte_cdk-6.51.0}/airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py

@@ -10,6 +10,7 @@ from typing import Any, List, Mapping, MutableMapping, Optional, Union
 import dpath
 from typing_extensions import deprecated
 
+from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
 from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever
@@ -126,6 +127,7 @@ class DynamicSchemaLoader(SchemaLoader):
     parameters: InitVar[Mapping[str, Any]]
     schema_type_identifier: SchemaTypeIdentifier
     schema_transformations: List[RecordTransformation] = field(default_factory=lambda: [])
+    schema_filter: Optional[RecordFilter] = None
 
     def get_json_schema(self) -> Mapping[str, Any]:
         """
@@ -151,20 +153,18 @@ class DynamicSchemaLoader(SchemaLoader):
             )
             properties[key] = value
 
-        transformed_properties = self._transform(properties, {})
+        filtered_transformed_properties = self._transform(self._filter(properties))
 
         return {
             "$schema": "https://json-schema.org/draft-07/schema#",
             "type": "object",
             "additionalProperties": True,
-            "properties": transformed_properties,
+            "properties": filtered_transformed_properties,
         }
 
     def _transform(
         self,
         properties: Mapping[str, Any],
-        stream_state: StreamState,
-        stream_slice: Optional[StreamSlice] = None,
     ) -> Mapping[str, Any]:
         for transformation in self.schema_transformations:
             transformation.transform(
@@ -173,6 +173,21 @@ class DynamicSchemaLoader(SchemaLoader):
             )
         return properties
 
+    def _filter(
+        self,
+        properties: Mapping[str, Any],
+    ) -> Mapping[str, Any]:
+        if not self.schema_filter:
+            return properties
+
+        filtered_properties: MutableMapping[str, Any] = {}
+        for item in self.schema_filter.filter_records(
+            ({k: v} for k, v in properties.items()),
+            {},
+        ):
+            filtered_properties.update(item)
+        return filtered_properties
+
     def _get_key(
         self,
         raw_schema: MutableMapping[str, Any],

{airbyte_cdk-6.49.1 → airbyte_cdk-6.51.0}/airbyte_cdk/sources/declarative/spec/spec.py

@@ -1,28 +1,21 @@
 #
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
 #
 
-import json
 from dataclasses import InitVar, dataclass, field
 from typing import Any, List, Mapping, MutableMapping, Optional
 
-import orjson
-
-from airbyte_cdk.config_observation import create_connector_config_control_message
-from airbyte_cdk.entrypoint import AirbyteEntrypoint
 from airbyte_cdk.models import (
     AdvancedAuth,
     ConnectorSpecification,
     ConnectorSpecificationSerializer,
 )
-from airbyte_cdk.models.airbyte_protocol_serializers import AirbyteMessageSerializer
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import AuthFlow
 from airbyte_cdk.sources.declarative.transformations.config_transformations.config_transformation import (
     ConfigTransformation,
 )
 from airbyte_cdk.sources.declarative.validators.validator import Validator
 from airbyte_cdk.sources.message.repository import InMemoryMessageRepository, MessageRepository
-from airbyte_cdk.sources.source import Source
 
 
 @dataclass
@@ -48,7 +41,6 @@ class Spec:
     config_migrations: List[ConfigMigration] = field(default_factory=list)
     config_transformations: List[ConfigTransformation] = field(default_factory=list)
     config_validations: List[Validator] = field(default_factory=list)
-    message_repository: MessageRepository = InMemoryMessageRepository()
 
     def generate_spec(self) -> ConnectorSpecification:
         """
@@ -69,34 +61,15 @@ class Spec:
         # We remap these keys to camel case because that's the existing format expected by the rest of the platform
         return ConnectorSpecificationSerializer.load(obj)
 
-    def migrate_config(
-        self, args: List[str], source: Source, config: MutableMapping[str, Any]
-    ) -> None:
+    def migrate_config(self, config: MutableMapping[str, Any]) -> None:
         """
-        Apply all specified config transformations to the provided config and save the modified config to the given path and emit a control message.
+        Apply all specified config transformations to the provided config and emit a control message.
 
-        :param args: Command line arguments
-        :param source: Source instance
         :param config: The user-provided config to migrate
         """
-        config_path = AirbyteEntrypoint(source).extract_config(args)
-
-        if not config_path:
-            return
-
-        mutable_config = dict(config)
         for migration in self.config_migrations:
             for transformation in migration.transformations:
-                transformation.transform(mutable_config)
-
-        if mutable_config != config:
-            with open(config_path, "w") as f:
-                json.dump(mutable_config, f)
-            self.message_repository.emit_message(
-                create_connector_config_control_message(mutable_config)
-            )
-            for message in self.message_repository.consume_queue():
-                print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode())
+                transformation.transform(config)
 
     def transform_config(self, config: MutableMapping[str, Any]) -> None:
         """
@@ -107,7 +80,7 @@ class Spec:
         for transformation in self.config_transformations:
             transformation.transform(config)
 
-    def validate_config(self, config: MutableMapping[str, Any]) -> None:
+    def validate_config(self, config: Mapping[str, Any]) -> None:
         """
         Apply all config validations to the provided config.
 

{airbyte_cdk-6.49.1 → airbyte_cdk-6.51.0}/airbyte_cdk/sources/declarative/yaml_declarative_source.py

@@ -24,6 +24,7 @@ class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage
         catalog: Optional[ConfiguredAirbyteCatalog] = None,
         config: Optional[Mapping[str, Any]] = None,
         state: Optional[List[AirbyteStateMessage]] = None,
+        config_path: Optional[str] = None,
     ) -> None:
         """
         :param path_to_yaml: Path to the yaml file describing the source
@@ -36,6 +37,7 @@ class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage
             config=config or {},
             state=state or [],
             source_config=source_config,
+            config_path=config_path,
         )
 
     def _read_and_parse_yaml_file(self, path_to_yaml_file: str) -> ConnectionDefinition:

{airbyte_cdk-6.49.1 → airbyte_cdk-6.51.0}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.49.1"  # Version will be calculated dynamically.
+version = "6.51.0"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false
@@ -170,7 +170,7 @@ fix-and-check = {sequence = ["fix-all", "check-all"], help = "Lint-fix and forma
 
 # PyTest tasks
 pytest = {cmd = "poetry run coverage run -m pytest --durations=10", help = "Run all pytest tests."}
-pytest-fast = {cmd = "poetry run coverage run -m pytest --skipslow --durations=5 --exitfirst -m 'not flaky and not slow and not requires_creds'", help = "Run pytest tests, failing fast and excluding slow tests."}
+pytest-fast = {cmd = "poetry run coverage run -m pytest unit_tests --durations=5 --exitfirst -m 'not flaky and not slow and not requires_creds'", help = "Run pytest tests, failing fast and excluding slow tests."}
 unit-test-with-cov = {cmd = "poetry run pytest -s unit_tests --cov=airbyte_cdk --cov-report=term --cov-config ./pyproject.toml", help = "Run unit tests and create a coverage report."}
 
 # Combined check tasks (other)