airbyte-cdk 6.48.16__py3-none-any.whl → 6.48.16.post14.dev15122056170__py3-none-any.whl

airbyte_cdk/cli/source_declarative_manifest/_run.py

@@ -16,15 +16,17 @@ source-declarative-manifest spec
 
 from __future__ import annotations
 
+import argparse
 import json
 import pkgutil
 import sys
 import traceback
-from collections.abc import Mapping
+from collections.abc import MutableMapping
 from pathlib import Path
 from typing import Any, cast
 
 import orjson
+import yaml
 
 from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
 from airbyte_cdk.models import (
@@ -54,7 +56,7 @@ class SourceLocalYaml(YamlDeclarativeSource):
     def __init__(
         self,
         catalog: ConfiguredAirbyteCatalog | None,
-        config: Mapping[str, Any] | None,
+        config: MutableMapping[str, Any] | None,
         state: TState,
         **kwargs: Any,
     ) -> None:
@@ -91,7 +93,8 @@ def handle_command(args: list[str]) -> None:
 
 def _get_local_yaml_source(args: list[str]) -> SourceLocalYaml:
     try:
-        config, catalog, state = _parse_inputs_into_config_catalog_state(args)
+        parsed_args = AirbyteEntrypoint.parse_args(args)
+        config, catalog, state = _parse_inputs_into_config_catalog_state(parsed_args)
         return SourceLocalYaml(config=config, catalog=catalog, state=state)
     except Exception as error:
         print(
@@ -162,14 +165,30 @@ def create_declarative_source(
     connector builder.
     """
     try:
-        config: Mapping[str, Any] | None
+        config: MutableMapping[str, Any] | None
         catalog: ConfiguredAirbyteCatalog | None
         state: list[AirbyteStateMessage]
-        config, catalog, state = _parse_inputs_into_config_catalog_state(args)
-        if config is None or "__injected_declarative_manifest" not in config:
+
+        parsed_args = AirbyteEntrypoint.parse_args(args)
+        config, catalog, state = _parse_inputs_into_config_catalog_state(parsed_args)
+
+        if config is None:
+            raise ValueError(
+                "Invalid config: `__injected_declarative_manifest` should be provided at the root "
+                "of the config or using the --manifest-path argument."
+            )
+
+        # If a manifest_path is provided in the args, inject it into the config
+        if hasattr(parsed_args, "manifest_path") and parsed_args.manifest_path:
+            injected_manifest = _parse_manifest_from_file(parsed_args.manifest_path)
+            if injected_manifest:
+                config["__injected_declarative_manifest"] = injected_manifest
+
+        if "__injected_declarative_manifest" not in config:
             raise ValueError(
                 "Invalid config: `__injected_declarative_manifest` should be provided at the root "
-                f"of the config but config only has keys: {list(config.keys() if config else [])}"
+                "of the config or using the --manifest-path argument. "
+                f"Config only has keys: {list(config.keys() if config else [])}"
             )
         if not isinstance(config["__injected_declarative_manifest"], dict):
             raise ValueError(
@@ -177,6 +196,9 @@ def create_declarative_source(
                 f"but got type: {type(config['__injected_declarative_manifest'])}"
             )
 
+        if hasattr(parsed_args, "components_path") and parsed_args.components_path:
+            _register_components_from_file(parsed_args.components_path)
+
         return ConcurrentDeclarativeSource(
             config=config,
             catalog=catalog,
@@ -205,13 +227,12 @@ def create_declarative_source(
 
 
 def _parse_inputs_into_config_catalog_state(
-    args: list[str],
+    parsed_args: argparse.Namespace,
 ) -> tuple[
-    Mapping[str, Any] | None,
+    MutableMapping[str, Any] | None,
     ConfiguredAirbyteCatalog | None,
     list[AirbyteStateMessage],
 ]:
-    parsed_args = AirbyteEntrypoint.parse_args(args)
     config = (
         ConcurrentDeclarativeSource.read_config(parsed_args.config)
         if hasattr(parsed_args, "config")
@@ -231,6 +252,44 @@ def _parse_inputs_into_config_catalog_state(
     return config, catalog, state
 
 
+def _parse_manifest_from_file(filepath: str) -> dict[str, Any] | None:
+    """Extract and parse a manifest file specified in the args."""
+    try:
+        with open(filepath, "r", encoding="utf-8") as manifest_file:
+            manifest_content = yaml.safe_load(manifest_file)
+            if manifest_content is None:
+                raise ValueError(f"Manifest file at {filepath} is empty")
+            if not isinstance(manifest_content, dict):
+                raise ValueError(f"Manifest must be a dictionary, got {type(manifest_content)}")
+            return manifest_content
+    except Exception as error:
+        raise ValueError(f"Failed to load manifest file from {filepath}: {error}")
+
+
+def _register_components_from_file(filepath: str) -> None:
+    """Load and register components from a Python file specified in the args."""
+    import importlib.util
+    import sys
+
+    components_path = Path(filepath)
+
+    module_name = "components"
+    sdm_module_name = "source_declarative_manifest.components"
+
+    # Create module spec
+    spec = importlib.util.spec_from_file_location(module_name, components_path)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Could not load module from {components_path}")
+
+    # Create module and execute code, registering the module before executing its code
+    # To avoid issues with dataclasses that look up the module
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    sys.modules[sdm_module_name] = module
+
+    spec.loader.exec_module(module)
+
+
 def run() -> None:
     args: list[str] = sys.argv[1:]
     handle_command(args)

airbyte_cdk/connector.py

@@ -8,7 +8,7 @@ import logging
 import os
 import pkgutil
 from abc import ABC, abstractmethod
-from typing import Any, Generic, Mapping, Optional, Protocol, TypeVar
+from typing import Any, Generic, Mapping, MutableMapping, Optional, Protocol, TypeVar
 
 import yaml
 
@@ -41,9 +41,9 @@ class BaseConnector(ABC, Generic[TConfig]):
         """
 
     @staticmethod
-    def read_config(config_path: str) -> Mapping[str, Any]:
+    def read_config(config_path: str) -> MutableMapping[str, Any]:
         config = BaseConnector._read_json_file(config_path)
-        if isinstance(config, Mapping):
+        if isinstance(config, MutableMapping):
             return config
         else:
             raise ValueError(

airbyte_cdk/entrypoint.py

@@ -84,6 +84,18 @@ class AirbyteEntrypoint(object):
         required_check_parser.add_argument(
             "--config", type=str, required=True, help="path to the json configuration file"
         )
+        check_parser.add_argument(
+            "--manifest-path",
+            type=str,
+            required=False,
+            help="path to the YAML manifest file to inject into the config",
+        )
+        check_parser.add_argument(
+            "--components-path",
+            type=str,
+            required=False,
+            help="path to the custom components file, if it exists",
+        )
 
         # discover
         discover_parser = subparsers.add_parser(
@@ -95,6 +107,18 @@ class AirbyteEntrypoint(object):
         required_discover_parser.add_argument(
             "--config", type=str, required=True, help="path to the json configuration file"
         )
+        discover_parser.add_argument(
+            "--manifest-path",
+            type=str,
+            required=False,
+            help="path to the YAML manifest file to inject into the config",
+        )
+        discover_parser.add_argument(
+            "--components-path",
+            type=str,
+            required=False,
+            help="path to the custom components file, if it exists",
+        )
 
         # read
         read_parser = subparsers.add_parser(
@@ -114,6 +138,18 @@ class AirbyteEntrypoint(object):
             required=True,
             help="path to the catalog used to determine which data to read",
         )
+        read_parser.add_argument(
+            "--manifest-path",
+            type=str,
+            required=False,
+            help="path to the YAML manifest file to inject into the config",
+        )
+        read_parser.add_argument(
+            "--components-path",
+            type=str,
+            required=False,
+            help="path to the custom components file, if it exists",
+        )
 
         return main_parser.parse_args(args)
 

{airbyte_cdk-6.48.16.dist-info → airbyte_cdk-6.48.16.post14.dev15122056170.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.48.16
+Version: 6.48.16.post14.dev15122056170
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.48.16.dist-info → airbyte_cdk-6.48.16.post14.dev15122056170.dist-info}/RECORD

@@ -8,10 +8,10 @@ airbyte_cdk/cli/airbyte_cdk/_secrets.py,sha256=jLtpkhFJHavABN3UuqddeDRGS8v1iEj0e
 airbyte_cdk/cli/airbyte_cdk/_version.py,sha256=ohZNIktLFk91sdzqFW5idaNrZAPX2dIRnz---_fcKOE,352
 airbyte_cdk/cli/airbyte_cdk/exceptions.py,sha256=bsGmlWN6cXL2jCD1WYAZMqFmK1OLg2xLrcC_60KHSeA,803
 airbyte_cdk/cli/source_declarative_manifest/__init__.py,sha256=-0ST722Nj65bgRokzpzPkD1NBBW5CytEHFUe38cB86Q,91
-airbyte_cdk/cli/source_declarative_manifest/_run.py,sha256=9qtbjt-I_stGWzWX6yVUKO_eE-Ga7g-uTuibML9qLBs,8330
+airbyte_cdk/cli/source_declarative_manifest/_run.py,sha256=3ZXAic1MJkDyjfOGvcmmkIDY5vAqsNPp8A7Z8xjzm-A,10829
 airbyte_cdk/cli/source_declarative_manifest/spec.json,sha256=Earc1L6ngcdIr514oFQlUoOxdF4RHqtUyStSIAquXdY,554
 airbyte_cdk/config_observation.py,sha256=7SSPxtN0nXPkm4euGNcTTr1iLbwUL01jy-24V1Hzde0,3986
-airbyte_cdk/connector.py,sha256=bO23kdGRkl8XKFytOgrrWFc_VagteTHVEF6IsbizVkM,4224
+airbyte_cdk/connector.py,sha256=N6TUlrZOMjLAI85JrNAKkfyTqnO5xfBCw4oEfgjJd9o,4254
 airbyte_cdk/connector_builder/README.md,sha256=Hw3wvVewuHG9-QgsAq1jDiKuLlStDxKBz52ftyNRnBw,1665
 airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=OFTzxyfAevI3Um8fXTOLTgoCc4Sx9NzF0boqYkAATfM,6590
@@ -33,7 +33,7 @@ airbyte_cdk/destinations/vector_db_based/indexer.py,sha256=beiSi2Uu67EoTr7yQSaCJ
 airbyte_cdk/destinations/vector_db_based/test_utils.py,sha256=MkqLiOJ5QyKbV4rNiJhe-BHM7FD-ADHQ4bQGf4c5lRY,1932
 airbyte_cdk/destinations/vector_db_based/utils.py,sha256=FOyEo8Lc-fY8UyhpCivhZtIqBRyxf3cUt6anmK03fUY,1127
 airbyte_cdk/destinations/vector_db_based/writer.py,sha256=nZ00xPiohElJmYktEZZIhr0m5EDETCHGhg0Lb2S7A20,5095
-airbyte_cdk/entrypoint.py,sha256=NRJv5BNZRSUEVTmNBa9N7ih6fW5sg4DwL0nkB9kI99Y,18570
+airbyte_cdk/entrypoint.py,sha256=R2kAsAnCAI7eZCctQpMCImLhFFwo7PniJVA0e7RhJVI,19774
 airbyte_cdk/exception_handler.py,sha256=D_doVl3Dt60ASXlJsfviOCswxGyKF2q0RL6rif3fNks,2013
 airbyte_cdk/logger.py,sha256=1cURbvawbunCAV178q-XhTHcbAQZTSf07WhU7U9AXWU,3744
 airbyte_cdk/manifest_migrations/README.md,sha256=PvnbrW1gyzhlkeucd0YAOXcXVxi0xBUUynzs4DMqjDo,2942
@@ -419,9 +419,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.48.16.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.48.16.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.48.16.dist-info/METADATA,sha256=IfVXYSA9oSF6gUmrAro4nKKSm9U92aEGUfXAyVjAxNM,6344
-airbyte_cdk-6.48.16.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.48.16.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
-airbyte_cdk-6.48.16.dist-info/RECORD,,
+airbyte_cdk-6.48.16.post14.dev15122056170.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.48.16.post14.dev15122056170.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.48.16.post14.dev15122056170.dist-info/METADATA,sha256=4-aZbDzT7hC6E84o6NC0UDVyW0zanMyD7b_T-i_GH6o,6366
+airbyte_cdk-6.48.16.post14.dev15122056170.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.48.16.post14.dev15122056170.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
+airbyte_cdk-6.48.16.post14.dev15122056170.dist-info/RECORD,,