airbyte-cdk 0.55.3__tar.gz → 0.55.5__tar.gz

{airbyte-cdk-0.55.3 → airbyte-cdk-0.55.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.55.3
+Version: 0.55.5
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.55.3 → airbyte-cdk-0.55.5}/airbyte_cdk/entrypoint.py

@@ -106,6 +106,9 @@ class AirbyteEntrypoint(object):
                     raw_config = self.source.read_config(parsed_args.config)
                     config = self.source.configure(raw_config, temp_dir)
 
+                    yield from [
+                        self.airbyte_message_to_string(queued_message) for queued_message in self._emit_queued_messages(self.source)
+                    ]
                     if cmd == "check":
                         yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.check(source_spec, config))
                     elif cmd == "discover":

{airbyte-cdk-0.55.3 → airbyte-cdk-0.55.5}/airbyte_cdk/sources/abstract_source.py

@@ -19,7 +19,7 @@ from airbyte_cdk.models import (
 )
 from airbyte_cdk.models import Type as MessageType
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
-from airbyte_cdk.sources.message import MessageRepository
+from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository
 from airbyte_cdk.sources.source import Source
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.core import StreamData
@@ -31,6 +31,8 @@ from airbyte_cdk.utils.event_timing import create_timer
 from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
+_default_message_repository = InMemoryMessageRepository()
+
 
 class AbstractSource(Source, ABC):
     """
@@ -269,4 +271,4 @@ class AbstractSource(Source, ABC):
 
     @property
     def message_repository(self) -> Union[None, MessageRepository]:
-        return None
+        return _default_message_repository

{airbyte-cdk-0.55.3 → airbyte-cdk-0.55.5}/airbyte_cdk/sources/file_based/config/file_based_stream_config.py

@@ -46,7 +46,9 @@ class FileBasedStreamConfig(BaseModel):
         description="The schema that will be used to validate records extracted from the file. This will override the stream schema that is auto-detected from incoming files.",
     )
     primary_key: Optional[str] = Field(
-        title="Primary Key", description="The column or columns (for a composite key) that serves as the unique identifier of a record."
+        title="Primary Key",
+        description="The column or columns (for a composite key) that serves as the unique identifier of a record.",
+        airbyte_hidden=True,  # Users can create/modify primary keys in the connection configuration so we shouldn't duplicate it here.
     )
     days_to_sync_if_history_is_full: int = Field(
         title="Days To Sync If History Is Full",

airbyte-cdk-0.55.5/airbyte_cdk/test/entrypoint_wrapper.py

@@ -0,0 +1,116 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+"""
+The AirbyteEntrypoint is important because it is a service layer that orchestrate how we execute commands from the
+[common interface](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#common-interface) through the source Python
+implementation. There is some logic about which message we send to the platform and when which is relevant for integration testing. Other
+than that, there are integrations point that are annoying to integrate with using Python code:
+* Sources communicate with the platform using stdout. The implication is that the source could just print every message instead of
+    returning things to source.<method> or to using the message repository. WARNING: As part of integration testing, we will not support
+    messages that are simply printed. The reason is that capturing stdout relies on overriding sys.stdout (see
+    https://docs.python.org/3/library/contextlib.html#contextlib.redirect_stdout) which clashes with how pytest captures logs and brings
+    considerations for multithreaded applications. If code you work with uses `print` statements, please migrate to
+    source.message_repository to emit those messages
+* The entrypoint interface relies on file being written on the file system
+"""
+
+import json
+import logging
+import tempfile
+from io import StringIO
+from pathlib import Path
+from typing import Any, List, Mapping, Optional, Union
+
+from airbyte_cdk.entrypoint import AirbyteEntrypoint
+from airbyte_cdk.logger import AirbyteLogFormatter
+from airbyte_cdk.sources import Source
+from airbyte_protocol.models import AirbyteLogMessage, AirbyteMessage, ConfiguredAirbyteCatalog, Level, TraceType, Type
+from pydantic.error_wrappers import ValidationError
+
+
+class EntrypointOutput:
+    def __init__(self, messages: List[str]):
+        try:
+            self._messages = [self._parse_message(message) for message in messages]
+        except ValidationError as exception:
+            raise ValueError("All messages are expected to be AirbyteMessage") from exception
+
+    @staticmethod
+    def _parse_message(message: str) -> AirbyteMessage:
+        try:
+            return AirbyteMessage.parse_obj(json.loads(message))
+        except (json.JSONDecodeError, ValidationError):
+            # The platform assumes that logs that are not of AirbyteMessage format are log messages
+            return AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message=message))
+
+    @property
+    def records_and_state_messages(self) -> List[AirbyteMessage]:
+        return self._get_message_by_types([Type.RECORD, Type.STATE])
+
+    @property
+    def records(self) -> List[AirbyteMessage]:
+        return self._get_message_by_types([Type.RECORD])
+
+    @property
+    def state_messages(self) -> List[AirbyteMessage]:
+        return self._get_message_by_types([Type.STATE])
+
+    @property
+    def logs(self) -> List[AirbyteMessage]:
+        return self._get_message_by_types([Type.LOG])
+
+    @property
+    def trace_messages(self) -> List[AirbyteMessage]:
+        return self._get_message_by_types([Type.TRACE])
+
+    @property
+    def analytics_messages(self) -> List[AirbyteMessage]:
+        return [message for message in self._get_message_by_types([Type.TRACE]) if message.trace.type == TraceType.ANALYTICS]
+
+    def _get_message_by_types(self, message_types: List[Type]) -> List[AirbyteMessage]:
+        return [message for message in self._messages if message.type in message_types]
+
+
+def read(source: Source, config: Mapping[str, Any], catalog: ConfiguredAirbyteCatalog, state: Optional[Any] = None) -> EntrypointOutput:
+    """
+    config and state must be json serializable
+    """
+    log_capture_buffer = StringIO()
+    stream_handler = logging.StreamHandler(log_capture_buffer)
+    stream_handler.setLevel(logging.INFO)
+    stream_handler.setFormatter(AirbyteLogFormatter())
+    parent_logger = logging.getLogger("")
+    parent_logger.addHandler(stream_handler)
+
+    with tempfile.TemporaryDirectory() as tmp_directory:
+        tmp_directory_path = Path(tmp_directory)
+        args = [
+            "read",
+            "--config",
+            make_file(tmp_directory_path / "config.json", config),
+            "--catalog",
+            make_file(tmp_directory_path / "catalog.json", catalog.json()),
+        ]
+        if state:
+            args.extend(
+                [
+                    "--state",
+                    make_file(tmp_directory_path / "state.json", state),
+                ]
+            )
+        source_entrypoint = AirbyteEntrypoint(source)
+        parsed_args = source_entrypoint.parse_args(args)
+        messages = list(source_entrypoint.run(parsed_args))
+        captured_logs = log_capture_buffer.getvalue().split("\n")[:-1]
+
+        parent_logger.removeHandler(stream_handler)
+
+        return EntrypointOutput(messages + captured_logs)
+
+
+def make_file(path: Path, file_contents: Optional[Union[str, Mapping[str, Any], List[Mapping[str, Any]]]]) -> str:
+    if isinstance(file_contents, str):
+        path.write_text(file_contents)
+    else:
+        path.write_text(json.dumps(file_contents))
+    return str(path)

{airbyte-cdk-0.55.3 → airbyte-cdk-0.55.5}/airbyte_cdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.55.3
+Version: 0.55.5
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.55.3 → airbyte-cdk-0.55.5}/airbyte_cdk.egg-info/SOURCES.txt

@@ -246,6 +246,7 @@ airbyte_cdk/sources/utils/slice_logger.py
 airbyte_cdk/sources/utils/transform.py
 airbyte_cdk/sources/utils/types.py
 airbyte_cdk/test/__init__.py
+airbyte_cdk/test/entrypoint_wrapper.py
 airbyte_cdk/test/http/__init__.py
 airbyte_cdk/test/http/matcher.py
 airbyte_cdk/test/http/mocker.py

{airbyte-cdk-0.55.3 → airbyte-cdk-0.55.5}/setup.py

@@ -36,7 +36,7 @@ setup(
     name="airbyte-cdk",
     # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be
     # updated if our semver format changes such as using release candidate versions.
-    version="0.55.3",
+    version="0.55.5",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

{airbyte-cdk-0.55.3 → airbyte-cdk-0.55.5}/unit_tests/sources/file_based/scenarios/csv_scenarios.py

@@ -100,6 +100,7 @@ single_csv_scenario: TestScenario[InMemoryFilesSource] = (
                                     "title": "Primary Key",
                                     "description": "The column or columns (for a composite key) that serves as the unique identifier of a record.",
                                     "type": "string",
+                                    "airbyte_hidden": True,
                                 },
                                 "days_to_sync_if_history_is_full": {
                                     "title": "Days To Sync If History Is Full",

{airbyte-cdk-0.55.3 → airbyte-cdk-0.55.5}/unit_tests/sources/file_based/scenarios/scenario_builder.py

@@ -81,9 +81,13 @@ class TestScenario(Generic[SourceType]):
         for stream in self.source.streams(self.config):
             catalog["streams"].append(
                 {
-                    "stream": stream.name,
+                    "stream": {
+                        "name": stream.name,
+                        "json_schema": {},
+                        "supported_sync_modes": [sync_mode.value],
+                    },
                     "sync_mode": sync_mode.value,
-                    "destination_sync_mode": "append",
+                    "destination_sync_mode": "append"
                 }
             )
 

{airbyte-cdk-0.55.3 → airbyte-cdk-0.55.5}/unit_tests/sources/file_based/test_file_based_scenarios.py

@@ -8,7 +8,6 @@ import pytest
 from _pytest.capture import CaptureFixture
 from airbyte_cdk.sources.abstract_source import AbstractSource
 from freezegun import freeze_time
-from pytest import LogCaptureFixture
 from unit_tests.sources.file_based.scenarios.avro_scenarios import (
     avro_all_types_scenario,
     avro_file_with_double_as_number_scenario,
@@ -248,10 +247,8 @@ def test_file_based_discover(capsys: CaptureFixture[str], tmp_path: PosixPath, s
 
 @pytest.mark.parametrize("scenario", read_scenarios, ids=[s.name for s in read_scenarios])
 @freeze_time("2023-06-09T00:00:00Z")
-def test_file_based_read(
-    capsys: CaptureFixture[str], caplog: LogCaptureFixture, tmp_path: PosixPath, scenario: TestScenario[AbstractSource]
-) -> None:
-    verify_read(capsys, caplog, tmp_path, scenario)
+def test_file_based_read(scenario: TestScenario[AbstractSource]) -> None:
+    verify_read(scenario)
 
 
 @pytest.mark.parametrize("scenario", spec_scenarios, ids=[c.name for c in spec_scenarios])

{airbyte-cdk-0.55.3 → airbyte-cdk-0.55.5}/unit_tests/sources/file_based/test_scenarios.py

@@ -11,11 +11,12 @@ import pytest
 from _pytest.capture import CaptureFixture
 from _pytest.reports import ExceptionInfo
 from airbyte_cdk.entrypoint import launch
-from airbyte_cdk.logger import AirbyteLogFormatter
 from airbyte_cdk.models import AirbyteAnalyticsTraceMessage, SyncMode
 from airbyte_cdk.sources import AbstractSource
+from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput
+from airbyte_cdk.test.entrypoint_wrapper import read as entrypoint_read
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
-from pytest import LogCaptureFixture
+from airbyte_protocol.models import AirbyteLogMessage, AirbyteMessage, ConfiguredAirbyteCatalog
 from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenario
 
 
@@ -37,58 +38,51 @@ def verify_discover(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario:
             _verify_expected_logs(logs, discover_logs)
 
 
-def verify_read(
-    capsys: CaptureFixture[str], caplog: LogCaptureFixture, tmp_path: PosixPath, scenario: TestScenario[AbstractSource]
-) -> None:
-    caplog.handler.setFormatter(AirbyteLogFormatter())
+def verify_read(scenario: TestScenario[AbstractSource]) -> None:
     if scenario.incremental_scenario_config:
-        run_test_read_incremental(capsys, caplog, tmp_path, scenario)
+        run_test_read_incremental(scenario)
     else:
-        run_test_read_full_refresh(capsys, caplog, tmp_path, scenario)
+        run_test_read_full_refresh(scenario)
 
 
-def run_test_read_full_refresh(
-    capsys: CaptureFixture[str], caplog: LogCaptureFixture, tmp_path: PosixPath, scenario: TestScenario[AbstractSource]
-) -> None:
+def run_test_read_full_refresh(scenario: TestScenario[AbstractSource]) -> None:
     expected_exc, expected_msg = scenario.expected_read_error
     if expected_exc:
         with pytest.raises(expected_exc) as exc:  # noqa
-            read(capsys, caplog, tmp_path, scenario)
+            read(scenario)
         if expected_msg:
             assert expected_msg in get_error_message_from_exc(exc)
     else:
-        output = read(capsys, caplog, tmp_path, scenario)
+        output = read(scenario)
         _verify_read_output(output, scenario)
 
 
-def run_test_read_incremental(
-    capsys: CaptureFixture[str], caplog: LogCaptureFixture, tmp_path: PosixPath, scenario: TestScenario[AbstractSource]
-) -> None:
+def run_test_read_incremental(scenario: TestScenario[AbstractSource]) -> None:
     expected_exc, expected_msg = scenario.expected_read_error
     if expected_exc:
         with pytest.raises(expected_exc):
-            read_with_state(capsys, caplog, tmp_path, scenario)
+            read_with_state(scenario)
     else:
-        output = read_with_state(capsys, caplog, tmp_path, scenario)
+        output = read_with_state(scenario)
         _verify_read_output(output, scenario)
 
 
-def _verify_read_output(output: Dict[str, Any], scenario: TestScenario[AbstractSource]) -> None:
-    records, logs = output["records"], output["logs"]
-    logs = [log for log in logs if log.get("level") in scenario.log_levels]
+def _verify_read_output(output: EntrypointOutput, scenario: TestScenario[AbstractSource]) -> None:
+    records, log_messages = output.records_and_state_messages, output.logs
+    logs = [message.log for message in log_messages if message.log.level.value in scenario.log_levels]
     expected_records = scenario.expected_records
     assert len(records) == len(expected_records)
     for actual, expected in zip(records, expected_records):
-        if "record" in actual:
-            assert len(actual["record"]["data"]) == len(expected["data"])
-            for key, value in actual["record"]["data"].items():
+        if actual.record:
+            assert len(actual.record.data) == len(expected["data"])
+            for key, value in actual.record.data.items():
                 if isinstance(value, float):
                     assert math.isclose(value, expected["data"][key], abs_tol=1e-04)
                 else:
                     assert value == expected["data"][key]
-            assert actual["record"]["stream"] == expected["stream"]
-        elif "state" in actual:
-            assert actual["state"]["data"] == expected
+            assert actual.record.stream == expected["stream"]
+        elif actual.state:
+            assert actual.state.data == expected
 
     if scenario.expected_logs:
         read_logs = scenario.expected_logs.get("read")
@@ -96,25 +90,25 @@ def _verify_read_output(output: Dict[str, Any], scenario: TestScenario[AbstractS
         _verify_expected_logs(logs, read_logs)
 
     if scenario.expected_analytics:
-        analytics = output["analytics"]
+        analytics = output.analytics_messages
 
         _verify_analytics(analytics, scenario.expected_analytics)
 
 
-def _verify_analytics(analytics: List[Dict[str, Any]], expected_analytics: Optional[List[AirbyteAnalyticsTraceMessage]]) -> None:
+def _verify_analytics(analytics: List[AirbyteMessage], expected_analytics: Optional[List[AirbyteAnalyticsTraceMessage]]) -> None:
     if expected_analytics:
         for actual, expected in zip(analytics, expected_analytics):
-            actual_type, actual_value = actual["type"], actual["value"]
+            actual_type, actual_value = actual.trace.analytics.type, actual.trace.analytics.value
             expected_type = expected.type
             expected_value = expected.value
             assert actual_type == expected_type
             assert actual_value == expected_value
 
 
-def _verify_expected_logs(logs: List[Dict[str, Any]], expected_logs: Optional[List[Mapping[str, Any]]]) -> None:
+def _verify_expected_logs(logs: List[AirbyteLogMessage], expected_logs: Optional[List[Mapping[str, Any]]]) -> None:
     if expected_logs:
         for actual, expected in zip(logs, expected_logs):
-            actual_level, actual_message = actual["level"], actual["message"]
+            actual_level, actual_message = actual.level.value, actual.message
             expected_level = expected["level"]
             expected_message = expected["message"]
             assert actual_level == expected_level
@@ -172,55 +166,21 @@ def discover(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestSce
     }
 
 
-def read(
-    capsys: CaptureFixture[str], caplog: LogCaptureFixture, tmp_path: PosixPath, scenario: TestScenario[AbstractSource]
-) -> Dict[str, Any]:
-    with caplog.handler.stream as logger_stream:
-        launch(
-            scenario.source,
-            [
-                "read",
-                "--config",
-                make_file(tmp_path / "config.json", scenario.config),
-                "--catalog",
-                make_file(tmp_path / "catalog.json", scenario.configured_catalog(SyncMode.full_refresh)),
-            ],
-        )
-        captured = capsys.readouterr().out.splitlines() + logger_stream.getvalue().split("\n")[:-1]
-
-        return {
-            "records": [msg for msg in (json.loads(line) for line in captured) if msg["type"] == "RECORD"],
-            "logs": [msg["log"] for msg in (json.loads(line) for line in captured) if msg["type"] == "LOG"],
-            "analytics": [
-                msg["trace"]["analytics"]
-                for msg in (json.loads(line) for line in captured)
-                if msg["type"] == "TRACE" and msg["trace"]["type"] == "ANALYTICS"
-            ],
-        }
-
-
-def read_with_state(
-    capsys: CaptureFixture[str], caplog: LogCaptureFixture, tmp_path: PosixPath, scenario: TestScenario[AbstractSource]
-) -> Dict[str, List[Any]]:
-    launch(
+def read(scenario: TestScenario[AbstractSource]) -> EntrypointOutput:
+    return entrypoint_read(
         scenario.source,
-        [
-            "read",
-            "--config",
-            make_file(tmp_path / "config.json", scenario.config),
-            "--catalog",
-            make_file(tmp_path / "catalog.json", scenario.configured_catalog(SyncMode.incremental)),
-            "--state",
-            make_file(tmp_path / "state.json", scenario.input_state()),
-        ],
+        scenario.config,
+        ConfiguredAirbyteCatalog.parse_obj(scenario.configured_catalog(SyncMode.full_refresh)),
+    )
+
+
+def read_with_state(scenario: TestScenario[AbstractSource]) -> EntrypointOutput:
+    return entrypoint_read(
+        scenario.source,
+        scenario.config,
+        ConfiguredAirbyteCatalog.parse_obj(scenario.configured_catalog(SyncMode.incremental)),
+        scenario.input_state(),
     )
-    captured = capsys.readouterr()
-    logs = caplog.records
-    return {
-        "records": [msg for msg in (json.loads(line) for line in captured.out.splitlines()) if msg["type"] in ("RECORD", "STATE")],
-        "logs": [msg["log"] for msg in (json.loads(line) for line in captured.out.splitlines()) if msg["type"] == "LOG"]
-        + [{"level": log.levelname, "message": log.message} for log in logs],
-    }
 
 
 def make_file(path: Path, file_contents: Optional[Union[Mapping[str, Any], List[Mapping[str, Any]]]]) -> str:

{airbyte-cdk-0.55.3 → airbyte-cdk-0.55.5}/unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py

@@ -7,7 +7,6 @@ from pathlib import PosixPath
 import pytest
 from _pytest.capture import CaptureFixture
 from freezegun import freeze_time
-from pytest import LogCaptureFixture
 from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenario
 from unit_tests.sources.file_based.test_scenarios import verify_discover, verify_read
 from unit_tests.sources.streams.concurrent.scenarios.incremental_scenarios import (
@@ -68,8 +67,8 @@ scenarios = [
 
 @pytest.mark.parametrize("scenario", scenarios, ids=[s.name for s in scenarios])
 @freeze_time("2023-06-09T00:00:00Z")
-def test_concurrent_read(capsys: CaptureFixture[str], caplog: LogCaptureFixture, tmp_path: PosixPath, scenario: TestScenario) -> None:
-    verify_read(capsys, caplog, tmp_path, scenario)
+def test_concurrent_read(scenario: TestScenario) -> None:
+    verify_read(scenario)
 
 
 @pytest.mark.parametrize("scenario", scenarios, ids=[s.name for s in scenarios])