airbyte-cdk 0.72.0__py3-none-any.whl → 6.13.1.dev4106__py3-none-any.whl

unit_tests/sources/declarative/test_types.py

@@ -1,39 +0,0 @@
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-
-import pytest
-from airbyte_cdk.sources.declarative.types import StreamSlice
-
-
-@pytest.mark.parametrize(
-    "stream_slice, expected_partition",
-    [
-        pytest.param(StreamSlice(partition={},cursor_slice={}), {}, id="test_partition_with_empty_partition"),
-        pytest.param(StreamSlice(partition=StreamSlice(partition={}, cursor_slice={}), cursor_slice={}), {}, id="test_partition_nested_empty"),
-        pytest.param(StreamSlice(partition={"key": "value"}, cursor_slice={}), {"key": "value"}, id="test_partition_with_mapping_partition"),
-        pytest.param(StreamSlice(partition={},cursor_slice={"cursor": "value"}), {}, id="test_partition_with_only_cursor"),
-        pytest.param(StreamSlice(partition=StreamSlice(partition={}, cursor_slice={}), cursor_slice={"cursor": "value"}), {}, id="test_partition_nested_empty_and_cursor_value_mapping"),
-        pytest.param(StreamSlice(partition=StreamSlice(partition={}, cursor_slice={"cursor": "value"}), cursor_slice={}), {}, id="test_partition_nested_empty_and_cursor_value"),
-    ]
-)
-def test_partition(stream_slice, expected_partition):
-    partition = stream_slice.partition
-
-    assert partition == expected_partition
-
-
-@pytest.mark.parametrize(
-    "stream_slice, expected_cursor_slice",
-    [
-        pytest.param(StreamSlice(partition={},cursor_slice={}), {}, id="test_cursor_slice_with_empty_cursor"),
-        pytest.param(StreamSlice(partition={}, cursor_slice=StreamSlice(partition={}, cursor_slice={})), {}, id="test_cursor_slice_nested_empty"),
-
-        pytest.param(StreamSlice(partition={}, cursor_slice={"key": "value"}), {"key": "value"}, id="test_cursor_slice_with_mapping_cursor_slice"),
-        pytest.param(StreamSlice(partition={"partition": "value"}, cursor_slice={}), {}, id="test_cursor_slice_with_only_partition"),
-        pytest.param(StreamSlice(partition={"partition": "value"}, cursor_slice=StreamSlice(partition={}, cursor_slice={})), {}, id="test_cursor_slice_nested_empty_and_partition_mapping"),
-        pytest.param(StreamSlice(partition=StreamSlice(partition={"partition": "value"}, cursor_slice={}), cursor_slice={}), {}, id="test_cursor_slice_nested_empty_and_partition"),
-    ]
-)
-def test_cursor_slice(stream_slice, expected_cursor_slice):
-    cursor_slice = stream_slice.cursor_slice
-
-    assert cursor_slice == expected_cursor_slice

unit_tests/sources/declarative/test_yaml_declarative_source.py

@@ -1,148 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import logging
-import os
-import tempfile
-
-import pytest
-from airbyte_cdk.sources.declarative.parsers.custom_exceptions import UndefinedReferenceException
-from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
-from yaml.parser import ParserError
-
-logger = logging.getLogger("airbyte")
-
-
-EXTERNAL_CONNECTION_SPECIFICATION = {
-    "type": "object",
-    "required": ["api_token"],
-    "additionalProperties": False,
-    "properties": {"api_token": {"type": "string"}},
-}
-
-
-class MockYamlDeclarativeSource(YamlDeclarativeSource):
-    """
-    Mock test class that is needed to monkey patch how we read from various files that make up a declarative source because of how our
-    tests write configuration files during testing. It is also used to properly namespace where files get written in specific
-    cases like when we temporarily write files like spec.yaml to the package unit_tests, which is the directory where it will
-    be read in during the tests.
-    """
-
-    def _read_and_parse_yaml_file(self, path_to_yaml_file):
-        """
-        We override the default behavior because we use tempfile to write the yaml manifest to a temporary directory which is
-        not mounted during runtime which prevents pkgutil.get_data() from being able to find the yaml file needed to generate
-        # the declarative source. For tests we use open() which supports using an absolute path.
-        """
-        with open(path_to_yaml_file, "r") as f:
-            config_content = f.read()
-            parsed_config = YamlDeclarativeSource._parse(config_content)
-            return parsed_config
-
-
-class TestYamlDeclarativeSource:
-    def test_source_is_created_if_toplevel_fields_are_known(self):
-        content = """
-        version: "0.29.3"
-        definitions:
-          schema_loader:
-            name: "{{ parameters.stream_name }}"
-            file_path: "./source_sendgrid/schemas/{{ parameters.name }}.yaml"
-          retriever:
-            paginator:
-              type: "DefaultPaginator"
-              page_size: 10
-              page_size_option:
-                inject_into: request_parameter
-                field_name: page_size
-              page_token_option:
-                type: RequestPath
-              pagination_strategy:
-                type: "CursorPagination"
-                cursor_value: "{{ response._metadata.next }}"
-            requester:
-              url_base: "https://api.sendgrid.com"
-              path: "/v3/marketing/lists"
-              authenticator:
-                type: "BearerAuthenticator"
-                api_token: "{{ config.apikey }}"
-              request_parameters:
-                page_size: "{{ 10 }}"
-            record_selector:
-              extractor:
-                field_path: ["result"]
-        streams:
-          - type: DeclarativeStream
-            $parameters:
-              name: "lists"
-              primary_key: id
-            schema_loader: "#/definitions/schema_loader"
-            retriever: "#/definitions/retriever"
-        check:
-          type: CheckStream
-          stream_names: ["lists"]
-        """
-        temporary_file = TestFileContent(content)
-        MockYamlDeclarativeSource(temporary_file.filename)
-
-    def test_source_fails_for_invalid_yaml(self):
-        content = """
-        version: "version"
-        definitions:
-          this is not parsable yaml: " at all
-        streams:
-          - type: DeclarativeStream
-            $parameters:
-              name: "lists"
-              primary_key: id
-              url_base: "https://api.sendgrid.com"
-        check:
-          type: CheckStream
-          stream_names: ["lists"]
-        """
-        temporary_file = TestFileContent(content)
-        with pytest.raises(ParserError):
-            MockYamlDeclarativeSource(temporary_file.filename)
-
-    def test_source_with_missing_reference_fails(self):
-        content = """
-        version: "version"
-        definitions:
-          schema_loader:
-            name: "{{ parameters.stream_name }}"
-            file_path: "./source_sendgrid/schemas/{{ parameters.name }}.yaml"
-        streams:
-          - type: DeclarativeStream
-            $parameters:
-              name: "lists"
-              primary_key: id
-              url_base: "https://api.sendgrid.com"
-            schema_loader: "#/definitions/schema_loader"
-            retriever: "#/definitions/retriever"
-        check:
-          type: CheckStream
-          stream_names: ["lists"]
-        """
-        temporary_file = TestFileContent(content)
-        with pytest.raises(UndefinedReferenceException):
-            MockYamlDeclarativeSource(temporary_file.filename)
-
-
-class TestFileContent:
-    def __init__(self, content):
-        self.file = tempfile.NamedTemporaryFile(mode="w", delete=False)
-
-        with self.file as f:
-            f.write(content)
-
-    @property
-    def filename(self):
-        return self.file.name
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, type, value, traceback):
-        os.unlink(self.filename)

unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py

@@ -1,100 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import unittest
-from datetime import datetime
-from unittest.mock import Mock, PropertyMock
-
-from airbyte_cdk.sources.file_based.availability_strategy.default_file_based_availability_strategy import (
-    DefaultFileBasedAvailabilityStrategy,
-)
-from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
-from airbyte_cdk.sources.file_based.config.jsonl_format import JsonlFormat
-from airbyte_cdk.sources.file_based.exceptions import CustomFileBasedException
-from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
-from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
-from airbyte_cdk.sources.file_based.remote_file import RemoteFile
-from airbyte_cdk.sources.file_based.stream import AbstractFileBasedStream
-
-_FILE_WITH_UNKNOWN_EXTENSION = RemoteFile(uri="a.unknown_extension", last_modified=datetime.now(), file_type="csv")
-_ANY_CONFIG = FileBasedStreamConfig(
-    name="config.name",
-    file_type="parquet",
-    format=JsonlFormat(),
-)
-_ANY_SCHEMA = {"key": "value"}
-
-
-class DefaultFileBasedAvailabilityStrategyTest(unittest.TestCase):
-    def setUp(self) -> None:
-        self._stream_reader = Mock(spec=AbstractFileBasedStreamReader)
-        self._strategy = DefaultFileBasedAvailabilityStrategy(self._stream_reader)
-
-        self._parser = Mock(spec=FileTypeParser)
-        self._parser.check_config.return_value = (True, None)
-        self._stream = Mock(spec=AbstractFileBasedStream)
-        self._stream.get_parser.return_value = self._parser
-        self._stream.catalog_schema = _ANY_SCHEMA
-        self._stream.config = _ANY_CONFIG
-        self._stream.validation_policy = PropertyMock(validate_schema_before_sync=False)
-        self._stream.stream_reader = self._stream_reader
-
-    def test_given_file_extension_does_not_match_when_check_availability_and_parsability_then_stream_is_still_available(self) -> None:
-        """
-        Before, we had a validation on the file extension but it turns out that in production, users sometimes have mismatch there. The
-        example we've seen was for JSONL parser but the file extension was just `.json`. Note that there we more than one record extracted
-        from this stream so it's not just that the file is one JSON object
-        """
-        self._stream.get_files.return_value = [_FILE_WITH_UNKNOWN_EXTENSION]
-        self._parser.parse_records.return_value = [{"a record": 1}]
-
-        is_available, reason = self._strategy.check_availability_and_parsability(self._stream, Mock(), Mock())
-
-        assert is_available
-
-    def test_not_available_given_no_files(self) -> None:
-        """
-        If no files are returned, then the stream is not available.
-        """
-        self._stream.get_files.return_value = []
-
-        is_available, reason = self._strategy.check_availability_and_parsability(self._stream, Mock(), Mock())
-
-        assert not is_available
-        assert "No files were identified in the stream" in reason
-
-    def test_parse_records_is_not_called_with_parser_max_n_files_for_parsability_set(self) -> None:
-        """
-        If the stream parser sets parser_max_n_files_for_parsability to 0, then we should not call parse_records on it
-        """
-        self._parser.parser_max_n_files_for_parsability = 0
-        self._stream.get_files.return_value = [_FILE_WITH_UNKNOWN_EXTENSION]
-
-        is_available, reason = self._strategy.check_availability_and_parsability(self._stream, Mock(), Mock())
-
-        assert is_available
-        assert not self._parser.parse_records.called
-        assert self._stream_reader.open_file.called
-
-    def test_passing_config_check(self) -> None:
-        """
-        Test if the DefaultFileBasedAvailabilityStrategy correctly handles the check_config method defined on the parser.
-        """
-        self._parser.check_config.return_value = (False, "Ran into error")
-        is_available, error_message = self._strategy.check_availability_and_parsability(self._stream, Mock(), Mock())
-        assert not is_available
-        assert "Ran into error" in error_message
-
-    def test_catching_and_raising_custom_file_based_exception(self) -> None:
-        """
-        Test if the DefaultFileBasedAvailabilityStrategy correctly handles the CustomFileBasedException
-        by raising a CheckAvailabilityError when the get_files method is called.
-        """
-        # Mock the get_files method to raise CustomFileBasedException when called
-        self._stream.get_files.side_effect = CustomFileBasedException("Custom exception for testing.")
-
-        # Invoke the check_availability_and_parsability method and check if it correctly handles the exception
-        is_available, error_message = self._strategy.check_availability_and_parsability(self._stream, Mock(), Mock())
-        assert not is_available
-        assert "Custom exception for testing." in error_message

unit_tests/sources/file_based/config/test_abstract_file_based_spec.py

@@ -1,28 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from typing import Type
-
-import pytest
-from airbyte_cdk.sources.file_based.config.file_based_stream_config import AvroFormat, CsvFormat, ParquetFormat
-from jsonschema import ValidationError, validate
-from pydantic import BaseModel
-
-
-@pytest.mark.parametrize(
-    "file_format, file_type, expected_error",
-    [
-        pytest.param(ParquetFormat, "parquet", None, id="test_parquet_format_is_a_valid_parquet_file_type"),
-        pytest.param(AvroFormat, "avro", None, id="test_avro_format_is_a_valid_avro_file_type"),
-        pytest.param(CsvFormat, "parquet", ValidationError, id="test_csv_format_is_not_a_valid_parquet_file_type"),
-    ],
-)
-def test_parquet_file_type_is_not_a_valid_csv_file_type(file_format: BaseModel, file_type: str, expected_error: Type[Exception]) -> None:
-    format_config = {file_type: {"filetype": file_type, "decimal_as_float": True}}
-
-    if expected_error:
-        with pytest.raises(expected_error):
-            validate(instance=format_config[file_type], schema=file_format.schema())
-    else:
-        validate(instance=format_config[file_type], schema=file_format.schema())

unit_tests/sources/file_based/config/test_csv_format.py

@@ -1,34 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import unittest
-
-import pytest
-from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat, CsvHeaderAutogenerated, CsvHeaderFromCsv, CsvHeaderUserProvided
-from pydantic import ValidationError
-
-
-class CsvHeaderDefinitionTest(unittest.TestCase):
-    def test_given_user_provided_and_not_column_names_provided_then_raise_exception(self) -> None:
-        with pytest.raises(ValidationError):
-            CsvHeaderUserProvided(column_names=[])
-
-    def test_given_user_provided_and_column_names_then_config_is_valid(self) -> None:
-        # no error means that this test succeeds
-        CsvHeaderUserProvided(column_names=["1", "2", "3"])
-
-    def test_given_user_provided_then_csv_does_not_have_header_row(self) -> None:
-        assert not CsvHeaderUserProvided(column_names=["1", "2", "3"]).has_header_row()
-
-    def test_given_autogenerated_then_csv_does_not_have_header_row(self) -> None:
-        assert not CsvHeaderAutogenerated().has_header_row()
-
-    def test_given_from_csv_then_csv_has_header_row(self) -> None:
-        assert CsvHeaderFromCsv().has_header_row()
-
-
-class CsvDelimiterTest(unittest.TestCase):
-    def test_tab_delimter(self):
-        assert CsvFormat(delimiter=r"\t").delimiter == '\t'
-        assert len(CsvFormat(delimiter=r"\t").delimiter) == 1

unit_tests/sources/file_based/config/test_file_based_stream_config.py

@@ -1,84 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from typing import Any, Mapping, Type
-
-import pytest as pytest
-from airbyte_cdk.sources.file_based.config.file_based_stream_config import CsvFormat, FileBasedStreamConfig
-from pydantic import ValidationError
-
-
-@pytest.mark.parametrize(
-    "file_type, input_format, expected_format, expected_error",
-    [
-        pytest.param(
-            "csv",
-            {"filetype": "csv", "delimiter": "d", "quote_char": "q", "escape_char": "e", "encoding": "ascii", "double_quote": True},
-            {"filetype": "csv", "delimiter": "d", "quote_char": "q", "escape_char": "e", "encoding": "ascii", "double_quote": True},
-            None,
-            id="test_valid_format",
-        ),
-        pytest.param(
-            "csv",
-            {"filetype": "csv", "double_quote": False},
-            {"delimiter": ",", "quote_char": '"', "encoding": "utf8", "double_quote": False},
-            None,
-            id="test_default_format_values",
-        ),
-        pytest.param(
-            "csv", {"filetype": "csv", "delimiter": "nope", "double_quote": True}, None, ValidationError, id="test_invalid_delimiter"
-        ),
-        pytest.param(
-            "csv", {"filetype": "csv", "quote_char": "nope", "double_quote": True}, None, ValidationError, id="test_invalid_quote_char"
-        ),
-        pytest.param(
-            "csv", {"filetype": "csv", "escape_char": "nope", "double_quote": True}, None, ValidationError, id="test_invalid_escape_char"
-        ),
-        pytest.param(
-            "csv",
-            {"filetype": "csv", "delimiter": ",", "quote_char": '"', "encoding": "not_a_format", "double_quote": True},
-            {},
-            ValidationError,
-            id="test_invalid_encoding_type",
-        ),
-        pytest.param(
-            "invalid", {"filetype": "invalid", "double_quote": False}, {}, ValidationError, id="test_config_format_file_type_mismatch"
-        ),
-    ],
-)
-def test_csv_config(
-    file_type: str, input_format: Mapping[str, Any], expected_format: Mapping[str, Any], expected_error: Type[Exception]
-) -> None:
-    stream_config = {"name": "stream1", "file_type": file_type, "globs": ["*"], "validation_policy": "Emit Record", "format": input_format}
-
-    if expected_error:
-        with pytest.raises(expected_error):
-            FileBasedStreamConfig(**stream_config)
-    else:
-        actual_config = FileBasedStreamConfig(**stream_config)
-        if actual_config.format is not None:
-            for expected_format_field, expected_format_value in expected_format.items():
-                assert isinstance(actual_config.format, CsvFormat)
-                assert getattr(actual_config.format, expected_format_field) == expected_format_value
-        else:
-            assert False, "Expected format to be set"
-
-
-def test_invalid_validation_policy() -> None:
-    stream_config = {
-        "name": "stream1",
-        "file_type": "csv",
-        "globs": ["*"],
-        "validation_policy": "Not Valid Policy",
-        "format": {
-            "filetype": "csv",
-            "delimiter": "d",
-            "quote_char": "q",
-            "escape_char": "e",
-            "encoding": "ascii",
-            "double_quote": True,
-        },
-    }
-    with pytest.raises(ValidationError):
-        FileBasedStreamConfig(**stream_config)

unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py

@@ -1,31 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import unittest
-from unittest.mock import Mock
-
-from airbyte_cdk.sources.file_based.discovery_policy.default_discovery_policy import DefaultDiscoveryPolicy
-from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
-
-
-class DefaultDiscoveryPolicyTest(unittest.TestCase):
-    def setUp(self) -> None:
-        self._policy = DefaultDiscoveryPolicy()
-
-        self._parser = Mock(spec=FileTypeParser)
-        self._parser.parser_max_n_files_for_schema_inference = None
-
-    def test_hardcoded_schema_inference_file_limit_is_returned(self) -> None:
-        """
-        If the parser is not providing a limit, then we should use the hardcoded limit
-        """
-        assert self._policy.get_max_n_files_for_schema_inference(self._parser) == 10
-
-    def test_parser_limit_is_respected(self) -> None:
-        """
-        If the parser is providing a limit, then we should use that limit
-        """
-        self._parser.parser_max_n_files_for_schema_inference = 1
-
-        assert self._policy.get_max_n_files_for_schema_inference(self._parser) == 1