airbyte-cdk 0.51.2__tar.gz → 0.51.4__tar.gz

{airbyte-cdk-0.51.2 → airbyte-cdk-0.51.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.51.2
+Version: 0.51.4
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.51.2 → airbyte-cdk-0.51.4}/airbyte_cdk/sources/file_based/file_types/csv_parser.py

@@ -158,7 +158,7 @@ class CsvParser(FileTypeParser):
             deduped_property_types = CsvParser._pre_propcess_property_types(property_types)
         else:
             deduped_property_types = {}
-        cast_fn = CsvParser._get_cast_function(deduped_property_types, config_format, logger)
+        cast_fn = CsvParser._get_cast_function(deduped_property_types, config_format, logger, config.schemaless)
         data_generator = self._csv_reader.read_data(config, file, stream_reader, logger, self.file_read_mode)
         for row in data_generator:
             yield CsvParser._to_nullable(cast_fn(row), deduped_property_types, config_format.null_values, config_format.strings_can_be_null)
@@ -170,10 +170,10 @@ class CsvParser(FileTypeParser):
 
     @staticmethod
     def _get_cast_function(
-        deduped_property_types: Mapping[str, str], config_format: CsvFormat, logger: logging.Logger
+        deduped_property_types: Mapping[str, str], config_format: CsvFormat, logger: logging.Logger, schemaless: bool
     ) -> Callable[[Mapping[str, str]], Mapping[str, str]]:
         # Only cast values if the schema is provided
-        if deduped_property_types:
+        if deduped_property_types and not schemaless:
             return partial(CsvParser._cast_types, deduped_property_types=deduped_property_types, config_format=config_format, logger=logger)
         else:
             # If no schema is provided, yield the rows as they are
@@ -275,11 +275,10 @@ class CsvParser(FileTypeParser):
                     except ValueError:
                         warnings.append(_format_warning(key, value, prop_type))
 
+                result[key] = cast_value
             else:
                 warnings.append(_format_warning(key, value, prop_type))
 
-            result[key] = cast_value
-
         if warnings:
             logger.warning(
                 f"{FileBasedSourceError.ERROR_CASTING_VALUE.value}: {','.join([w for w in warnings])}",

{airbyte-cdk-0.51.2 → airbyte-cdk-0.51.4}/airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -65,7 +65,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         slices = [{"files": list(group[1])} for group in itertools.groupby(sorted_files_to_read, lambda f: f.last_modified)]
         return slices
 
-    def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[Mapping[str, Any]]:
+    def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[AirbyteMessage]:
         """
         Yield all records from all remote files in `list_files_for_this_sync`.
 
@@ -127,9 +127,8 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
                         stack_trace=traceback.format_exc(),
                     ),
                 )
-                break
 
-            else:
+            finally:
                 if n_skipped:
                     yield AirbyteMessage(
                         type=MessageType.LOG,

{airbyte-cdk-0.51.2 → airbyte-cdk-0.51.4}/airbyte_cdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.51.2
+Version: 0.51.4
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.51.2 → airbyte-cdk-0.51.4}/setup.py

@@ -21,7 +21,7 @@ setup(
     name="airbyte-cdk",
     # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be
     # updated if our semver format changes such as using release candidate versions.
-    version="0.51.2",
+    version="0.51.4",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

{airbyte-cdk-0.51.2 → airbyte-cdk-0.51.4}/unit_tests/sources/file_based/file_types/test_csv_parser.py

@@ -100,7 +100,7 @@ logger = logging.getLogger()
         pytest.param(
             {"col9": "['a', 'b']"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {"col9": "['a', 'b']"}, id="cannot-cast-to-list-of-objects"
         ),
-        pytest.param({"col11": "x"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {"col11": "x"}, id="item-not-in-props-doesn't-error"),
+        pytest.param({"col11": "x"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {}, id="item-not-in-props-doesn't-error"),
     ],
 )
 def test_cast_to_python_type(row: Dict[str, str], true_values: Set[str], false_values: Set[str], expected_output: Dict[str, Any]) -> None:

{airbyte-cdk-0.51.2 → airbyte-cdk-0.51.4}/unit_tests/sources/file_based/scenarios/csv_scenarios.py

@@ -481,7 +481,6 @@ multi_csv_stream_n_file_exceeds_limit_for_inference = (
                 "data": {
                     "col1": "val11b",
                     "col2": "val12b",
-                    "col3": "val13b",
                     "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
                     "_ab_source_file_url": "b.csv",
                 },
@@ -491,7 +490,6 @@ multi_csv_stream_n_file_exceeds_limit_for_inference = (
                 "data": {
                     "col1": "val21b",
                     "col2": "val22b",
-                    "col3": "val23b",
                     "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
                     "_ab_source_file_url": "b.csv",
                 },

airbyte-cdk-0.51.4/unit_tests/sources/file_based/stream/test_default_file_based_stream.py

@@ -0,0 +1,130 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import unittest
+from datetime import datetime, timezone
+from typing import Any, Iterable, Iterator, Mapping
+from unittest.mock import Mock
+
+import pytest
+from airbyte_cdk.models import Level
+from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy
+from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy
+from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
+from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
+from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy
+from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
+from airbyte_cdk.sources.file_based.stream.default_file_based_stream import DefaultFileBasedStream
+
+
+@pytest.mark.parametrize(
+    "input_schema, expected_output",
+    [
+        pytest.param({}, {}, id="empty-schema"),
+        pytest.param(
+            {"type": "string"},
+            {"type": ["null", "string"]},
+            id="simple-schema",
+        ),
+        pytest.param(
+            {"type": ["string"]},
+            {"type": ["null", "string"]},
+            id="simple-schema-list-type",
+        ),
+        pytest.param(
+            {"type": ["null", "string"]},
+            {"type": ["null", "string"]},
+            id="simple-schema-already-has-null",
+        ),
+        pytest.param(
+            {"properties": {"type": "string"}},
+            {"properties": {"type": ["null", "string"]}},
+            id="nested-schema",
+        ),
+        pytest.param(
+            {"items": {"type": "string"}},
+            {"items": {"type": ["null", "string"]}},
+            id="array-schema",
+        ),
+        pytest.param(
+            {"type": "object", "properties": {"prop": {"type": "string"}}},
+            {"type": ["null", "object"], "properties": {"prop": {"type": ["null", "string"]}}},
+            id="deeply-nested-schema",
+        ),
+    ],
+)
+def test_fill_nulls(input_schema: Mapping[str, Any], expected_output: Mapping[str, Any]) -> None:
+    assert DefaultFileBasedStream._fill_nulls(input_schema) == expected_output
+
+
+class DefaultFileBasedStreamTest(unittest.TestCase):
+    _FILE_TYPE = "file_type"
+    _NOW = datetime(2022, 10, 22, tzinfo=timezone.utc)
+    _A_RECORD = {"a_record": 1}
+
+    def setUp(self) -> None:
+        self._stream_config = Mock()
+        self._stream_config.file_type = self._FILE_TYPE
+        self._stream_config.name = "a stream name"
+        self._catalog_schema = Mock()
+        self._stream_reader = Mock(spec=AbstractFileBasedStreamReader)
+        self._availability_strategy = Mock(spec=AbstractFileBasedAvailabilityStrategy)
+        self._discovery_policy = Mock(spec=AbstractDiscoveryPolicy)
+        self._parser = Mock(spec=FileTypeParser)
+        self._validation_policy = Mock(spec=AbstractSchemaValidationPolicy)
+        self._validation_policy.name = "validation policy name"
+        self._cursor = Mock(spec=AbstractFileBasedCursor)
+
+        self._stream = DefaultFileBasedStream(
+            config=self._stream_config,
+            catalog_schema=self._catalog_schema,
+            stream_reader=self._stream_reader,
+            availability_strategy=self._availability_strategy,
+            discovery_policy=self._discovery_policy,
+            parsers={self._FILE_TYPE: self._parser},
+            validation_policy=self._validation_policy,
+            cursor=self._cursor,
+        )
+
+    def test_when_read_records_from_slice_then_return_records(self) -> None:
+        self._parser.parse_records.return_value = [self._A_RECORD]
+        messages = list(self._stream.read_records_from_slice({"files": [RemoteFile(uri="uri", last_modified=self._NOW)]}))
+        assert list(map(lambda message: message.record.data["data"], messages)) == [self._A_RECORD]
+
+    def test_given_exception_when_read_records_from_slice_then_do_process_other_files(self) -> None:
+        """
+        The current behavior for source-s3 v3 does not fail sync on some errors and hence, we will keep this behaviour for now. One example
+        we can easily reproduce this is by having a file with gzip extension that is not actually a gzip file. The reader will fail to open
+        the file but the sync won't fail.
+        Ticket: https://github.com/airbytehq/airbyte/issues/29680
+        """
+        self._parser.parse_records.side_effect = [ValueError("An error"), [self._A_RECORD]]
+
+        messages = list(self._stream.read_records_from_slice({"files": [
+            RemoteFile(uri="invalid_file", last_modified=self._NOW),
+            RemoteFile(uri="valid_file", last_modified=self._NOW),
+        ]}))
+
+        assert messages[0].log.level == Level.ERROR
+        assert messages[1].record.data["data"] == self._A_RECORD
+
+    def test_given_exception_after_skipping_records_when_read_records_from_slice_then_send_warning(self) -> None:
+        self._stream_config.schemaless = False
+        self._validation_policy.record_passes_validation_policy.return_value = False
+        self._parser.parse_records.side_effect = [self._iter([self._A_RECORD, ValueError("An error")])]
+
+        messages = list(self._stream.read_records_from_slice({"files": [
+            RemoteFile(uri="invalid_file", last_modified=self._NOW),
+            RemoteFile(uri="valid_file", last_modified=self._NOW),
+        ]}))
+
+        assert messages[0].log.level == Level.ERROR
+        assert messages[1].log.level == Level.WARN
+
+    def _iter(self, x: Iterable[Any]) -> Iterator[Any]:
+        for item in x:
+            if isinstance(item, Exception):
+                raise item
+            yield item

airbyte-cdk-0.51.2/unit_tests/sources/file_based/stream/test_default_file_based_stream.py

@@ -1,48 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from typing import Any, Mapping
-
-import pytest
-from airbyte_cdk.sources.file_based.stream.default_file_based_stream import DefaultFileBasedStream
-
-
-@pytest.mark.parametrize(
-    "input_schema, expected_output",
-    [
-        pytest.param({}, {}, id="empty-schema"),
-        pytest.param(
-            {"type": "string"},
-            {"type": ["null", "string"]},
-            id="simple-schema",
-        ),
-        pytest.param(
-            {"type": ["string"]},
-            {"type": ["null", "string"]},
-            id="simple-schema-list-type",
-        ),
-        pytest.param(
-            {"type": ["null", "string"]},
-            {"type": ["null", "string"]},
-            id="simple-schema-already-has-null",
-        ),
-        pytest.param(
-            {"properties": {"type": "string"}},
-            {"properties": {"type": ["null", "string"]}},
-            id="nested-schema",
-        ),
-        pytest.param(
-            {"items": {"type": "string"}},
-            {"items": {"type": ["null", "string"]}},
-            id="array-schema",
-        ),
-        pytest.param(
-            {"type": "object", "properties": {"prop": {"type": "string"}}},
-            {"type": ["null", "object"], "properties": {"prop": {"type": ["null", "string"]}}},
-            id="deeply-nested-schema",
-        ),
-    ],
-)
-def test_fill_nulls(input_schema: Mapping[str, Any], expected_output: Mapping[str, Any]) -> None:
-    assert DefaultFileBasedStream._fill_nulls(input_schema) == expected_output