airbyte-cdk 0.51.2__py3-none-any.whl → 0.51.4__py3-none-any.whl

airbyte_cdk/sources/file_based/file_types/csv_parser.py

@@ -158,7 +158,7 @@ class CsvParser(FileTypeParser):
             deduped_property_types = CsvParser._pre_propcess_property_types(property_types)
         else:
             deduped_property_types = {}
-        cast_fn = CsvParser._get_cast_function(deduped_property_types, config_format, logger)
+        cast_fn = CsvParser._get_cast_function(deduped_property_types, config_format, logger, config.schemaless)
         data_generator = self._csv_reader.read_data(config, file, stream_reader, logger, self.file_read_mode)
         for row in data_generator:
             yield CsvParser._to_nullable(cast_fn(row), deduped_property_types, config_format.null_values, config_format.strings_can_be_null)
@@ -170,10 +170,10 @@ class CsvParser(FileTypeParser):
 
     @staticmethod
     def _get_cast_function(
-        deduped_property_types: Mapping[str, str], config_format: CsvFormat, logger: logging.Logger
+        deduped_property_types: Mapping[str, str], config_format: CsvFormat, logger: logging.Logger, schemaless: bool
     ) -> Callable[[Mapping[str, str]], Mapping[str, str]]:
         # Only cast values if the schema is provided
-        if deduped_property_types:
+        if deduped_property_types and not schemaless:
             return partial(CsvParser._cast_types, deduped_property_types=deduped_property_types, config_format=config_format, logger=logger)
         else:
             # If no schema is provided, yield the rows as they are
@@ -275,11 +275,10 @@ class CsvParser(FileTypeParser):
                     except ValueError:
                         warnings.append(_format_warning(key, value, prop_type))
 
+                result[key] = cast_value
             else:
                 warnings.append(_format_warning(key, value, prop_type))
 
-            result[key] = cast_value
-
         if warnings:
             logger.warning(
                 f"{FileBasedSourceError.ERROR_CASTING_VALUE.value}: {','.join([w for w in warnings])}",

airbyte_cdk/sources/file_based/stream/default_file_based_stream.py

@@ -65,7 +65,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
         slices = [{"files": list(group[1])} for group in itertools.groupby(sorted_files_to_read, lambda f: f.last_modified)]
         return slices
 
-    def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[Mapping[str, Any]]:
+    def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[AirbyteMessage]:
         """
         Yield all records from all remote files in `list_files_for_this_sync`.
 
@@ -127,9 +127,8 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
                         stack_trace=traceback.format_exc(),
                     ),
                 )
-                break
 
-            else:
+            finally:
                 if n_skipped:
                     yield AirbyteMessage(
                         type=MessageType.LOG,

{airbyte_cdk-0.51.2.dist-info → airbyte_cdk-0.51.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.51.2
+Version: 0.51.4
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte_cdk-0.51.2.dist-info → airbyte_cdk-0.51.4.dist-info}/RECORD

@@ -157,7 +157,7 @@ airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py,sha
 airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py,sha256=Mx3zT9Dem4uNfaUT0oOtrESsuB1LrGAi5N-uw2swZZA,701
 airbyte_cdk/sources/file_based/file_types/__init__.py,sha256=N3a8cjXwRUN2__46IJTwrWlsyFiSA1xtSgPcPH28sn0,476
 airbyte_cdk/sources/file_based/file_types/avro_parser.py,sha256=7PVaW17wn80HYW1mu074X2dy0UgFoqFqGIOKN2ZMKD0,8686
-airbyte_cdk/sources/file_based/file_types/csv_parser.py,sha256=pOUEuAgodXwDFlSw52P2cAu4elyST6-fOspeJkb7SY8,16617
+airbyte_cdk/sources/file_based/file_types/csv_parser.py,sha256=SsWy_8KunUz1MYKroix6fWv37mbPEH0h5SoW4g3Qjf4,16676
 airbyte_cdk/sources/file_based/file_types/file_type_parser.py,sha256=cThTLc1YKSAapOn70lB09SzruRIPSShGIMz1f92QYV8,1555
 airbyte_cdk/sources/file_based/file_types/jsonl_parser.py,sha256=Kz6HLF0CrFHQ1Y6rJKGr7KmBWSLeDYFQmkg0WIi7Frg,5395
 airbyte_cdk/sources/file_based/file_types/parquet_parser.py,sha256=QulQ_soGb1LpQ_KTxqWZjmfACGkTUDUOeuSmNFtcSLk,8717
@@ -166,7 +166,7 @@ airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_valida
 airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py,sha256=ZeAa0z50ywMU2chNjQ7JpL4yePU1NajhBa8FS7rXLVo,1643
 airbyte_cdk/sources/file_based/stream/__init__.py,sha256=QPDqdgjsabOQD93dSFqHGaFS_3pIwm-chEabZHiPJi0,265
 airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py,sha256=tvVew6din9y8a3hItzU0PjTQrMxbVI7bK-3pRTvOswg,5810
-airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=N4JUzXeM5T_KAU4OBlGtw9dzjOJRS_Dvn2nb7RstUyA,11861
+airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=eFYqN657J5A0sf9og_w7qea8lu2xtUobjYYDldfmbmA,11839
 airbyte_cdk/sources/file_based/stream/cursor/__init__.py,sha256=MhFB5hOo8sjwvCh8gangaymdg3EJWYt_72brFOZt068,191
 airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py,sha256=i-FPeK8lwCzX34GCcmvL5Yvdh8-uu7FeCVYDoFbD7IY,1920
 airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py,sha256=kuJRKgDYOGXRk0V0I8BpFxg0hGv7SfV_nBpmmn45F88,6815
@@ -315,13 +315,13 @@ unit_tests/sources/file_based/config/test_abstract_file_based_spec.py,sha256=wmZ
 unit_tests/sources/file_based/config/test_file_based_stream_config.py,sha256=1eMsHlMQIFwyw20HjnhgKuiw6399sMcLTQ4LP09kTT4,3060
 unit_tests/sources/file_based/file_types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/sources/file_based/file_types/test_avro_parser.py,sha256=INqwKXcgNb3h_tktNXYU6WNUD-iNwRYHCd3IrnQa5R4,11051
-unit_tests/sources/file_based/file_types/test_csv_parser.py,sha256=PIdee-wnaRNVERZV69u9LtJIxI9y_Myj9zax9UP9_0E,20315
+unit_tests/sources/file_based/file_types/test_csv_parser.py,sha256=KB4WDy3aMAZ0CmJiqFaTUOZlK4urpvG9bwcwQ-h2-VY,20303
 unit_tests/sources/file_based/file_types/test_jsonl_parser.py,sha256=foTf9U9LyAS8OR0BonwNgFWPqTrmzFV2lpPUfRMrioE,6134
 unit_tests/sources/file_based/file_types/test_parquet_parser.py,sha256=D7sKTty8aEqMDWWGKWUqDbWjTxhGkygU7ns4-_JceRY,13543
 unit_tests/sources/file_based/scenarios/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/sources/file_based/scenarios/avro_scenarios.py,sha256=xUfw0crAvSTgQ2-chJx2ZiigQyo5IfrCuOFC1TWXXsQ,29795
 unit_tests/sources/file_based/scenarios/check_scenarios.py,sha256=7DR49LCjns72Dv5-R-cg6_SUR1zpHtE9_uFEWoYwx1s,5834
-unit_tests/sources/file_based/scenarios/csv_scenarios.py,sha256=M-qxXZzYx96RYzknyInZGu7L54IKC6iuv2inpPn6LJM,95783
+unit_tests/sources/file_based/scenarios/csv_scenarios.py,sha256=UVdDblKj3R5qQkh-dj4xqZ2822GyJuymaAerWbX9HeE,95707
 unit_tests/sources/file_based/scenarios/incremental_scenarios.py,sha256=0maHng11cFmvzFLOniyBxOEYoKj4DYR3NO9-pSYoFLs,60710
 unit_tests/sources/file_based/scenarios/jsonl_scenarios.py,sha256=N83fga4gMKkbm6hYnen1Z5p5eEgjnMB_M_sXx6B96cU,27503
 unit_tests/sources/file_based/scenarios/parquet_scenarios.py,sha256=-cBO1ZwberBxNMqDOtKz8yGwm3zB7elz_st2NKHeczM,26955
@@ -330,7 +330,7 @@ unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py,sha256=sx
 unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py,sha256=GFACt5-0BMFVtrYueA1_6va2euHfhKcBLcYfDHHnmFI,26715
 unit_tests/sources/file_based/stream/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/sources/file_based/stream/test_default_file_based_cursor.py,sha256=fzggaGwtXgNk-sAjQ8D71CPTCNBVxBS6HW63FKdkKME,12491
-unit_tests/sources/file_based/stream/test_default_file_based_stream.py,sha256=22Rgp1HvZss2WdOcfenbnvx18tfFJ_trPuWp299RW5E,1545
+unit_tests/sources/file_based/stream/test_default_file_based_stream.py,sha256=gZlNs6s9dRgcFLrLZtkJDRSje_8gDylHj3xktjsUMVo,5782
 unit_tests/sources/fixtures/__init__.py,sha256=ZnqYNxHsKCgO38IwB34RQyRMXTs4GTvlRi3ImKnIioo,61
 unit_tests/sources/fixtures/source_test_fixture.py,sha256=r-UtR241EGQMZTw1RoKaatrpCGeQn7OIuRPWfG9f7nI,5380
 unit_tests/sources/message/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -352,8 +352,8 @@ unit_tests/utils/test_schema_inferrer.py,sha256=Z2jHBZ540wnYkylIdV_2xr75Vtwlxuyg
 unit_tests/utils/test_secret_utils.py,sha256=XKe0f1RHYii8iwE6ATmBr5JGDI1pzzrnZUGdUSMJQP4,4886
 unit_tests/utils/test_stream_status_utils.py,sha256=NpV155JMXA6CG-2Zvofa14lItobyh3Onttc59X4m5DI,3382
 unit_tests/utils/test_traced_exception.py,sha256=bDFP5zMBizFenz6V2WvEZTRCKGB5ijh3DBezjbfoYIs,4198
-airbyte_cdk-0.51.2.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-0.51.2.dist-info/METADATA,sha256=l-yiYXAQKz7KgIO8yxCtNI2fC-qS3XyrrZ1wuUptaP0,9399
-airbyte_cdk-0.51.2.dist-info/WHEEL,sha256=5sUXSg9e4bi7lTLOHcm6QEYwO5TIF1TNbTSVFVjcJcc,92
-airbyte_cdk-0.51.2.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
-airbyte_cdk-0.51.2.dist-info/RECORD,,
+airbyte_cdk-0.51.4.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-0.51.4.dist-info/METADATA,sha256=imzQYjx2YvmpJDtuZK6lUo96gKmLJBYNOZf06KeAg-s,9399
+airbyte_cdk-0.51.4.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+airbyte_cdk-0.51.4.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
+airbyte_cdk-0.51.4.dist-info/RECORD,,

{airbyte_cdk-0.51.2.dist-info → airbyte_cdk-0.51.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.1)
+Generator: bdist_wheel (0.41.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

unit_tests/sources/file_based/file_types/test_csv_parser.py

@@ -100,7 +100,7 @@ logger = logging.getLogger()
         pytest.param(
             {"col9": "['a', 'b']"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {"col9": "['a', 'b']"}, id="cannot-cast-to-list-of-objects"
         ),
-        pytest.param({"col11": "x"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {"col11": "x"}, id="item-not-in-props-doesn't-error"),
+        pytest.param({"col11": "x"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {}, id="item-not-in-props-doesn't-error"),
     ],
 )
 def test_cast_to_python_type(row: Dict[str, str], true_values: Set[str], false_values: Set[str], expected_output: Dict[str, Any]) -> None:

unit_tests/sources/file_based/scenarios/csv_scenarios.py

@@ -481,7 +481,6 @@ multi_csv_stream_n_file_exceeds_limit_for_inference = (
                 "data": {
                     "col1": "val11b",
                     "col2": "val12b",
-                    "col3": "val13b",
                     "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
                     "_ab_source_file_url": "b.csv",
                 },
@@ -491,7 +490,6 @@ multi_csv_stream_n_file_exceeds_limit_for_inference = (
                 "data": {
                     "col1": "val21b",
                     "col2": "val22b",
-                    "col3": "val23b",
                     "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
                     "_ab_source_file_url": "b.csv",
                 },

unit_tests/sources/file_based/stream/test_default_file_based_stream.py

@@ -2,9 +2,20 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from typing import Any, Mapping
+import unittest
+from datetime import datetime, timezone
+from typing import Any, Iterable, Iterator, Mapping
+from unittest.mock import Mock
 
 import pytest
+from airbyte_cdk.models import Level
+from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy
+from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy
+from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
+from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
+from airbyte_cdk.sources.file_based.remote_file import RemoteFile
+from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy
+from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
 from airbyte_cdk.sources.file_based.stream.default_file_based_stream import DefaultFileBasedStream
 
 
@@ -46,3 +57,74 @@ from airbyte_cdk.sources.file_based.stream.default_file_based_stream import Defa
 )
 def test_fill_nulls(input_schema: Mapping[str, Any], expected_output: Mapping[str, Any]) -> None:
     assert DefaultFileBasedStream._fill_nulls(input_schema) == expected_output
+
+
+class DefaultFileBasedStreamTest(unittest.TestCase):
+    _FILE_TYPE = "file_type"
+    _NOW = datetime(2022, 10, 22, tzinfo=timezone.utc)
+    _A_RECORD = {"a_record": 1}
+
+    def setUp(self) -> None:
+        self._stream_config = Mock()
+        self._stream_config.file_type = self._FILE_TYPE
+        self._stream_config.name = "a stream name"
+        self._catalog_schema = Mock()
+        self._stream_reader = Mock(spec=AbstractFileBasedStreamReader)
+        self._availability_strategy = Mock(spec=AbstractFileBasedAvailabilityStrategy)
+        self._discovery_policy = Mock(spec=AbstractDiscoveryPolicy)
+        self._parser = Mock(spec=FileTypeParser)
+        self._validation_policy = Mock(spec=AbstractSchemaValidationPolicy)
+        self._validation_policy.name = "validation policy name"
+        self._cursor = Mock(spec=AbstractFileBasedCursor)
+
+        self._stream = DefaultFileBasedStream(
+            config=self._stream_config,
+            catalog_schema=self._catalog_schema,
+            stream_reader=self._stream_reader,
+            availability_strategy=self._availability_strategy,
+            discovery_policy=self._discovery_policy,
+            parsers={self._FILE_TYPE: self._parser},
+            validation_policy=self._validation_policy,
+            cursor=self._cursor,
+        )
+
+    def test_when_read_records_from_slice_then_return_records(self) -> None:
+        self._parser.parse_records.return_value = [self._A_RECORD]
+        messages = list(self._stream.read_records_from_slice({"files": [RemoteFile(uri="uri", last_modified=self._NOW)]}))
+        assert list(map(lambda message: message.record.data["data"], messages)) == [self._A_RECORD]
+
+    def test_given_exception_when_read_records_from_slice_then_do_process_other_files(self) -> None:
+        """
+        The current behavior for source-s3 v3 does not fail sync on some errors and hence, we will keep this behaviour for now. One example
+        we can easily reproduce this is by having a file with gzip extension that is not actually a gzip file. The reader will fail to open
+        the file but the sync won't fail.
+        Ticket: https://github.com/airbytehq/airbyte/issues/29680
+        """
+        self._parser.parse_records.side_effect = [ValueError("An error"), [self._A_RECORD]]
+
+        messages = list(self._stream.read_records_from_slice({"files": [
+            RemoteFile(uri="invalid_file", last_modified=self._NOW),
+            RemoteFile(uri="valid_file", last_modified=self._NOW),
+        ]}))
+
+        assert messages[0].log.level == Level.ERROR
+        assert messages[1].record.data["data"] == self._A_RECORD
+
+    def test_given_exception_after_skipping_records_when_read_records_from_slice_then_send_warning(self) -> None:
+        self._stream_config.schemaless = False
+        self._validation_policy.record_passes_validation_policy.return_value = False
+        self._parser.parse_records.side_effect = [self._iter([self._A_RECORD, ValueError("An error")])]
+
+        messages = list(self._stream.read_records_from_slice({"files": [
+            RemoteFile(uri="invalid_file", last_modified=self._NOW),
+            RemoteFile(uri="valid_file", last_modified=self._NOW),
+        ]}))
+
+        assert messages[0].log.level == Level.ERROR
+        assert messages[1].log.level == Level.WARN
+
+    def _iter(self, x: Iterable[Any]) -> Iterator[Any]:
+        for item in x:
+            if isinstance(item, Exception):
+                raise item
+            yield item