airbyte-cdk 0.51.16__py3-none-any.whl → 0.51.17__py3-none-any.whl

unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py

@@ -78,7 +78,7 @@ valid_single_stream_user_input_schema_scenario = (
             "streams": [
                 {
                     "name": "stream1",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["*"],
                     "validation_policy": "Emit Record",
                     "input_schema": '{"col1": "string", "col2": "string"}',
@@ -98,7 +98,7 @@ single_stream_user_input_schema_scenario_schema_is_invalid = (
             "streams": [
                 {
                     "name": "stream1",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["*"],
                     "validation_policy": "Emit Record",
                     "input_schema": '{"col1": "x", "col2": "string"}',
@@ -121,7 +121,7 @@ single_stream_user_input_schema_scenario_emit_nonconforming_records = (
             "streams": [
                 {
                     "name": "stream1",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["*"],
                     "validation_policy": "Emit Record",
                     "input_schema": '{"col1": "integer", "col2": "string"}',
@@ -171,7 +171,7 @@ single_stream_user_input_schema_scenario_skip_nonconforming_records = (
             "streams": [
                 {
                     "name": "stream1",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["*"],
                     "validation_policy": "Skip Record",
                     "input_schema": '{"col1": "integer", "col2": "string"}',
@@ -364,21 +364,21 @@ valid_multi_stream_user_input_schema_scenario = (
             "streams": [
                 {
                     "name": "stream1",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["a.csv"],
                     "validation_policy": "Emit Record",
                     "input_schema": '{"col1": "string", "col2": "integer"}',
                 },
                 {
                     "name": "stream2",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["b.csv"],
                     "validation_policy": "Emit Record",
                     "input_schema": '{"col1": "string", "col2": "string", "col3": "string"}',
                 },
                 {
                     "name": "stream3",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["c.csv"],
                     "validation_policy": "Emit Record",
                 },
@@ -398,21 +398,21 @@ multi_stream_user_input_schema_scenario_schema_is_invalid = (
             "streams": [
                 {
                     "name": "stream1",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["a.csv"],
                     "validation_policy": "Emit Record",
                     "input_schema": '{"col1": "string", "col2": "integer"}',
                 },
                 {
                     "name": "stream2",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["b.csv"],
                     "validation_policy": "Emit Record",
                     "input_schema": '{"col1": "x", "col2": "string", "col3": "string"}',  # this stream's schema is invalid
                 },
                 {
                     "name": "stream3",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["c.csv"],
                     "validation_policy": "Emit Record",
                 },
@@ -435,21 +435,21 @@ multi_stream_user_input_schema_scenario_emit_nonconforming_records = (
             "streams": [
                 {
                     "name": "stream1",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["a.csv"],
                     "validation_policy": "Emit Record",
                     "input_schema": '{"col1": "string", "col2": "integer"}',
                 },
                 {
                     "name": "stream2",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["b.csv"],
                     "validation_policy": "Emit Record",
                     "input_schema": '{"col1": "string", "col2": "integer", "col3": "string"}',  # this stream's records do not conform to the schema
                 },
                 {
                     "name": "stream3",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["c.csv"],
                     "validation_policy": "Emit Record",
                 },
@@ -574,21 +574,21 @@ multi_stream_user_input_schema_scenario_skip_nonconforming_records = (
             "streams": [
                 {
                     "name": "stream1",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["a.csv"],
                     "validation_policy": "Emit Record",
                     "input_schema": '{"col1": "string", "col2": "integer"}',
                 },
                 {
                     "name": "stream2",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["b.csv"],
                     "validation_policy": "Skip Record",
                     "input_schema": '{"col1": "string", "col2": "integer", "col3": "string"}',  # this stream's records do not conform to the schema
                 },
                 {
                     "name": "stream3",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["c.csv"],
                     "validation_policy": "Emit Record",
                 },

unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py

@@ -204,7 +204,7 @@ skip_record_scenario_single_stream = (
             "streams": [
                 {
                     "name": "stream1",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["*.csv"],
                     "validation_policy": "Skip Record",
                 }
@@ -250,13 +250,13 @@ skip_record_scenario_multi_stream = (
             "streams": [
                 {
                     "name": "stream1",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["a/*.csv"],
                     "validation_policy": "Skip Record",
                 },
                 {
                     "name": "stream2",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["b/*.csv"],
                     "validation_policy": "Skip Record",
                 }
@@ -317,7 +317,7 @@ emit_record_scenario_single_stream = (
             "streams": [
                 {
                     "name": "stream1",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["*.csv"],
                     "validation_policy": "Emit Record",
                 }
@@ -359,13 +359,13 @@ emit_record_scenario_multi_stream = (
             "streams": [
                 {
                     "name": "stream1",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["a/*.csv"],
                     "validation_policy": "Emit Record",
                 },
                 {
                     "name": "stream2",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["b/*.csv"],
                     "validation_policy": "Emit Record",
                 }
@@ -418,7 +418,7 @@ wait_for_rediscovery_scenario_single_stream = (
             "streams": [
                 {
                     "name": "stream1",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["*.csv"],
                     "validation_policy": "Wait for Discover",
                 }
@@ -453,13 +453,13 @@ wait_for_rediscovery_scenario_multi_stream = (
             "streams": [
                 {
                     "name": "stream1",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["a/*.csv"],
                     "validation_policy": "Wait for Discover",
                 },
                 {
                     "name": "stream2",
-                    "file_type": "csv",
+                    "format": {"filetype": "csv"},
                     "globs": ["b/*.csv"],
                     "validation_policy": "Wait for Discover",
                 }

unit_tests/sources/file_based/stream/test_default_file_based_cursor.py

@@ -7,6 +7,7 @@ from typing import Any, List, Mapping
 from unittest.mock import MagicMock
 
 import pytest
+from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, ValidationPolicy
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor
@@ -264,5 +265,5 @@ def get_cursor(max_history_size: int, days_to_sync_if_history_is_full: int) -> D
     cursor_cls = DefaultFileBasedCursor
     cursor_cls.DEFAULT_MAX_HISTORY_SIZE = max_history_size
     config = FileBasedStreamConfig(
-        file_type="csv", name="test", validation_policy=ValidationPolicy.emit_record, days_to_sync_if_history_is_full=days_to_sync_if_history_is_full)
+        format=CsvFormat(), name="test", validation_policy=ValidationPolicy.emit_record, days_to_sync_if_history_is_full=days_to_sync_if_history_is_full)
     return cursor_cls(config)

unit_tests/sources/file_based/stream/test_default_file_based_stream.py

@@ -19,6 +19,10 @@ from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
 from airbyte_cdk.sources.file_based.stream.default_file_based_stream import DefaultFileBasedStream
 
 
+class MockFormat:
+    pass
+
+
 @pytest.mark.parametrize(
     "input_schema, expected_output",
     [
@@ -60,13 +64,12 @@ def test_fill_nulls(input_schema: Mapping[str, Any], expected_output: Mapping[st
 
 
 class DefaultFileBasedStreamTest(unittest.TestCase):
-    _FILE_TYPE = "file_type"
     _NOW = datetime(2022, 10, 22, tzinfo=timezone.utc)
     _A_RECORD = {"a_record": 1}
 
     def setUp(self) -> None:
         self._stream_config = Mock()
-        self._stream_config.file_type = self._FILE_TYPE
+        self._stream_config.format = MockFormat()
         self._stream_config.name = "a stream name"
         self._catalog_schema = Mock()
         self._stream_reader = Mock(spec=AbstractFileBasedStreamReader)
@@ -83,7 +86,7 @@ class DefaultFileBasedStreamTest(unittest.TestCase):
             stream_reader=self._stream_reader,
             availability_strategy=self._availability_strategy,
             discovery_policy=self._discovery_policy,
-            parsers={self._FILE_TYPE: self._parser},
+            parsers={MockFormat: self._parser},
             validation_policy=self._validation_policy,
             cursor=self._cursor,
         )