airbyte-cdk 0.68.4__tar.gz → 0.69.1__tar.gz

{airbyte-cdk-0.68.4 → airbyte-cdk-0.69.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.68.4
+Version: 0.69.1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.68.4 → airbyte-cdk-0.69.1}/airbyte_cdk/entrypoint.py

@@ -10,23 +10,24 @@ import os.path
 import socket
 import sys
 import tempfile
+from collections import defaultdict
 from functools import wraps
-from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union
+from typing import Any, DefaultDict, Iterable, List, Mapping, MutableMapping, Optional, Union
 from urllib.parse import urlparse
 
 import requests
 from airbyte_cdk.connector import TConfig
 from airbyte_cdk.exception_handler import init_uncaught_exception_handler
 from airbyte_cdk.logger import init_logger
-from airbyte_cdk.models import AirbyteMessage, Status, Type
-from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification  # type: ignore [attr-defined]
+from airbyte_cdk.models import AirbyteMessage, FailureType, Status, Type
+from airbyte_cdk.models.airbyte_protocol import AirbyteStateStats, ConnectorSpecification  # type: ignore [attr-defined]
 from airbyte_cdk.sources import Source
+from airbyte_cdk.sources.connector_state_manager import HashableStreamDescriptor
 from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config
-from airbyte_cdk.utils import is_cloud_environment
+from airbyte_cdk.utils import is_cloud_environment, message_utils
 from airbyte_cdk.utils.airbyte_secrets_utils import get_secrets, update_secrets
 from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
-from airbyte_protocol.models import FailureType
 from requests import PreparedRequest, Response, Session
 
 logger = init_logger("airbyte")
@@ -160,8 +161,27 @@ class AirbyteEntrypoint(object):
         if self.source.check_config_against_spec:
             self.validate_connection(source_spec, config)
 
-        yield from self.source.read(self.logger, config, catalog, state)
-        yield from self._emit_queued_messages(self.source)
+        stream_message_counter: DefaultDict[HashableStreamDescriptor, int] = defaultdict(int)
+        for message in self.source.read(self.logger, config, catalog, state):
+            yield self.handle_record_counts(message, stream_message_counter)
+        for message in self._emit_queued_messages(self.source):
+            yield self.handle_record_counts(message, stream_message_counter)
+
+    @staticmethod
+    def handle_record_counts(message: AirbyteMessage, stream_message_count: DefaultDict[HashableStreamDescriptor, int]) -> AirbyteMessage:
+        if message.type == Type.RECORD:
+            stream_message_count[message_utils.get_stream_descriptor(message)] += 1
+
+        elif message.type == Type.STATE:
+            stream_descriptor = message_utils.get_stream_descriptor(message)
+
+            # Set record count from the counter onto the state message
+            message.state.sourceStats = message.state.sourceStats or AirbyteStateStats()
+            message.state.sourceStats.recordCount = stream_message_count.get(stream_descriptor, 0)
+
+            # Reset the counter
+            stream_message_count[stream_descriptor] = 0
+        return message
 
     @staticmethod
     def validate_connection(source_spec: ConnectorSpecification, config: TConfig) -> None:

{airbyte-cdk-0.68.4 → airbyte-cdk-0.69.1}/airbyte_cdk/sources/connector_state_manager.py

@@ -82,7 +82,6 @@ class ConnectorStateManager:
         Generates an AirbyteMessage using the current per-stream state of a specified stream in either the per-stream or legacy format
         :param stream_name: The name of the stream for the message that is being created
         :param namespace: The namespace of the stream for the message that is being created
-        :param send_per_stream_state: Decides which state format the message should be generated as
         :return: The Airbyte state message to be emitted by the connector during a sync
         """
         hashable_descriptor = HashableStreamDescriptor(name=stream_name, namespace=namespace)

{airbyte-cdk-0.68.4 → airbyte-cdk-0.69.1}/airbyte_cdk/sources/file_based/file_based_source.py

@@ -36,7 +36,7 @@ from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedS
 from airbyte_cdk.sources.file_based.stream.concurrent.cursor import (
     AbstractConcurrentFileBasedCursor,
     FileBasedConcurrentCursor,
-    FileBasedNoopCursor,
+    FileBasedFinalStateCursor,
 )
 from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
 from airbyte_cdk.sources.message.repository import InMemoryMessageRepository, MessageRepository
@@ -170,7 +170,9 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
                 sync_mode = self._get_sync_mode_from_catalog(stream_config.name)
 
                 if sync_mode == SyncMode.full_refresh and hasattr(self, "_concurrency_level") and self._concurrency_level is not None:
-                    cursor = FileBasedNoopCursor(stream_config)
+                    cursor = FileBasedFinalStateCursor(
+                        stream_config=stream_config, stream_namespace=None, message_repository=self.message_repository
+                    )
                     stream = FileBasedStreamFacade.create_from_stream(
                         self._make_default_stream(stream_config, cursor), self, self.logger, stream_state, cursor
                     )

{airbyte-cdk-0.68.4 → airbyte-cdk-0.69.1}/airbyte_cdk/sources/file_based/stream/concurrent/adapters.py

@@ -18,7 +18,7 @@ from airbyte_cdk.sources.file_based.config.file_based_stream_config import Prima
 from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from airbyte_cdk.sources.file_based.stream import AbstractFileBasedStream
-from airbyte_cdk.sources.file_based.stream.concurrent.cursor import FileBasedNoopCursor
+from airbyte_cdk.sources.file_based.stream.concurrent.cursor import FileBasedFinalStateCursor
 from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
 from airbyte_cdk.sources.file_based.types import StreamSlice
 from airbyte_cdk.sources.message import MessageRepository
@@ -71,7 +71,7 @@ class FileBasedStreamFacade(AbstractStreamFacade[DefaultStream], AbstractFileBas
                 partition_generator=FileBasedStreamPartitionGenerator(
                     stream,
                     message_repository,
-                    SyncMode.full_refresh if isinstance(cursor, FileBasedNoopCursor) else SyncMode.incremental,
+                    SyncMode.full_refresh if isinstance(cursor, FileBasedFinalStateCursor) else SyncMode.incremental,
                     [cursor_field] if cursor_field is not None else None,
                     state,
                     cursor,

{airbyte-cdk-0.68.4 → airbyte-cdk-0.69.1}/airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py

@@ -1,5 +1,5 @@
 from .abstract_concurrent_file_based_cursor import AbstractConcurrentFileBasedCursor
 from .file_based_concurrent_cursor import FileBasedConcurrentCursor
-from .file_based_noop_cursor import FileBasedNoopCursor
+from .file_based_final_state_cursor import FileBasedFinalStateCursor
 
-__all__ = ["AbstractConcurrentFileBasedCursor", "FileBasedConcurrentCursor", "FileBasedNoopCursor"]
+__all__ = ["AbstractConcurrentFileBasedCursor", "FileBasedConcurrentCursor", "FileBasedFinalStateCursor"]

airbyte-cdk-0.68.4/airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_noop_cursor.py → airbyte-cdk-0.69.1/airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py

@@ -4,12 +4,15 @@
 
 import logging
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Iterable, List, MutableMapping
+from typing import TYPE_CHECKING, Any, Iterable, List, MutableMapping, Optional
 
+from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from airbyte_cdk.sources.file_based.stream.concurrent.cursor.abstract_concurrent_file_based_cursor import AbstractConcurrentFileBasedCursor
 from airbyte_cdk.sources.file_based.types import StreamState
+from airbyte_cdk.sources.message import MessageRepository
+from airbyte_cdk.sources.streams import FULL_REFRESH_SENTINEL_STATE_KEY
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
 from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
 
@@ -17,13 +20,23 @@ if TYPE_CHECKING:
     from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamPartition
 
 
-class FileBasedNoopCursor(AbstractConcurrentFileBasedCursor):
-    def __init__(self, stream_config: FileBasedStreamConfig, **kwargs: Any):
-        pass
+class FileBasedFinalStateCursor(AbstractConcurrentFileBasedCursor):
+    """Cursor that is used to guarantee at least one state message is emitted for a concurrent file-based stream."""
+
+    def __init__(
+        self, stream_config: FileBasedStreamConfig, message_repository: MessageRepository, stream_namespace: Optional[str], **kwargs: Any
+    ):
+        self._stream_name = stream_config.name
+        self._stream_namespace = stream_namespace
+        self._message_repository = message_repository
+        # Normally the connector state manager operates at the source-level. However, we only need it to write the sentinel
+        # state message rather than manage overall source state. This is also only temporary as we move to the resumable
+        # full refresh world where every stream uses a FileBasedConcurrentCursor with incremental state.
+        self._connector_state_manager = ConnectorStateManager(stream_instance_map={})
 
     @property
     def state(self) -> MutableMapping[str, Any]:
-        return {}
+        return {FULL_REFRESH_SENTINEL_STATE_KEY: True}
 
     def observe(self, record: Record) -> None:
         pass
@@ -53,4 +66,6 @@ class FileBasedNoopCursor(AbstractConcurrentFileBasedCursor):
         pass
 
     def ensure_at_least_one_state_emitted(self) -> None:
-        pass
+        self._connector_state_manager.update_state_for_stream(self._stream_name, self._stream_namespace, self.state)
+        state_message = self._connector_state_manager.create_state_message(self._stream_name, self._stream_namespace)
+        self._message_repository.emit_message(state_message)

{airbyte-cdk-0.68.4 → airbyte-cdk-0.69.1}/airbyte_cdk/sources/streams/concurrent/adapters.py

@@ -21,7 +21,7 @@ from airbyte_cdk.sources.streams.concurrent.availability_strategy import (
     StreamAvailable,
     StreamUnavailable,
 )
-from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, NoopCursor
+from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, FinalStateCursor
 from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
 from airbyte_cdk.sources.streams.concurrent.exceptions import ExceptionWithDisplayMessage
 from airbyte_cdk.sources.streams.concurrent.helpers import get_cursor_field_from_stream, get_primary_key_from_stream
@@ -77,7 +77,7 @@ class StreamFacade(AbstractStreamFacade[DefaultStream], Stream):
                 partition_generator=StreamPartitionGenerator(
                     stream,
                     message_repository,
-                    SyncMode.full_refresh if isinstance(cursor, NoopCursor) else SyncMode.incremental,
+                    SyncMode.full_refresh if isinstance(cursor, FinalStateCursor) else SyncMode.incremental,
                     [cursor_field] if cursor_field is not None else None,
                     state,
                     cursor,

{airbyte-cdk-0.68.4 → airbyte-cdk-0.69.1}/airbyte_cdk/sources/streams/concurrent/cursor.py

@@ -8,6 +8,7 @@ from typing import Any, List, Mapping, MutableMapping, Optional, Protocol, Tuple
 
 from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
 from airbyte_cdk.sources.message import MessageRepository
+from airbyte_cdk.sources.streams import FULL_REFRESH_SENTINEL_STATE_KEY
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
 from airbyte_cdk.sources.streams.concurrent.partitions.record import Record
 from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_state_converter import AbstractStreamStateConverter
@@ -65,10 +66,27 @@ class Cursor(ABC):
         raise NotImplementedError()
 
 
-class NoopCursor(Cursor):
+class FinalStateCursor(Cursor):
+    """Cursor that is used to guarantee at least one state message is emitted for a concurrent stream."""
+
+    def __init__(
+        self,
+        stream_name: str,
+        stream_namespace: Optional[str],
+        message_repository: MessageRepository,
+    ) -> None:
+        self._stream_name = stream_name
+        self._stream_namespace = stream_namespace
+        self._message_repository = message_repository
+        # Normally the connector state manager operates at the source-level. However, we only need it to write the sentinel
+        # state message rather than manage overall source state. This is also only temporary as we move to the resumable
+        # full refresh world where every stream uses a FileBasedConcurrentCursor with incremental state.
+        self._connector_state_manager = ConnectorStateManager(stream_instance_map={})
+        self._has_closed_at_least_one_slice = False
+
     @property
     def state(self) -> MutableMapping[str, Any]:
-        return {}
+        return {FULL_REFRESH_SENTINEL_STATE_KEY: True}
 
     def observe(self, record: Record) -> None:
         pass
@@ -77,7 +95,13 @@ class NoopCursor(Cursor):
         pass
 
     def ensure_at_least_one_state_emitted(self) -> None:
-        pass
+        """
+        Used primarily for full refresh syncs that do not have a valid cursor value to emit at the end of a sync
+        """
+
+        self._connector_state_manager.update_state_for_stream(self._stream_name, self._stream_namespace, self.state)
+        state_message = self._connector_state_manager.create_state_message(self._stream_name, self._stream_namespace)
+        self._message_repository.emit_message(state_message)
 
 
 class ConcurrentCursor(Cursor):

{airbyte-cdk-0.68.4 → airbyte-cdk-0.69.1}/airbyte_cdk/sources/streams/concurrent/default_stream.py

@@ -9,7 +9,7 @@ from typing import Any, Iterable, List, Mapping, Optional
 from airbyte_cdk.models import AirbyteStream, SyncMode
 from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
 from airbyte_cdk.sources.streams.concurrent.availability_strategy import AbstractAvailabilityStrategy, StreamAvailability
-from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, NoopCursor
+from airbyte_cdk.sources.streams.concurrent.cursor import Cursor
 from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
 from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator
 
@@ -24,7 +24,7 @@ class DefaultStream(AbstractStream):
         primary_key: List[str],
         cursor_field: Optional[str],
         logger: Logger,
-        cursor: Optional[Cursor],
+        cursor: Cursor,
         namespace: Optional[str] = None,
     ) -> None:
         self._stream_partition_generator = partition_generator
@@ -34,7 +34,7 @@ class DefaultStream(AbstractStream):
         self._primary_key = primary_key
         self._cursor_field = cursor_field
         self._logger = logger
-        self._cursor = cursor or NoopCursor()
+        self._cursor = cursor
         self._namespace = namespace
 
     def generate_partitions(self) -> Iterable[Partition]:
@@ -44,6 +44,10 @@ class DefaultStream(AbstractStream):
     def name(self) -> str:
         return self._name
 
+    @property
+    def namespace(self) -> Optional[str]:
+        return self._namespace
+
     def check_availability(self) -> StreamAvailability:
         return self._availability_strategy.check_availability(self._logger)
 

{airbyte-cdk-0.68.4 → airbyte-cdk-0.69.1}/airbyte_cdk/test/entrypoint_wrapper.py

@@ -74,7 +74,7 @@ class EntrypointOutput:
         state_messages = self._get_message_by_types([Type.STATE])
         if not state_messages:
             raise ValueError("Can't provide most recent state as there are no state messages")
-        return state_messages[-1].state.data
+        return state_messages[-1].state.stream
 
     @property
     def logs(self) -> List[AirbyteMessage]:

airbyte-cdk-0.69.1/airbyte_cdk/utils/message_utils.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from airbyte_cdk.sources.connector_state_manager import HashableStreamDescriptor
+from airbyte_protocol.models import AirbyteMessage, Type
+
+
+def get_stream_descriptor(message: AirbyteMessage) -> HashableStreamDescriptor:
+    if message.type == Type.RECORD:
+        return HashableStreamDescriptor(name=message.record.stream, namespace=message.record.namespace)
+    elif message.type == Type.STATE:
+        if not message.state.stream or not message.state.stream.stream_descriptor:
+            raise ValueError("State message was not in per-stream state format, which is required for record counts.")
+        return HashableStreamDescriptor(
+            name=message.state.stream.stream_descriptor.name, namespace=message.state.stream.stream_descriptor.namespace
+        )
+    else:
+        raise NotImplementedError(f"get_stream_descriptor is not implemented for message type '{message.type}'.")

{airbyte-cdk-0.68.4 → airbyte-cdk-0.69.1}/airbyte_cdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.68.4
+Version: 0.69.1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.68.4 → airbyte-cdk-0.69.1}/airbyte_cdk.egg-info/SOURCES.txt

@@ -199,7 +199,7 @@ airbyte_cdk/sources/file_based/stream/concurrent/adapters.py
 airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py
 airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py
 airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py
-airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_noop_cursor.py
+airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_cursor.py
 airbyte_cdk/sources/file_based/stream/cursor/__init__.py
 airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py
 airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py
@@ -274,6 +274,7 @@ airbyte_cdk/utils/datetime_format_inferrer.py
 airbyte_cdk/utils/event_timing.py
 airbyte_cdk/utils/is_cloud_environment.py
 airbyte_cdk/utils/mapping_helpers.py
+airbyte_cdk/utils/message_utils.py
 airbyte_cdk/utils/oneof_option_config.py
 airbyte_cdk/utils/schema_inferrer.py
 airbyte_cdk/utils/spec_schema_transformations.py
@@ -461,6 +462,7 @@ unit_tests/test/mock_http/test_response_builder.py
 unit_tests/utils/__init__.py
 unit_tests/utils/test_datetime_format_inferrer.py
 unit_tests/utils/test_mapping_helpers.py
+unit_tests/utils/test_message_utils.py
 unit_tests/utils/test_rate_limiting.py
 unit_tests/utils/test_schema_inferrer.py
 unit_tests/utils/test_secret_utils.py

{airbyte-cdk-0.68.4 → airbyte-cdk-0.69.1}/setup.py

@@ -36,7 +36,7 @@ setup(
     name="airbyte-cdk",
     # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be
     # updated if our semver format changes such as using release candidate versions.
-    version="0.68.4",
+    version="0.69.1",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

{airbyte-cdk-0.68.4 → airbyte-cdk-0.69.1}/unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py

@@ -20,7 +20,7 @@ from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import Conc
 from airbyte_cdk.sources.message import InMemoryMessageRepository
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade
-from airbyte_cdk.sources.streams.concurrent.cursor import NoopCursor
+from airbyte_cdk.sources.streams.concurrent.cursor import FinalStateCursor
 
 
 class _MockSource(ConcurrentSourceAdapter):
@@ -36,7 +36,7 @@ class _MockSource(ConcurrentSourceAdapter):
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         return [
-            StreamFacade.create_from_stream(s, self, self._logger, None, NoopCursor()) if is_concurrent else s
+            StreamFacade.create_from_stream(s, self, self._logger, None, FinalStateCursor(stream_name=s.name, stream_namespace=s.namespace, message_repository=InMemoryMessageRepository())) if is_concurrent else s
             for s, is_concurrent in self._streams_to_is_concurrent.items()
         ]
 

{airbyte-cdk-0.68.4 → airbyte-cdk-0.69.1}/unit_tests/sources/file_based/scenarios/csv_scenarios.py

@@ -467,30 +467,24 @@ single_csv_scenario: TestScenario[InMemoryFilesSource] = (
     )
 ).build()
 
-multi_format_analytics_scenario: TestScenario[InMemoryFilesSource] = (
+csv_analytics_scenario: TestScenario[InMemoryFilesSource] = (
     TestScenarioBuilder[InMemoryFilesSource]()
-    .set_name("multi_format_analytics")
+    .set_name("csv_analytics")
     .set_config(
         {
             "streams": [
                 {
                     "name": "stream1",
                     "format": {"filetype": "csv"},
-                    "globs": ["file1.csv"],
+                    "globs": ["a.csv"],
                     "validation_policy": "Emit Record",
                 },
                 {
                     "name": "stream2",
                     "format": {"filetype": "csv"},
-                    "globs": ["file2.csv"],
-                    "validation_policy": "Emit Record",
-                },
-                {
-                    "name": "stream3",
-                    "format": {"filetype": "jsonl"},
-                    "globs": ["file3.jsonl"],
+                    "globs": ["b.csv"],
                     "validation_policy": "Emit Record",
-                },
+                }
             ]
         }
     )
@@ -498,17 +492,21 @@ multi_format_analytics_scenario: TestScenario[InMemoryFilesSource] = (
         FileBasedSourceBuilder()
         .set_files(
             {
-                "file1.csv": {
-                    "contents": [],
+                "a.csv": {
+                    "contents": [
+                        ("col1", "col2"),
+                        ("val11a", "val12a"),
+                        ("val21a", "val22a"),
+                    ],
                     "last_modified": "2023-06-05T03:54:07.000Z",
                 },
-                "file2.csv": {
-                    "contents": [],
-                    "last_modified": "2023-06-06T03:54:07.000Z",
-                },
-                "file3.jsonl": {
-                    "contents": [],
-                    "last_modified": "2023-06-07T03:54:07.000Z",
+                "b.csv": {
+                    "contents": [
+                        ("col1", "col2", "col3"),
+                        ("val11b", "val12b", "val13b"),
+                        ("val21b", "val22b", "val23b"),
+                    ],
+                    "last_modified": "2023-06-05T03:54:07.000Z",
                 },
             }
         )
@@ -521,7 +519,12 @@ multi_format_analytics_scenario: TestScenario[InMemoryFilesSource] = (
                     "default_cursor_field": ["_ab_source_file_last_modified"],
                     "json_schema": {
                         "type": "object",
-                        "properties": {},
+                        "properties": {
+                            "col1": {"type": ["null", "string"]},
+                            "col2": {"type": ["null", "string"]},
+                            "_ab_source_file_last_modified": {"type": "string"},
+                            "_ab_source_file_url": {"type": "string"},
+                        },
                     },
                     "name": "stream1",
                     "source_defined_cursor": True,
@@ -531,30 +534,64 @@ multi_format_analytics_scenario: TestScenario[InMemoryFilesSource] = (
                     "default_cursor_field": ["_ab_source_file_last_modified"],
                     "json_schema": {
                         "type": "object",
-                        "properties": {},
+                        "properties": {
+                            "col1": {"type": ["null", "string"]},
+                            "col2": {"type": ["null", "string"]},
+                            "col3": {"type": ["null", "string"]},
+                            "_ab_source_file_last_modified": {"type": "string"},
+                            "_ab_source_file_url": {"type": "string"},
+                        },
                     },
                     "name": "stream2",
                     "source_defined_cursor": True,
                     "supported_sync_modes": ["full_refresh", "incremental"],
-                },
-                {
-                    "default_cursor_field": ["_ab_source_file_last_modified"],
-                    "json_schema": {
-                        "type": "object",
-                        "properties": {},
-                    },
-                    "name": "stream3",
-                    "source_defined_cursor": True,
-                    "supported_sync_modes": ["full_refresh", "incremental"],
-                },
+                }
             ]
         }
     )
-    .set_expected_records([])
+    .set_expected_records([
+        {
+            "data": {
+                "col1": "val11a",
+                "col2": "val12a",
+                "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
+                "_ab_source_file_url": "a.csv",
+            },
+            "stream": "stream1",
+        },
+        {
+            "data": {
+                "col1": "val21a",
+                "col2": "val22a",
+                "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
+                "_ab_source_file_url": "a.csv",
+            },
+            "stream": "stream1",
+        },
+        {
+            "data": {
+                "col1": "val11b",
+                "col2": "val12b",
+                "col3": "val13b",
+                "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
+                "_ab_source_file_url": "b.csv",
+            },
+            "stream": "stream2",
+        },
+        {
+            "data": {
+                "col1": "val21b",
+                "col2": "val22b",
+                "col3": "val23b",
+                "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z",
+                "_ab_source_file_url": "b.csv",
+            },
+            "stream": "stream2",
+        },
+    ])
     .set_expected_analytics(
         [
             AirbyteAnalyticsTraceMessage(type="file-cdk-csv-stream-count", value="2"),
-            AirbyteAnalyticsTraceMessage(type="file-cdk-jsonl-stream-count", value="1"),
         ]
     )
 ).build()
@@ -1450,7 +1487,6 @@ empty_schema_inference_scenario: TestScenario[InMemoryFilesSource] = (
         }
     )
     .set_expected_discover_error(AirbyteTracedException, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value)
-    .set_expected_records([])
 ).build()
 
 schemaless_csv_scenario: TestScenario[InMemoryFilesSource] = (
@@ -3009,6 +3045,61 @@ earlier_csv_scenario: TestScenario[InMemoryFilesSource] = (
             ]
         }
     )
-    .set_expected_records([])
     .set_expected_discover_error(AirbyteTracedException, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value)
 ).build()
+
+csv_no_records_scenario: TestScenario[InMemoryFilesSource] = (
+    TestScenarioBuilder[InMemoryFilesSource]()
+    .set_name("csv_empty_no_records")
+    .set_config(
+        {
+            "streams": [
+                {
+                    "name": "stream1",
+                    "globs": ["*"],
+                    "validation_policy": "Emit Record",
+                    "input_schema": '{"col1": "boolean", "col2": "string"}',
+                    "format": {
+                        "filetype": "csv",
+                        "null_values": ["null"],
+                    },
+                }
+            ],
+            "start_date": "2023-06-04T03:54:07.000000Z",
+        }
+    )
+    .set_source_builder(
+        FileBasedSourceBuilder()
+        .set_files(
+            {
+                "a.csv": {
+                    "contents": [("col1", "col2")],  # column headers, but no data rows
+                    "last_modified": "2023-06-05T03:54:07.000Z",
+                }
+            }
+        )
+        .set_file_type("csv")
+    )
+    .set_expected_catalog(
+        {
+            "streams": [
+                {
+                    "default_cursor_field": ["_ab_source_file_last_modified"],
+                    "json_schema": {
+                        "type": "object",
+                        "properties": {
+                            "col1": {"type": "boolean"},
+                            "col2": {"type": "string"},
+                            "_ab_source_file_last_modified": {"type": "string"},
+                            "_ab_source_file_url": {"type": "string"},
+                        },
+                    },
+                    "name": "stream1",
+                    "source_defined_cursor": True,
+                    "supported_sync_modes": ["full_refresh", "incremental"],
+                }
+            ]
+        }
+    )
+    .set_expected_records([])
+).build()

{airbyte-cdk-0.68.4 → airbyte-cdk-0.69.1}/unit_tests/sources/file_based/stream/concurrent/test_adapters.py

@@ -23,7 +23,7 @@ from airbyte_cdk.sources.file_based.stream.concurrent.adapters import (
     FileBasedStreamPartition,
     FileBasedStreamPartitionGenerator,
 )
-from airbyte_cdk.sources.file_based.stream.concurrent.cursor import FileBasedNoopCursor
+from airbyte_cdk.sources.file_based.stream.concurrent.cursor import FileBasedFinalStateCursor
 from airbyte_cdk.sources.message import InMemoryMessageRepository
 from airbyte_cdk.sources.streams.concurrent.cursor import Cursor
 from airbyte_cdk.sources.streams.concurrent.exceptions import ExceptionWithDisplayMessage
@@ -36,7 +36,7 @@ _ANY_SYNC_MODE = SyncMode.full_refresh
 _ANY_STATE = {"state_key": "state_value"}
 _ANY_CURSOR_FIELD = ["a", "cursor", "key"]
 _STREAM_NAME = "stream"
-_ANY_CURSOR = Mock(spec=FileBasedNoopCursor)
+_ANY_CURSOR = Mock(spec=FileBasedFinalStateCursor)
 
 
 @pytest.mark.parametrize(
@@ -165,7 +165,7 @@ class StreamFacadeTest(unittest.TestCase):
             supported_sync_modes=[SyncMode.full_refresh],
         )
         self._legacy_stream = DefaultFileBasedStream(
-            cursor=FileBasedNoopCursor(MagicMock()),
+            cursor=FileBasedFinalStateCursor(stream_config=MagicMock(), stream_namespace=None, message_repository=Mock()),
             config=FileBasedStreamConfig(name="stream", format=CsvFormat()),
             catalog_schema={},
             stream_reader=MagicMock(),