airbyte-cdk 0.70.2__tar.gz → 0.72.0__tar.gz

{airbyte-cdk-0.70.2 → airbyte-cdk-0.72.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.70.2
+Version: 0.72.0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.70.2 → airbyte-cdk-0.72.0}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -602,6 +602,27 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
+  CustomSchemaLoader:
+    title: Custom Schema Loader
+    description: Schema Loader component whose behavior is derived from a custom code implementation of the connector.
+    type: object
+    additionalProperties: true
+    required:
+      - type
+      - class_name
+    properties:
+      type:
+        type: string
+        enum: [CustomSchemaLoader]
+      class_name:
+        title: Class Name
+        description: Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.
+        type: string
+        examples:
+          - "source_railz.components.MyCustomSchemaLoader"
+      $parameters:
+        type: object
+        additionalProperties: true
   CustomTransformation:
     title: Custom Transformation
     description: Transformation component whose behavior is derived from a custom code implementation of the connector.
@@ -948,6 +969,7 @@ definitions:
         anyOf:
           - "$ref": "#/definitions/InlineSchemaLoader"
           - "$ref": "#/definitions/JsonFileSchemaLoader"
+          - "$ref": "#/definitions/CustomSchemaLoader"
       # TODO we have move the transformation to the RecordSelector level in the code but kept this here for
       # compatibility reason. We should eventually move this to align with the code.
       transformations:

{airbyte-cdk-0.70.2 → airbyte-cdk-0.72.0}/airbyte_cdk/sources/declarative/incremental/cursor.py

@@ -24,18 +24,28 @@ class Cursor(ABC, StreamSlicer):
         :param stream_state: The state of the stream as returned by get_stream_state
         """
 
+    def observe(self, stream_slice: StreamSlice, record: Record) -> None:
+        """
+        Register a record with the cursor; the cursor instance can then use it to manage the state of the in-progress stream read.
+
+        :param stream_slice: The current slice, which may or may not contain the most recently observed record
+        :param record: the most recently-read record, which the cursor can use to update the stream state. Outwardly-visible changes to the
+          stream state may need to be deferred depending on whether the source reliably orders records by the cursor field.
+        """
+        pass
+
     @abstractmethod
     def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None:
         """
         Update state based on the stream slice and the latest record. Note that `stream_slice.cursor_slice` and
-        `last_record.associated_slice` are expected to be the same but we make it explicit here that `stream_slice` should be leveraged to
+        `most_recent_record.associated_slice` are expected to be the same but we make it explicit here that `stream_slice` should be leveraged to
         update the state.
 
         :param stream_slice: slice to close
-        :param last_record: the latest record we have received for the slice. This is important to consider because even if the cursor emits
-          a slice, some APIs are not able to enforce the upper boundary. The outcome is that the last_record might have a higher cursor
-          value than the slice upper boundary and if we want to reduce the duplication as much as possible, we need to consider the highest
-          value between the internal cursor, the stream slice upper boundary and the record cursor value.
+        :param most_recent_record: the latest record we have received for the slice. This is important to consider because even if the
+          cursor emits a slice, some APIs are not able to enforce the upper boundary. The outcome is that the last_record might have a
+          higher cursor value than the slice upper boundary and if we want to reduce the duplication as much as possible, we need to
+          consider the highest value between the internal cursor, the stream slice upper boundary and the record cursor value.
         """
 
     @abstractmethod

{airbyte-cdk-0.70.2 → airbyte-cdk-0.72.0}/airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py

@@ -52,7 +52,12 @@ class DatetimeBasedCursor(Cursor):
     datetime_format: str
     config: Config
     parameters: InitVar[Mapping[str, Any]]
-    _cursor: Optional[str] = field(repr=False, default=None)  # tracks current datetime
+    _highest_observed_cursor_field_value: Optional[str] = field(
+        repr=False, default=None
+    )  # tracks the latest observed datetime, which may not be safe to emit in the case of out-of-order records
+    _cursor: Optional[str] = field(
+        repr=False, default=None
+    )  # tracks the latest observed datetime that is appropriate to emit as stream state
     end_datetime: Optional[Union[MinMaxDatetime, str]] = None
     step: Optional[Union[InterpolatedString, str]] = None
     cursor_granularity: Optional[str] = None
@@ -109,20 +114,39 @@ class DatetimeBasedCursor(Cursor):
         """
         self._cursor = stream_state.get(self._cursor_field.eval(self.config)) if stream_state else None
 
-    def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None:
+    def observe(self, stream_slice: StreamSlice, record: Record) -> None:
+        """
+        Register a record with the cursor; the cursor instance can then use it to manage the state of the in-progress stream read.
+
+        :param stream_slice: The current slice, which may or may not contain the most recently observed record
+        :param record: the most recently-read record, which the cursor can use to update the stream state. Outwardly-visible changes to the
+          stream state may need to be deferred depending on whether the source reliably orders records by the cursor field.
+        """
+        record_cursor_value = record.get(self._cursor_field.eval(self.config))
+        # if the current record has no cursor value, we cannot meaningfully update the state based on it, so there is nothing more to do
+        if not record_cursor_value:
+            return
+
+        start_field = self._partition_field_start.eval(self.config)
+        end_field = self._partition_field_end.eval(self.config)
+        is_highest_observed_cursor_value = not self._highest_observed_cursor_field_value or self.parse_date(
+            record_cursor_value
+        ) > self.parse_date(self._highest_observed_cursor_field_value)
+        if (
+            self._is_within_daterange_boundaries(record, stream_slice.get(start_field), stream_slice.get(end_field))  # type: ignore # we know that stream_slices for these cursors will use a string representing an unparsed date
+            and is_highest_observed_cursor_value
+        ):
+            self._highest_observed_cursor_field_value = record_cursor_value
+
+    def close_slice(self, stream_slice: StreamSlice, _most_recent_record: Optional[Record]) -> None:
         if stream_slice.partition:
             raise ValueError(f"Stream slice {stream_slice} should not have a partition. Got {stream_slice.partition}.")
-        last_record_cursor_value = most_recent_record.get(self._cursor_field.eval(self.config)) if most_recent_record else None
-        stream_slice_value_end = stream_slice.get(self._partition_field_end.eval(self.config))
-        potential_cursor_values = [
-            cursor_value for cursor_value in [self._cursor, last_record_cursor_value, stream_slice_value_end] if cursor_value
-        ]
         cursor_value_str_by_cursor_value_datetime = dict(
             map(
                 # we need to ensure the cursor value is preserved as is in the state else the CATs might complain of something like
                 # 2023-01-04T17:30:19.000Z' <= '2023-01-04T17:30:19.000000Z'
-                lambda datetime_str: (self.parse_date(datetime_str), datetime_str),
-                potential_cursor_values,
+                lambda datetime_str: (self.parse_date(datetime_str), datetime_str),  # type: ignore # because of the filter on the next line, this will only be called with a str
+                filter(lambda item: item, [self._cursor, self._highest_observed_cursor_field_value]),
             )
         )
         self._cursor = (
@@ -279,10 +303,26 @@ class DatetimeBasedCursor(Cursor):
                 f"Could not find cursor field `{cursor_field}` in record. The incremental sync will assume it needs to be synced",
             )
             return True
-
         latest_possible_cursor_value = self._select_best_end_datetime()
         earliest_possible_cursor_value = self._calculate_earliest_possible_value(latest_possible_cursor_value)
-        return earliest_possible_cursor_value <= self.parse_date(record_cursor_value) <= latest_possible_cursor_value
+        return self._is_within_daterange_boundaries(record, earliest_possible_cursor_value, latest_possible_cursor_value)
+
+    def _is_within_daterange_boundaries(
+        self, record: Record, start_datetime_boundary: Union[datetime.datetime, str], end_datetime_boundary: Union[datetime.datetime, str]
+    ) -> bool:
+        cursor_field = self._cursor_field.eval(self.config)
+        record_cursor_value = record.get(cursor_field)
+        if not record_cursor_value:
+            self._send_log(
+                Level.WARN,
+                f"Could not find cursor field `{cursor_field}` in record. The record will not be considered when emitting sync state",
+            )
+            return False
+        if isinstance(start_datetime_boundary, str):
+            start_datetime_boundary = self.parse_date(start_datetime_boundary)
+        if isinstance(end_datetime_boundary, str):
+            end_datetime_boundary = self.parse_date(end_datetime_boundary)
+        return start_datetime_boundary <= self.parse_date(record_cursor_value) <= end_datetime_boundary
 
     def _send_log(self, level: Level, message: str) -> None:
         if self.message_repository:

{airbyte-cdk-0.70.2 → airbyte-cdk-0.72.0}/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py

@@ -86,6 +86,11 @@ class PerPartitionCursor(Cursor):
         for state in stream_state["states"]:
             self._cursor_per_partition[self._to_partition_key(state["partition"])] = self._create_cursor(state["cursor"])
 
+    def observe(self, stream_slice: StreamSlice, record: Record) -> None:
+        self._cursor_per_partition[self._to_partition_key(stream_slice.partition)].observe(
+            StreamSlice(partition={}, cursor_slice=stream_slice.cursor_slice), record
+        )
+
     def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None:
         try:
             cursor_most_recent_record = (

{airbyte-cdk-0.70.2 → airbyte-cdk-0.72.0}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -208,6 +208,20 @@ class CustomPartitionRouter(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
 
 
+class CustomSchemaLoader(BaseModel):
+    class Config:
+        extra = Extra.allow
+
+    type: Literal['CustomSchemaLoader']
+    class_name: str = Field(
+        ...,
+        description='Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_<name>.<package>.<class_name>`.',
+        examples=['source_railz.components.MyCustomSchemaLoader'],
+        title='Class Name',
+    )
+    parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters')
+
+
 class CustomTransformation(BaseModel):
     class Config:
         extra = Extra.allow
@@ -1161,7 +1175,9 @@ class DeclarativeStream(BaseModel):
     primary_key: Optional[PrimaryKey] = Field(
         '', description='The primary key of the stream.', title='Primary Key'
     )
-    schema_loader: Optional[Union[InlineSchemaLoader, JsonFileSchemaLoader]] = Field(
+    schema_loader: Optional[
+        Union[InlineSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]
+    ] = Field(
         None,
         description='Component used to retrieve the schema for the current stream.',
         title='Schema Loader',

{airbyte-cdk-0.70.2 → airbyte-cdk-0.72.0}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -49,6 +49,7 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomRecordFilter as CustomRecordFilterModel
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomRequester as CustomRequesterModel
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomRetriever as CustomRetrieverModel
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomSchemaLoader as CustomSchemaLoader
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import CustomTransformation as CustomTransformationModel
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import DatetimeBasedCursor as DatetimeBasedCursorModel
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import DeclarativeStream as DeclarativeStreamModel
@@ -165,6 +166,7 @@ class ModelToComponentFactory:
             CustomRecordFilterModel: self.create_custom_component,
             CustomRequesterModel: self.create_custom_component,
             CustomRetrieverModel: self.create_custom_component,
+            CustomSchemaLoader: self.create_custom_component,
             CustomPaginationStrategyModel: self.create_custom_component,
             CustomPartitionRouterModel: self.create_custom_component,
             CustomTransformationModel: self.create_custom_component,

{airbyte-cdk-0.70.2 → airbyte-cdk-0.72.0}/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py

@@ -322,7 +322,13 @@ class SimpleRetriever(Retriever):
             records_schema=records_schema,
         )
         for stream_data in self._read_pages(record_generator, self.state, _slice):
-            most_recent_record_from_slice = self._get_most_recent_record(most_recent_record_from_slice, stream_data, _slice)
+            current_record = self._extract_record(stream_data, _slice)
+            if self.cursor and current_record:
+                self.cursor.observe(_slice, current_record)
+
+            # TODO this is just the most recent record *read*, not necessarily the most recent record *within slice boundaries*; once all
+            # cursors implement a meaningful `observe` method, it can be removed, both from here and the `Cursor.close_slice` method args
+            most_recent_record_from_slice = self._get_most_recent_record(most_recent_record_from_slice, current_record, _slice)
             yield stream_data
 
         if self.cursor:
@@ -330,13 +336,13 @@ class SimpleRetriever(Retriever):
         return
 
     def _get_most_recent_record(
-        self, current_most_recent: Optional[Record], stream_data: StreamData, stream_slice: StreamSlice
+        self, current_most_recent: Optional[Record], current_record: Optional[Record], stream_slice: StreamSlice
     ) -> Optional[Record]:
-        if self.cursor and (record := self._extract_record(stream_data, stream_slice)):
+        if self.cursor and current_record:
             if not current_most_recent:
-                return record
+                return current_record
             else:
-                return current_most_recent if self.cursor.is_greater_than_or_equal(current_most_recent, record) else record
+                return current_most_recent if self.cursor.is_greater_than_or_equal(current_most_recent, current_record) else current_record
         else:
             return None
 

{airbyte-cdk-0.70.2 → airbyte-cdk-0.72.0}/airbyte_cdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.70.2
+Version: 0.72.0
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.70.2 → airbyte-cdk-0.72.0}/setup.py

@@ -36,7 +36,7 @@ setup(
     name="airbyte-cdk",
     # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be
     # updated if our semver format changes such as using release candidate versions.
-    version="0.70.2",
+    version="0.72.0",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

{airbyte-cdk-0.70.2 → airbyte-cdk-0.72.0}/unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py

@@ -338,55 +338,96 @@ def test_stream_slices(
 
 
 @pytest.mark.parametrize(
-    "test_name, previous_cursor, stream_slice, latest_record_data, expected_state",
+    "test_name, previous_cursor, stream_slice, observed_records, expected_state",
     [
         (
             "test_close_slice_previous_cursor_is_highest",
             "2023-01-01",
-            StreamSlice(partition={}, cursor_slice={"end_time": "2022-01-01"}),
-            {cursor_field: "2021-01-01"},
+            StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-01", "end_time": "2022-01-01"}),
+            [{cursor_field: "2021-01-01"}],
             {cursor_field: "2023-01-01"},
         ),
         (
             "test_close_slice_stream_slice_partition_end_is_highest",
-            "2021-01-01",
-            StreamSlice(partition={}, cursor_slice={"end_time": "2023-01-01"}),
+            "2020-01-01",
+            StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-01", "end_time": "2023-01-01"}),
+            [{cursor_field: "2021-01-01"}],
             {cursor_field: "2021-01-01"},
-            {cursor_field: "2023-01-01"},
         ),
         (
-            "test_close_slice_latest_record_cursor_value_is_highest",
+            "test_close_slice_latest_record_cursor_value_is_higher_than_slice_end",
             "2021-01-01",
-            StreamSlice(partition={}, cursor_slice={"end_time": "2022-01-01"}),
-            {cursor_field: "2023-01-01"},
-            {cursor_field: "2023-01-01"},
+            StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-01", "end_time": "2022-01-01"}),
+            [{cursor_field: "2023-01-01"}],
+            {cursor_field: "2021-01-01"},
         ),
         (
-            "test_close_slice_without_latest_record",
+            "test_close_slice_with_no_records_observed",
             "2021-01-01",
-            StreamSlice(partition={}, cursor_slice={"end_time": "2022-01-01"}),
+            StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-01", "end_time": "2022-01-01"}),
+            [],
+            {cursor_field: "2021-01-01"},
+        ),
+        (
+            "test_close_slice_with_no_records_observed_and_no_previous_state",
             None,
+            StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-01", "end_time": "2022-01-01"}),
+            [],
+            {},
+        ),
+        (
+            "test_close_slice_without_previous_cursor",
+            None,
+            StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-01", "end_time": "2023-01-01"}),
+            [{cursor_field: "2022-01-01"}],
             {cursor_field: "2022-01-01"},
         ),
         (
-            "test_close_slice_without_cursor",
+            "test_close_slice_with_out_of_order_records",
+            "2021-01-01",
+            StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-01", "end_time": "2022-01-01"}),
+            [{cursor_field: "2021-04-01"}, {cursor_field: "2021-02-01"}, {cursor_field: "2021-03-01"}],
+            {cursor_field: "2021-04-01"},
+        ),
+        (
+            "test_close_slice_with_some_records_out_of_slice_boundaries",
+            "2021-01-01",
+            StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-01", "end_time": "2022-01-01"}),
+            [{cursor_field: "2021-02-01"}, {cursor_field: "2021-03-01"}, {cursor_field: "2023-01-01"}],
+            {cursor_field: "2021-03-01"},
+        ),
+        (
+            "test_close_slice_with_all_records_out_of_slice_boundaries",
+            "2021-01-01",
+            StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-01", "end_time": "2022-01-01"}),
+            [{cursor_field: "2023-01-01"}],
+            {cursor_field: "2021-01-01"},
+        ),
+        (
+            "test_close_slice_with_all_records_out_of_slice_and_no_previous_cursor",
             None,
-            StreamSlice(partition={}, cursor_slice={"end_time": "2022-01-01"}),
-            {cursor_field: "2023-01-01"},
-            {cursor_field: "2023-01-01"},
+            StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-01", "end_time": "2022-01-01"}),
+            [{cursor_field: "2023-01-01"}],
+            {},
         ),
     ],
 )
-def test_close_slice(test_name, previous_cursor, stream_slice, latest_record_data, expected_state):
+def test_close_slice(test_name, previous_cursor, stream_slice, observed_records, expected_state):
     cursor = DatetimeBasedCursor(
         start_datetime=MinMaxDatetime(datetime="2021-01-01T00:00:00.000000+0000", parameters={}),
         cursor_field=InterpolatedString(string=cursor_field, parameters={}),
         datetime_format="%Y-%m-%d",
         config=config,
         parameters={},
+        partition_field_start="start_time",
+        partition_field_end="end_time",
     )
-    cursor._cursor = previous_cursor
-    cursor.close_slice(stream_slice, Record(latest_record_data, stream_slice) if latest_record_data else None)
+    cursor.set_initial_state({cursor_field: previous_cursor})
+    for record_data in observed_records:
+        record = Record(record_data, stream_slice)
+        cursor.observe(stream_slice, record)
+    last_record = observed_records[-1] if observed_records else None
+    cursor.close_slice(stream_slice, Record(last_record, stream_slice) if last_record else None)
     updated_state = cursor.get_stream_state()
     assert updated_state == expected_state
 
@@ -404,37 +445,42 @@ def test_close_slice_fails_if_slice_has_a_partition():
         cursor.close_slice(stream_slice, Record({"id": 1}, stream_slice))
 
 
-def test_given_different_format_and_slice_is_highest_when_close_slice_then_slice_datetime_format():
+def test_compares_cursor_values_by_chronological_order():
     cursor = DatetimeBasedCursor(
         start_datetime=MinMaxDatetime(datetime="2021-01-01T00:00:00.000000+0000", parameters={}),
         cursor_field=cursor_field,
-        datetime_format="%Y-%m-%dT%H:%M:%S.%fZ",
-        cursor_datetime_formats=["%Y-%m-%d"],
+        datetime_format="%d-%m-%Y",
         config=config,
         parameters={},
     )
 
-    _slice = StreamSlice(partition={}, cursor_slice={"end_time": "2023-01-04T17:30:19.000Z"})
-    record_cursor_value = "2023-01-03"
-    cursor.close_slice(_slice, Record({cursor_field: record_cursor_value}, _slice))
+    _slice = StreamSlice(partition={}, cursor_slice={"start_time": "01-01-2023", "end_time": "01-04-2023"})
+    first_record = Record({cursor_field: "21-02-2023"}, _slice)
+    cursor.observe(_slice, first_record)
+    second_record = Record({cursor_field: "01-03-2023"}, _slice)
+    cursor.observe(_slice, second_record)
+    cursor.close_slice(_slice, second_record)
 
-    assert cursor.get_stream_state()[cursor_field] == "2023-01-04T17:30:19.000Z"
+    assert cursor.get_stream_state()[cursor_field] == "01-03-2023"
 
 
-def test_given_partition_end_is_specified_and_greater_than_record_when_close_slice_then_use_partition_end():
-    partition_field_end = "partition_field_end"
+def test_given_different_format_and_slice_is_highest_when_close_slice_then_state_uses_record_format():
     cursor = DatetimeBasedCursor(
         start_datetime=MinMaxDatetime(datetime="2021-01-01T00:00:00.000000+0000", parameters={}),
-        cursor_field=InterpolatedString(string=cursor_field, parameters={}),
-        datetime_format="%Y-%m-%d",
-        partition_field_end=partition_field_end,
+        cursor_field=cursor_field,
+        datetime_format="%Y-%m-%dT%H:%M:%S.%fZ",
+        cursor_datetime_formats=["%Y-%m-%d"],
         config=config,
         parameters={},
     )
-    stream_slice = StreamSlice(partition={}, cursor_slice={partition_field_end: "2025-01-01"})
-    cursor.close_slice(stream_slice, Record({cursor_field: "2020-01-01"}, stream_slice))
-    updated_state = cursor.get_stream_state()
-    assert {cursor_field: "2025-01-01"} == updated_state
+
+    _slice = StreamSlice(partition={}, cursor_slice={"start_time": "2023-01-01T17:30:19.000Z", "end_time": "2023-01-04T17:30:19.000Z"})
+    record_cursor_value = "2023-01-03"
+    record = Record({cursor_field: record_cursor_value}, _slice)
+    cursor.observe(_slice, record)
+    cursor.close_slice(_slice, record)
+
+    assert cursor.get_stream_state()[cursor_field] == "2023-01-03"
 
 
 @pytest.mark.parametrize(

{airbyte-cdk-0.70.2 → airbyte-cdk-0.72.0}/unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py

@@ -200,14 +200,14 @@ def test_given_record_for_partition_when_read_then_update_state():
         "states": [
             {
                 "partition": {"partition_field": "1"},
-                "cursor": {CURSOR_FIELD: "2022-01-31"},
+                "cursor": {CURSOR_FIELD: "2022-01-15"},
             }
         ]
     }
 
 
 def test_substream_without_input_state():
-    source = ManifestDeclarativeSource(
+    test_source = ManifestDeclarativeSource(
         source_config=ManifestBuilder()
         .with_substream_partition_router("AnotherStream")
         .with_incremental_sync(
@@ -231,14 +231,14 @@ def test_substream_without_input_state():
         .build()
     )
 
-    stream_instance = source.streams({})[1]
+    stream_instance = test_source.streams({})[1]
 
     stream_slice = StreamSlice(partition={"parent_id": "1"},
                                cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"})
 
     with patch.object(
             SimpleRetriever, "_read_pages", side_effect=[[Record({"id": "1", CURSOR_FIELD: "2022-01-15"}, stream_slice)],
-                                                         Record({"id": "2", CURSOR_FIELD: "2022-01-15"}, stream_slice)]
+                                                         [Record({"id": "2", CURSOR_FIELD: "2022-01-15"}, stream_slice)]]
     ):
         slices = list(stream_instance.stream_slices(sync_mode=SYNC_MODE))
         assert list(slices) == [
@@ -246,6 +246,10 @@ def test_substream_without_input_state():
                         cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"}),
             StreamSlice(partition={"parent_id": "1", "parent_slice": {}, },
                         cursor_slice={"start_time": "2022-02-01", "end_time": "2022-02-28"}),
+            StreamSlice(partition={"parent_id": "2", "parent_slice": {}, },
+                        cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"}),
+            StreamSlice(partition={"parent_id": "2", "parent_slice": {}, },
+                        cursor_slice={"start_time": "2022-02-01", "end_time": "2022-02-28"}),
         ]
 
 
@@ -307,7 +311,7 @@ def test_substream_with_legacy_input_state():
     with patch.object(
             SimpleRetriever, "_read_pages", side_effect=[
                 [Record({"id": "1", CURSOR_FIELD: "2022-01-15"}, stream_slice)],
-                [Record({"parent_id": "1"}, stream_slice)],
+                [Record({"parent_id": "1", CURSOR_FIELD: "2022-01-15"}, stream_slice)],
                 [Record({"id": "2", CURSOR_FIELD: "2022-01-15"}, stream_slice)],
                 [Record({"parent_id": "2", CURSOR_FIELD: "2022-01-15"}, stream_slice)]
             ]
@@ -319,7 +323,7 @@ def test_substream_with_legacy_input_state():
         expected_state = {"states": [
             {
                 "cursor": {
-                    "cursor_field": "2022-01-31"
+                    CURSOR_FIELD: "2022-01-15"
                 },
                 "partition": {"parent_id": "1", "parent_slice": {}}
             }

{airbyte-cdk-0.70.2 → airbyte-cdk-0.72.0}/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py

@@ -5,6 +5,7 @@
 # mypy: ignore-errors
 
 import datetime
+from typing import Any, Mapping
 
 import pytest
 from airbyte_cdk.models import Level
@@ -27,6 +28,7 @@ from airbyte_cdk.sources.declarative.models import CheckStream as CheckStreamMod
 from airbyte_cdk.sources.declarative.models import CompositeErrorHandler as CompositeErrorHandlerModel
 from airbyte_cdk.sources.declarative.models import CustomErrorHandler as CustomErrorHandlerModel
 from airbyte_cdk.sources.declarative.models import CustomPartitionRouter as CustomPartitionRouterModel
+from airbyte_cdk.sources.declarative.models import CustomSchemaLoader as CustomSchemaLoaderModel
 from airbyte_cdk.sources.declarative.models import DatetimeBasedCursor as DatetimeBasedCursorModel
 from airbyte_cdk.sources.declarative.models import DeclarativeStream as DeclarativeStreamModel
 from airbyte_cdk.sources.declarative.models import DefaultPaginator as DefaultPaginatorModel
@@ -66,6 +68,7 @@ from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath
 from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod
 from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever, SimpleRetrieverTestReadDecorator
 from airbyte_cdk.sources.declarative.schema import JsonFileSchemaLoader
+from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
 from airbyte_cdk.sources.declarative.spec import Spec
 from airbyte_cdk.sources.declarative.stream_slicers import CartesianProductStreamSlicer
 from airbyte_cdk.sources.declarative.transformations import AddFields, RemoveFields
@@ -1820,3 +1823,19 @@ def test_create_offset_increment():
     assert strategy.page_size == expected_strategy.page_size
     assert strategy.inject_on_first_request == expected_strategy.inject_on_first_request
     assert strategy.config == input_config
+
+
+class MyCustomSchemaLoader(SchemaLoader):
+    def get_json_schema(self) -> Mapping[str, Any]:
+        """Returns a mapping describing the stream's schema"""
+        return {}
+
+
+def test_create_custom_schema_loader():
+
+    definition = {
+        "type": "CustomSchemaLoader",
+        "class_name": "unit_tests.sources.declarative.parsers.test_model_to_component_factory.MyCustomSchemaLoader"
+    }
+    component = factory.create_component(CustomSchemaLoaderModel, definition, {})
+    assert isinstance(component, MyCustomSchemaLoader)

{airbyte-cdk-0.70.2 → airbyte-cdk-0.72.0}/unit_tests/sources/declarative/retrievers/test_simple_retriever.py

@@ -477,6 +477,7 @@ def test_given_stream_data_is_not_record_when_read_records_then_update_slice_wit
         side_effect=retriever_read_pages,
     ):
         list(retriever.read_records(stream_slice=stream_slice, records_schema={}))
+        cursor.observe.assert_not_called()
         cursor.close_slice.assert_called_once_with(stream_slice, None)