airbyte-cdk 0.40.2__py3-none-any.whl → 0.40.3__py3-none-any.whl

airbyte_cdk/connector_builder/message_grouper.py

@@ -14,6 +14,7 @@ from airbyte_cdk.entrypoint import AirbyteEntrypoint
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
 from airbyte_cdk.utils import AirbyteTracedException
+from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer
 from airbyte_cdk.utils.schema_inferrer import SchemaInferrer
 from airbyte_protocol.models.airbyte_protocol import (
     AirbyteControlMessage,
@@ -46,6 +47,7 @@ class MessageGrouper:
         if record_limit is not None and not (1 <= record_limit <= 1000):
             raise ValueError(f"Record limit must be between 1 and 1000. Got {record_limit}")
         schema_inferrer = SchemaInferrer()
+        datetime_format_inferrer = DatetimeFormatInferrer()
 
         if record_limit is None:
             record_limit = self._max_record_limit
@@ -58,6 +60,7 @@ class MessageGrouper:
         for message_group in self._get_message_groups(
                 self._read_stream(source, config, configured_catalog),
                 schema_inferrer,
+                datetime_format_inferrer,
                 record_limit,
         ):
             if isinstance(message_group, AirbyteLogMessage):
@@ -80,10 +83,11 @@ class MessageGrouper:
                 configured_catalog.streams[0].stream.name
             ),  # The connector builder currently only supports reading from a single stream at a time
             latest_config_update=latest_config_update.connectorConfig.config if latest_config_update else self._clean_config(config),
+            inferred_datetime_formats=datetime_format_inferrer.get_inferred_datetime_formats(),
         )
 
     def _get_message_groups(
-            self, messages: Iterator[AirbyteMessage], schema_inferrer: SchemaInferrer, limit: int
+            self, messages: Iterator[AirbyteMessage], schema_inferrer: SchemaInferrer, datetime_format_inferrer: DatetimeFormatInferrer, limit: int
     ) -> Iterable[Union[StreamReadPages, AirbyteControlMessage, AirbyteLogMessage, AirbyteTraceMessage]]:
         """
         Message groups are partitioned according to when request log messages are received. Subsequent response log messages
@@ -141,6 +145,7 @@ class MessageGrouper:
                 current_page_records.append(message.record.data)
                 records_count += 1
                 schema_inferrer.accumulate(message.record)
+                datetime_format_inferrer.accumulate(message.record)
             elif message.type == MessageType.CONTROL and message.control.type == OrchestratorType.CONNECTOR_CONFIG:
                 yield message.control
         else:

airbyte_cdk/connector_builder/models.py

@@ -48,6 +48,7 @@ class StreamRead(object):
     slices: List[StreamReadSlices]
     test_read_limit_reached: bool
     inferred_schema: Optional[Dict[str, Any]]
+    inferred_datetime_formats: Optional[Dict[str, str]]
     latest_config_update: Optional[Dict[str, Any]]
 
 

airbyte_cdk/utils/datetime_format_inferrer.py

@@ -0,0 +1,80 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Any, Dict, Union
+
+from airbyte_cdk.models import AirbyteRecordMessage
+from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
+
+
+class DatetimeFormatInferrer:
+    """
+    This class is used to detect toplevel fields in records that might be datetime values, along with the used format.
+    """
+
+    def __init__(self):
+        self._parser = DatetimeParser()
+        self._datetime_candidates: Union[None, Dict[str, str]] = None
+        self._formats = [
+            "%Y-%m-%d",
+            "%Y-%m-%d %H:%M:%S",
+            "%Y-%m-%d %H:%M:%S.%f+00:00",
+            "%Y-%m-%dT%H:%M:%S.%f%z",
+            "%s",
+            "%d/%m/%Y %H:%M",
+            "%Y-%m",
+            "%d-%m-%Y",
+            "%Y-%m-%dT%H:%M:%SZ",
+        ]
+        self._timestamp_heuristic_range = range(1_000_000_000, 2_000_000_000)
+
+    def _can_be_datetime(self, value: Any) -> bool:
+        """Checks if the value can be a datetime. This is the case if the value is a string or an integer between 1_000_000_000 and 2_000_000_000. This is separate from the format check for performance reasons"""
+        if isinstance(value, str) and (not value.isdecimal() or int(value) in self._timestamp_heuristic_range):
+            return True
+        if isinstance(value, int) and value in self._timestamp_heuristic_range:
+            return True
+        return False
+
+    def _matches_format(self, value: Any, format: str) -> bool:
+        """Checks if the value matches the format"""
+        try:
+            self._parser.parse(value, format)
+            return True
+        except ValueError:
+            return False
+
+    def _initialize(self, record: AirbyteRecordMessage):
+        """Initializes the internal state of the class"""
+        self._datetime_candidates = {}
+        for field_name, field_value in record.data.items():
+            if not self._can_be_datetime(field_value):
+                continue
+            for format in self._formats:
+                if self._matches_format(field_value, format):
+                    self._datetime_candidates[field_name] = format
+                    break
+
+    def _validate(self, record: AirbyteRecordMessage):
+        """Validates that the record is consistent with the inferred datetime formats"""
+        for candidate_field_name in list(self._datetime_candidates.keys()):
+            candidate_field_format = self._datetime_candidates[candidate_field_name]
+            current_value = record.data.get(candidate_field_name, None)
+            if (
+                current_value is None
+                or not self._can_be_datetime(current_value)
+                or not self._matches_format(current_value, candidate_field_format)
+            ):
+                self._datetime_candidates.pop(candidate_field_name)
+
+    def accumulate(self, record: AirbyteRecordMessage):
+        """Analyzes the record and updates the internal state of candidate datetime fields"""
+        self._initialize(record) if self._datetime_candidates is None else self._validate(record)
+
+    def get_inferred_datetime_formats(self) -> Dict[str, str]:
+        """
+        Returns the list of candidate datetime fields - the keys are the field names and the values are the inferred datetime formats.
+        For these fields the format was consistent across all visited records.
+        """
+        return self._datetime_candidates or {}

{airbyte_cdk-0.40.2.dist-info → airbyte_cdk-0.40.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.40.2
+Version: 0.40.3
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte_cdk-0.40.2.dist-info → airbyte_cdk-0.40.3.dist-info}/RECORD

@@ -8,8 +8,8 @@ airbyte_cdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
 airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=q8mqQjNqpvHZgwVbNuvSe19o4Aw6MQTuhA2URmdz0K0,5443
 airbyte_cdk/connector_builder/main.py,sha256=jn2gqaYAvd6uDoFe0oVhnY23grm5sL-jfIX6kGvhVxk,2994
-airbyte_cdk/connector_builder/message_grouper.py,sha256=uJGOBhinvbisgAa-bQN3XE2L2xFTeVeykLwDCRYcxgc,12110
-airbyte_cdk/connector_builder/models.py,sha256=yW_j91B-3FYNTNbWjR2ZVYTXBHlskT55uxdAqg7FhAE,1221
+airbyte_cdk/connector_builder/message_grouper.py,sha256=yEjvwdXgzYK29xwjl88-4s-J49iaud8_aOrAlOkAzsg,12504
+airbyte_cdk/connector_builder/models.py,sha256=jL2SJIWJTLCbBqobw5Qo8WGS0aN-K9TRmfSpDHM5vYc,1277
 airbyte_cdk/destinations/__init__.py,sha256=0Uxmz3iBAyZJdk_bqUVt2pb0UwRTpFjTnFE6fQFbWKY,126
 airbyte_cdk/destinations/destination.py,sha256=_tIMnKcRQbtIsjVvNOVjfbIxgCNLuBXQwQj8MyVm3BI,5420
 airbyte_cdk/models/__init__.py,sha256=LPQcYdDPwrCXiBPe_jexO4UAcbovIb1V9tHB6I7Un30,633
@@ -156,6 +156,7 @@ airbyte_cdk/sources/utils/schema_models.py,sha256=m1vOqNkkVYGblc492wKo11Zm5FK9F0
 airbyte_cdk/sources/utils/transform.py,sha256=4GYmO6bq33HF-a1in0dKQKqUOYI1bWItyuYF875bSQg,9493
 airbyte_cdk/utils/__init__.py,sha256=kFLcs2P-tbPyeVOJS9rOv1jZdnSpjG24ro0CHgt_CIk,215
 airbyte_cdk/utils/airbyte_secrets_utils.py,sha256=q3aDl8T10ufGbeqnUPqbZLxQcHdkf2kDfQK_upWzBbI,2894
+airbyte_cdk/utils/datetime_format_inferrer.py,sha256=1z5lGq_DI9LFrT68ftlJSqndS6i-Rs1PX7T_RBtOJpA,3443
 airbyte_cdk/utils/event_timing.py,sha256=Hn5kCc9xGKLcV5EYpJCZwNiz9neKKu2WG8FJF_hy278,2377
 airbyte_cdk/utils/schema_inferrer.py,sha256=j0us_mEMj8PVVzSZfoS1adK7V7a--mSHQozo6xmsiIc,3720
 airbyte_cdk/utils/stream_status_utils.py,sha256=X1Vy7BhglycjdIWpfKDfwJussNCxYffelKt6Utjx-qY,1005
@@ -163,8 +164,8 @@ airbyte_cdk/utils/traced_exception.py,sha256=9G2sG9eYkvn6Aa7rMuUW_KIRszRaTc_xdnT
 source_declarative_manifest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 source_declarative_manifest/main.py,sha256=HXzuRsRyhHwPrGU-hc4S7RrgoOoHImqkdfbmO2geBeE,1027
 unit_tests/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-unit_tests/connector_builder/test_connector_builder_handler.py,sha256=V9p7AFECaLqSK-iGvu0OqwV6qREQC2BhWo0H4OoiiK4,26895
-unit_tests/connector_builder/test_message_grouper.py,sha256=XMVRW45RDTgy1YVzkV-jOXj7Ar2mzgDV8OW2QDzZjYU,28510
+unit_tests/connector_builder/test_connector_builder_handler.py,sha256=UtGSzZshZeWZcc5lt3Kt6-8aDFFwj2sLvzjCBfPkrkg,27054
+unit_tests/connector_builder/test_message_grouper.py,sha256=Rek2qmuexLtfsQmHEUR_7FH-eDg3CnFiOOWVUgB9ow8,28802
 unit_tests/connector_builder/utils.py,sha256=AAggdGWP-mNuWOZUHLAVIbjTeIcdPo-3pbMm5zdYpS0,796
 unit_tests/destinations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/destinations/test_destination.py,sha256=koG_j812KMkcIxoUH6XlAL3zsephZJmlHvyzJXm0dCs,10269
@@ -258,12 +259,13 @@ unit_tests/sources/streams/http/auth/test_auth.py,sha256=gdWpJ-cR64qRXmmPOQWhVd4
 unit_tests/sources/streams/http/requests_native_auth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py,sha256=_BZVsG_LZUXfBmHWTlKIw65eGkdwFSiKRlpjsccj61U,12396
 unit_tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+unit_tests/utils/test_datetime_format_inferrer.py,sha256=Io2o5flTre9gyI_IDDMpzxOjCz3sr16LO0GRqOD59uk,2946
 unit_tests/utils/test_schema_inferrer.py,sha256=Z2jHBZ540wnYkylIdV_2xr75Vtwlxuyg4MNPAG-xhpk,7817
 unit_tests/utils/test_secret_utils.py,sha256=XKe0f1RHYii8iwE6ATmBr5JGDI1pzzrnZUGdUSMJQP4,4886
 unit_tests/utils/test_stream_status_utils.py,sha256=NpV155JMXA6CG-2Zvofa14lItobyh3Onttc59X4m5DI,3382
 unit_tests/utils/test_traced_exception.py,sha256=bDFP5zMBizFenz6V2WvEZTRCKGB5ijh3DBezjbfoYIs,4198
-airbyte_cdk-0.40.2.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-0.40.2.dist-info/METADATA,sha256=Ts5OITHn2vkPFfNNDy5V3dylvBgEZXRPIKVmuCvijZw,8902
-airbyte_cdk-0.40.2.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-airbyte_cdk-0.40.2.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
-airbyte_cdk-0.40.2.dist-info/RECORD,,
+airbyte_cdk-0.40.3.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-0.40.3.dist-info/METADATA,sha256=pAfHdGCbN9Iz4q4xcnO3z3sATNNzWz4h7KX5eUQGq1I,8902
+airbyte_cdk-0.40.3.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+airbyte_cdk-0.40.3.dist-info/top_level.txt,sha256=edvsDKTnE6sD2wfCUaeTfKf5gQIL6CPVMwVL2sWZzqo,51
+airbyte_cdk-0.40.3.dist-info/RECORD,,

unit_tests/connector_builder/test_connector_builder_handler.py

@@ -354,6 +354,7 @@ def test_read():
         ],
         test_read_limit_reached=False,
         inferred_schema=None,
+        inferred_datetime_formats=None,
         latest_config_update={}
     )
 
@@ -368,6 +369,7 @@ def test_read():
                 ],
                 "test_read_limit_reached": False,
                 "inferred_schema": None,
+                "inferred_datetime_formats": None,
                 "latest_config_update": {}
             },
             emitted_at=1,
@@ -410,6 +412,7 @@ def test_read_returns_error_response(mock_from_exception):
                                           slice_descriptor=None, state=None)],
                                       test_read_limit_reached=False,
                                       inferred_schema=None,
+                                      inferred_datetime_formats={},
                                       latest_config_update={})
 
     expected_message = AirbyteMessage(

unit_tests/connector_builder/test_message_grouper.py

@@ -94,7 +94,8 @@ def test_get_grouped_messages(mock_entrypoint_read):
         "body": {"custom": "field"},
     }
     response = {"status_code": 200, "headers": {"field": "value"}, "body": '{"name": "field"}', "http_method": "GET"}
-    expected_schema = {"$schema": "http://json-schema.org/schema#", "properties": {"name": {"type": "string"}}, "type": "object"}
+    expected_schema = {"$schema": "http://json-schema.org/schema#", "properties": {"name": {"type": "string"}, "date": {"type": "string"}}, "type": "object"}
+    expected_datetime_fields = {"date":"%Y-%m-%d"}
     expected_pages = [
         StreamReadPages(
             request=HttpRequest(
@@ -105,7 +106,7 @@ def test_get_grouped_messages(mock_entrypoint_read):
                 http_method="GET",
             ),
             response=HttpResponse(status=200, headers={"field": "value"}, body='{"name": "field"}'),
-            records=[{"name": "Shinobu Kocho"}, {"name": "Muichiro Tokito"}],
+            records=[{"name": "Shinobu Kocho", "date": "2023-03-03"}, {"name": "Muichiro Tokito", "date": "2023-03-04"}],
         ),
         StreamReadPages(
             request=HttpRequest(
@@ -116,7 +117,7 @@ def test_get_grouped_messages(mock_entrypoint_read):
                 http_method="GET",
             ),
             response=HttpResponse(status=200, headers={"field": "value"}, body='{"name": "field"}'),
-            records=[{"name": "Mitsuri Kanroji"}],
+            records=[{"name": "Mitsuri Kanroji", "date": "2023-03-05"}],
         ),
     ]
 
@@ -124,11 +125,11 @@ def test_get_grouped_messages(mock_entrypoint_read):
         [
             request_log_message(request),
             response_log_message(response),
-            record_message("hashiras", {"name": "Shinobu Kocho"}),
-            record_message("hashiras", {"name": "Muichiro Tokito"}),
+            record_message("hashiras", {"name": "Shinobu Kocho", "date": "2023-03-03"}),
+            record_message("hashiras", {"name": "Muichiro Tokito", "date": "2023-03-04"}),
             request_log_message(request),
             response_log_message(response),
-            record_message("hashiras", {"name": "Mitsuri Kanroji"}),
+            record_message("hashiras", {"name": "Mitsuri Kanroji", "date": "2023-03-05"}),
         ]
     ))
 
@@ -138,6 +139,7 @@ def test_get_grouped_messages(mock_entrypoint_read):
     )
 
     assert actual_response.inferred_schema == expected_schema
+    assert actual_response.inferred_datetime_formats == expected_datetime_fields
 
     single_slice = actual_response.slices[0]
     for i, actual_page in enumerate(single_slice.pages):

unit_tests/utils/test_datetime_format_inferrer.py

@@ -0,0 +1,53 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Dict, List
+
+import pytest
+from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage
+from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer
+
+NOW = 1234567
+
+
+@pytest.mark.parametrize(
+    "test_name,input_records,expected_candidate_fields",
+    [
+        ("empty", [], {}),
+        ("simple_match", [{"d": "2022-02-03"}], {"d": "%Y-%m-%d"}),
+        ("timestamp_match_integer", [{"d": 1686058051}], {"d": "%s"}),
+        ("timestamp_match_string", [{"d": "1686058051"}], {"d": "%s"}),
+        ("timestamp_no_match_integer", [{"d": 99}], {}),
+        ("timestamp_no_match_string", [{"d": "99999999999999999999"}], {}),
+        ("simple_no_match", [{"d": "20220203"}], {}),
+        ("multiple_match", [{"d": "2022-02-03", "e": "2022-02-03"}], {"d": "%Y-%m-%d", "e": "%Y-%m-%d"}),
+        (
+            "multiple_no_match",
+            [{"d": "20220203", "r": "ccc", "e": {"something-else": "2023-03-03"}, "s": ["2023-03-03"], "x": False, "y": 123}],
+            {},
+        ),
+        ("format_1", [{"d": "2022-02-03"}], {"d": "%Y-%m-%d"}),
+        ("format_2", [{"d": "2022-02-03 12:34:56"}], {"d": "%Y-%m-%d %H:%M:%S"}),
+        ("format_3", [{"d": "2022-02-03 12:34:56.123456+00:00"}], {"d": "%Y-%m-%d %H:%M:%S.%f+00:00"}),
+        ("format_4", [{"d": "2022-02-03T12:34:56.123456+0000"}], {"d": "%Y-%m-%dT%H:%M:%S.%f%z"}),
+        ("format_4 2", [{"d": "2022-02-03T12:34:56.000Z"}], {"d": "%Y-%m-%dT%H:%M:%S.%f%z"}),
+        ("format_4 2", [{"d": "2022-02-03T12:34:56.000000Z"}], {"d": "%Y-%m-%dT%H:%M:%S.%f%z"}),
+        ("format_6", [{"d": "03/02/2022 12:34"}], {"d": "%d/%m/%Y %H:%M"}),
+        ("format_7", [{"d": "2022-02"}], {"d": "%Y-%m"}),
+        ("format_8", [{"d": "03-02-2022"}], {"d": "%d-%m-%Y"}),
+        ("limit_down", [{"d": "2022-02-03", "x": "2022-02-03"}, {"d": "2022-02-03", "x": "another thing"}], {"d": "%Y-%m-%d"}),
+        ("limit_down all", [{"d": "2022-02-03", "x": "2022-02-03"}, {"d": "also another thing", "x": "another thing"}], {}),
+        ("limit_down empty", [{"d": "2022-02-03", "x": "2022-02-03"}, {}], {}),
+        ("limit_down unsupported type", [{"d": "2022-02-03"}, {"d": False}], {}),
+        ("limit_down complex type", [{"d": "2022-02-03"}, {"d": {"date": "2022-03-03"}}], {}),
+        ("limit_down different format", [{"d": "2022-02-03"}, {"d": 1686058051}], {}),
+        ("limit_down different format", [{"d": "2022-02-03"}, {"d": "2022-02-03T12:34:56.000000Z"}], {}),
+        ("no scope expand", [{}, {"d": "2022-02-03"}], {}),
+    ],
+)
+def test_schema_inferrer(test_name, input_records: List, expected_candidate_fields: Dict[str, str]):
+    inferrer = DatetimeFormatInferrer()
+    for record in input_records:
+        inferrer.accumulate(AirbyteRecordMessage(stream="abc", data=record, emitted_at=NOW))
+    assert inferrer.get_inferred_datetime_formats() == expected_candidate_fields