airbyte-cdk 0.40.1__tar.gz → 0.40.3__tar.gz

{airbyte-cdk-0.40.1 → airbyte-cdk-0.40.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.40.1
+Version: 0.40.3
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.40.1 → airbyte-cdk-0.40.3}/airbyte_cdk/connector_builder/message_grouper.py

@@ -14,6 +14,7 @@ from airbyte_cdk.entrypoint import AirbyteEntrypoint
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
 from airbyte_cdk.utils import AirbyteTracedException
+from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer
 from airbyte_cdk.utils.schema_inferrer import SchemaInferrer
 from airbyte_protocol.models.airbyte_protocol import (
     AirbyteControlMessage,
@@ -46,6 +47,7 @@ class MessageGrouper:
         if record_limit is not None and not (1 <= record_limit <= 1000):
             raise ValueError(f"Record limit must be between 1 and 1000. Got {record_limit}")
         schema_inferrer = SchemaInferrer()
+        datetime_format_inferrer = DatetimeFormatInferrer()
 
         if record_limit is None:
             record_limit = self._max_record_limit
@@ -58,6 +60,7 @@ class MessageGrouper:
         for message_group in self._get_message_groups(
                 self._read_stream(source, config, configured_catalog),
                 schema_inferrer,
+                datetime_format_inferrer,
                 record_limit,
         ):
             if isinstance(message_group, AirbyteLogMessage):
@@ -80,10 +83,11 @@ class MessageGrouper:
                 configured_catalog.streams[0].stream.name
             ),  # The connector builder currently only supports reading from a single stream at a time
             latest_config_update=latest_config_update.connectorConfig.config if latest_config_update else self._clean_config(config),
+            inferred_datetime_formats=datetime_format_inferrer.get_inferred_datetime_formats(),
         )
 
     def _get_message_groups(
-            self, messages: Iterator[AirbyteMessage], schema_inferrer: SchemaInferrer, limit: int
+            self, messages: Iterator[AirbyteMessage], schema_inferrer: SchemaInferrer, datetime_format_inferrer: DatetimeFormatInferrer, limit: int
     ) -> Iterable[Union[StreamReadPages, AirbyteControlMessage, AirbyteLogMessage, AirbyteTraceMessage]]:
         """
         Message groups are partitioned according to when request log messages are received. Subsequent response log messages
@@ -141,6 +145,7 @@ class MessageGrouper:
                 current_page_records.append(message.record.data)
                 records_count += 1
                 schema_inferrer.accumulate(message.record)
+                datetime_format_inferrer.accumulate(message.record)
             elif message.type == MessageType.CONTROL and message.control.type == OrchestratorType.CONNECTOR_CONFIG:
                 yield message.control
         else:

{airbyte-cdk-0.40.1 → airbyte-cdk-0.40.3}/airbyte_cdk/connector_builder/models.py

@@ -48,6 +48,7 @@ class StreamRead(object):
     slices: List[StreamReadSlices]
     test_read_limit_reached: bool
     inferred_schema: Optional[Dict[str, Any]]
+    inferred_datetime_formats: Optional[Dict[str, str]]
     latest_config_update: Optional[Dict[str, Any]]
 
 

{airbyte-cdk-0.40.1 → airbyte-cdk-0.40.3}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -27,6 +27,10 @@ properties:
     type: object
   spec:
     "$ref": "#/definitions/Spec"
+  metadata:
+    type: object
+    description: For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.
+    additionalProperties: true
 additionalProperties: false
 definitions:
   AddedFieldDefinition:

{airbyte-cdk-0.40.1 → airbyte-cdk-0.40.3}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -1095,6 +1095,10 @@ class DeclarativeSource(BaseModel):
     schemas: Optional[Schemas] = None
     definitions: Optional[Dict[str, Any]] = None
     spec: Optional[Spec] = None
+    metadata: Optional[Dict[str, Any]] = Field(
+        None,
+        description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.",
+    )
 
 
 class DeclarativeStream(BaseModel):

airbyte-cdk-0.40.3/airbyte_cdk/utils/datetime_format_inferrer.py

@@ -0,0 +1,80 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Any, Dict, Union
+
+from airbyte_cdk.models import AirbyteRecordMessage
+from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
+
+
+class DatetimeFormatInferrer:
+    """
+    This class is used to detect toplevel fields in records that might be datetime values, along with the used format.
+    """
+
+    def __init__(self):
+        self._parser = DatetimeParser()
+        self._datetime_candidates: Union[None, Dict[str, str]] = None
+        self._formats = [
+            "%Y-%m-%d",
+            "%Y-%m-%d %H:%M:%S",
+            "%Y-%m-%d %H:%M:%S.%f+00:00",
+            "%Y-%m-%dT%H:%M:%S.%f%z",
+            "%s",
+            "%d/%m/%Y %H:%M",
+            "%Y-%m",
+            "%d-%m-%Y",
+            "%Y-%m-%dT%H:%M:%SZ",
+        ]
+        self._timestamp_heuristic_range = range(1_000_000_000, 2_000_000_000)
+
+    def _can_be_datetime(self, value: Any) -> bool:
+        """Checks if the value can be a datetime. This is the case if the value is a string or an integer between 1_000_000_000 and 2_000_000_000. This is separate from the format check for performance reasons"""
+        if isinstance(value, str) and (not value.isdecimal() or int(value) in self._timestamp_heuristic_range):
+            return True
+        if isinstance(value, int) and value in self._timestamp_heuristic_range:
+            return True
+        return False
+
+    def _matches_format(self, value: Any, format: str) -> bool:
+        """Checks if the value matches the format"""
+        try:
+            self._parser.parse(value, format)
+            return True
+        except ValueError:
+            return False
+
+    def _initialize(self, record: AirbyteRecordMessage):
+        """Initializes the internal state of the class"""
+        self._datetime_candidates = {}
+        for field_name, field_value in record.data.items():
+            if not self._can_be_datetime(field_value):
+                continue
+            for format in self._formats:
+                if self._matches_format(field_value, format):
+                    self._datetime_candidates[field_name] = format
+                    break
+
+    def _validate(self, record: AirbyteRecordMessage):
+        """Validates that the record is consistent with the inferred datetime formats"""
+        for candidate_field_name in list(self._datetime_candidates.keys()):
+            candidate_field_format = self._datetime_candidates[candidate_field_name]
+            current_value = record.data.get(candidate_field_name, None)
+            if (
+                current_value is None
+                or not self._can_be_datetime(current_value)
+                or not self._matches_format(current_value, candidate_field_format)
+            ):
+                self._datetime_candidates.pop(candidate_field_name)
+
+    def accumulate(self, record: AirbyteRecordMessage):
+        """Analyzes the record and updates the internal state of candidate datetime fields"""
+        self._initialize(record) if self._datetime_candidates is None else self._validate(record)
+
+    def get_inferred_datetime_formats(self) -> Dict[str, str]:
+        """
+        Returns the list of candidate datetime fields - the keys are the field names and the values are the inferred datetime formats.
+        For these fields the format was consistent across all visited records.
+        """
+        return self._datetime_candidates or {}

{airbyte-cdk-0.40.1 → airbyte-cdk-0.40.3}/airbyte_cdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 0.40.1
+Version: 0.40.3
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://github.com/airbytehq/airbyte
 Author: Airbyte

{airbyte-cdk-0.40.1 → airbyte-cdk-0.40.3}/airbyte_cdk.egg-info/SOURCES.txt

@@ -165,6 +165,7 @@ airbyte_cdk/sources/utils/schema_models.py
 airbyte_cdk/sources/utils/transform.py
 airbyte_cdk/utils/__init__.py
 airbyte_cdk/utils/airbyte_secrets_utils.py
+airbyte_cdk/utils/datetime_format_inferrer.py
 airbyte_cdk/utils/event_timing.py
 airbyte_cdk/utils/schema_inferrer.py
 airbyte_cdk/utils/stream_status_utils.py
@@ -267,6 +268,7 @@ unit_tests/sources/streams/http/auth/test_auth.py
 unit_tests/sources/streams/http/requests_native_auth/__init__.py
 unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py
 unit_tests/utils/__init__.py
+unit_tests/utils/test_datetime_format_inferrer.py
 unit_tests/utils/test_schema_inferrer.py
 unit_tests/utils/test_secret_utils.py
 unit_tests/utils/test_stream_status_utils.py

{airbyte-cdk-0.40.1 → airbyte-cdk-0.40.3}/setup.py

@@ -17,7 +17,7 @@ setup(
     name="airbyte-cdk",
     # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be
     # updated if our semver format changes such as using release candidate versions.
-    version="0.40.1",
+    version="0.40.3",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

{airbyte-cdk-0.40.1 → airbyte-cdk-0.40.3}/unit_tests/connector_builder/test_connector_builder_handler.py

@@ -354,6 +354,7 @@ def test_read():
         ],
         test_read_limit_reached=False,
         inferred_schema=None,
+        inferred_datetime_formats=None,
         latest_config_update={}
     )
 
@@ -368,6 +369,7 @@ def test_read():
                 ],
                 "test_read_limit_reached": False,
                 "inferred_schema": None,
+                "inferred_datetime_formats": None,
                 "latest_config_update": {}
             },
             emitted_at=1,
@@ -410,6 +412,7 @@ def test_read_returns_error_response(mock_from_exception):
                                           slice_descriptor=None, state=None)],
                                       test_read_limit_reached=False,
                                       inferred_schema=None,
+                                      inferred_datetime_formats={},
                                       latest_config_update={})
 
     expected_message = AirbyteMessage(

{airbyte-cdk-0.40.1 → airbyte-cdk-0.40.3}/unit_tests/connector_builder/test_message_grouper.py

@@ -94,7 +94,8 @@ def test_get_grouped_messages(mock_entrypoint_read):
         "body": {"custom": "field"},
     }
     response = {"status_code": 200, "headers": {"field": "value"}, "body": '{"name": "field"}', "http_method": "GET"}
-    expected_schema = {"$schema": "http://json-schema.org/schema#", "properties": {"name": {"type": "string"}}, "type": "object"}
+    expected_schema = {"$schema": "http://json-schema.org/schema#", "properties": {"name": {"type": "string"}, "date": {"type": "string"}}, "type": "object"}
+    expected_datetime_fields = {"date":"%Y-%m-%d"}
     expected_pages = [
         StreamReadPages(
             request=HttpRequest(
@@ -105,7 +106,7 @@ def test_get_grouped_messages(mock_entrypoint_read):
                 http_method="GET",
             ),
             response=HttpResponse(status=200, headers={"field": "value"}, body='{"name": "field"}'),
-            records=[{"name": "Shinobu Kocho"}, {"name": "Muichiro Tokito"}],
+            records=[{"name": "Shinobu Kocho", "date": "2023-03-03"}, {"name": "Muichiro Tokito", "date": "2023-03-04"}],
         ),
         StreamReadPages(
             request=HttpRequest(
@@ -116,7 +117,7 @@ def test_get_grouped_messages(mock_entrypoint_read):
                 http_method="GET",
             ),
             response=HttpResponse(status=200, headers={"field": "value"}, body='{"name": "field"}'),
-            records=[{"name": "Mitsuri Kanroji"}],
+            records=[{"name": "Mitsuri Kanroji", "date": "2023-03-05"}],
         ),
     ]
 
@@ -124,11 +125,11 @@ def test_get_grouped_messages(mock_entrypoint_read):
         [
             request_log_message(request),
             response_log_message(response),
-            record_message("hashiras", {"name": "Shinobu Kocho"}),
-            record_message("hashiras", {"name": "Muichiro Tokito"}),
+            record_message("hashiras", {"name": "Shinobu Kocho", "date": "2023-03-03"}),
+            record_message("hashiras", {"name": "Muichiro Tokito", "date": "2023-03-04"}),
             request_log_message(request),
             response_log_message(response),
-            record_message("hashiras", {"name": "Mitsuri Kanroji"}),
+            record_message("hashiras", {"name": "Mitsuri Kanroji", "date": "2023-03-05"}),
         ]
     ))
 
@@ -138,6 +139,7 @@ def test_get_grouped_messages(mock_entrypoint_read):
     )
 
     assert actual_response.inferred_schema == expected_schema
+    assert actual_response.inferred_datetime_formats == expected_datetime_fields
 
     single_slice = actual_response.slices[0]
     for i, actual_page in enumerate(single_slice.pages):

airbyte-cdk-0.40.3/unit_tests/utils/test_datetime_format_inferrer.py

@@ -0,0 +1,53 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Dict, List
+
+import pytest
+from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage
+from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer
+
+NOW = 1234567
+
+
+@pytest.mark.parametrize(
+    "test_name,input_records,expected_candidate_fields",
+    [
+        ("empty", [], {}),
+        ("simple_match", [{"d": "2022-02-03"}], {"d": "%Y-%m-%d"}),
+        ("timestamp_match_integer", [{"d": 1686058051}], {"d": "%s"}),
+        ("timestamp_match_string", [{"d": "1686058051"}], {"d": "%s"}),
+        ("timestamp_no_match_integer", [{"d": 99}], {}),
+        ("timestamp_no_match_string", [{"d": "99999999999999999999"}], {}),
+        ("simple_no_match", [{"d": "20220203"}], {}),
+        ("multiple_match", [{"d": "2022-02-03", "e": "2022-02-03"}], {"d": "%Y-%m-%d", "e": "%Y-%m-%d"}),
+        (
+            "multiple_no_match",
+            [{"d": "20220203", "r": "ccc", "e": {"something-else": "2023-03-03"}, "s": ["2023-03-03"], "x": False, "y": 123}],
+            {},
+        ),
+        ("format_1", [{"d": "2022-02-03"}], {"d": "%Y-%m-%d"}),
+        ("format_2", [{"d": "2022-02-03 12:34:56"}], {"d": "%Y-%m-%d %H:%M:%S"}),
+        ("format_3", [{"d": "2022-02-03 12:34:56.123456+00:00"}], {"d": "%Y-%m-%d %H:%M:%S.%f+00:00"}),
+        ("format_4", [{"d": "2022-02-03T12:34:56.123456+0000"}], {"d": "%Y-%m-%dT%H:%M:%S.%f%z"}),
+        ("format_4 2", [{"d": "2022-02-03T12:34:56.000Z"}], {"d": "%Y-%m-%dT%H:%M:%S.%f%z"}),
+        ("format_4 2", [{"d": "2022-02-03T12:34:56.000000Z"}], {"d": "%Y-%m-%dT%H:%M:%S.%f%z"}),
+        ("format_6", [{"d": "03/02/2022 12:34"}], {"d": "%d/%m/%Y %H:%M"}),
+        ("format_7", [{"d": "2022-02"}], {"d": "%Y-%m"}),
+        ("format_8", [{"d": "03-02-2022"}], {"d": "%d-%m-%Y"}),
+        ("limit_down", [{"d": "2022-02-03", "x": "2022-02-03"}, {"d": "2022-02-03", "x": "another thing"}], {"d": "%Y-%m-%d"}),
+        ("limit_down all", [{"d": "2022-02-03", "x": "2022-02-03"}, {"d": "also another thing", "x": "another thing"}], {}),
+        ("limit_down empty", [{"d": "2022-02-03", "x": "2022-02-03"}, {}], {}),
+        ("limit_down unsupported type", [{"d": "2022-02-03"}, {"d": False}], {}),
+        ("limit_down complex type", [{"d": "2022-02-03"}, {"d": {"date": "2022-03-03"}}], {}),
+        ("limit_down different format", [{"d": "2022-02-03"}, {"d": 1686058051}], {}),
+        ("limit_down different format", [{"d": "2022-02-03"}, {"d": "2022-02-03T12:34:56.000000Z"}], {}),
+        ("no scope expand", [{}, {"d": "2022-02-03"}], {}),
+    ],
+)
+def test_schema_inferrer(test_name, input_records: List, expected_candidate_fields: Dict[str, str]):
+    inferrer = DatetimeFormatInferrer()
+    for record in input_records:
+        inferrer.accumulate(AirbyteRecordMessage(stream="abc", data=record, emitted_at=NOW))
+    assert inferrer.get_inferred_datetime_formats() == expected_candidate_fields