airbyte-cdk 6.27.2__py3-none-any.whl → 6.28.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/cli/source_declarative_manifest/_run.py +3 -3
- airbyte_cdk/connector_builder/connector_builder_handler.py +2 -2
- airbyte_cdk/sources/declarative/auth/oauth.py +22 -13
- airbyte_cdk/sources/declarative/auth/token_provider.py +4 -5
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +22 -13
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +17 -6
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +24 -12
- airbyte_cdk/sources/utils/transform.py +23 -2
- airbyte_cdk/utils/datetime_helpers.py +517 -0
- airbyte_cdk-6.28.0.dist-info/LICENSE_SHORT +1 -0
- {airbyte_cdk-6.27.2.dist-info → airbyte_cdk-6.28.0.dist-info}/METADATA +5 -4
- {airbyte_cdk-6.27.2.dist-info → airbyte_cdk-6.28.0.dist-info}/RECORD +15 -13
- {airbyte_cdk-6.27.2.dist-info → airbyte_cdk-6.28.0.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.27.2.dist-info → airbyte_cdk-6.28.0.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.27.2.dist-info → airbyte_cdk-6.28.0.dist-info}/entry_points.txt +0 -0
@@ -21,7 +21,6 @@ import pkgutil
|
|
21
21
|
import sys
|
22
22
|
import traceback
|
23
23
|
from collections.abc import Mapping
|
24
|
-
from datetime import datetime
|
25
24
|
from pathlib import Path
|
26
25
|
from typing import Any, cast
|
27
26
|
|
@@ -44,6 +43,7 @@ from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
|
|
44
43
|
)
|
45
44
|
from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
|
46
45
|
from airbyte_cdk.sources.source import TState
|
46
|
+
from airbyte_cdk.utils.datetime_helpers import ab_datetime_now
|
47
47
|
|
48
48
|
|
49
49
|
class SourceLocalYaml(YamlDeclarativeSource):
|
@@ -101,7 +101,7 @@ def _get_local_yaml_source(args: list[str]) -> SourceLocalYaml:
|
|
101
101
|
type=Type.TRACE,
|
102
102
|
trace=AirbyteTraceMessage(
|
103
103
|
type=TraceType.ERROR,
|
104
|
-
emitted_at=
|
104
|
+
emitted_at=ab_datetime_now().to_epoch_millis(),
|
105
105
|
error=AirbyteErrorTraceMessage(
|
106
106
|
message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
|
107
107
|
stack_trace=traceback.format_exc(),
|
@@ -191,7 +191,7 @@ def create_declarative_source(
|
|
191
191
|
type=Type.TRACE,
|
192
192
|
trace=AirbyteTraceMessage(
|
193
193
|
type=TraceType.ERROR,
|
194
|
-
emitted_at=
|
194
|
+
emitted_at=ab_datetime_now().to_epoch_millis(),
|
195
195
|
error=AirbyteErrorTraceMessage(
|
196
196
|
message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
|
197
197
|
stack_trace=traceback.format_exc(),
|
@@ -3,7 +3,6 @@
|
|
3
3
|
#
|
4
4
|
|
5
5
|
import dataclasses
|
6
|
-
from datetime import datetime
|
7
6
|
from typing import Any, List, Mapping
|
8
7
|
|
9
8
|
from airbyte_cdk.connector_builder.message_grouper import MessageGrouper
|
@@ -21,6 +20,7 @@ from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
|
|
21
20
|
ModelToComponentFactory,
|
22
21
|
)
|
23
22
|
from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
|
23
|
+
from airbyte_cdk.utils.datetime_helpers import ab_datetime_now
|
24
24
|
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
25
25
|
|
26
26
|
DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE = 5
|
@@ -114,4 +114,4 @@ def resolve_manifest(source: ManifestDeclarativeSource) -> AirbyteMessage:
|
|
114
114
|
|
115
115
|
|
116
116
|
def _emitted_at() -> int:
|
117
|
-
return
|
117
|
+
return ab_datetime_now().to_epoch_millis()
|
@@ -3,10 +3,9 @@
|
|
3
3
|
#
|
4
4
|
|
5
5
|
from dataclasses import InitVar, dataclass, field
|
6
|
+
from datetime import timedelta
|
6
7
|
from typing import Any, List, Mapping, MutableMapping, Optional, Union
|
7
8
|
|
8
|
-
import pendulum
|
9
|
-
|
10
9
|
from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
|
11
10
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
|
12
11
|
from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
|
@@ -18,6 +17,7 @@ from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth import
|
|
18
17
|
from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import (
|
19
18
|
SingleUseRefreshTokenOauth2Authenticator,
|
20
19
|
)
|
20
|
+
from airbyte_cdk.utils.datetime_helpers import AirbyteDateTime, ab_datetime_now, ab_datetime_parse
|
21
21
|
|
22
22
|
|
23
23
|
@dataclass
|
@@ -53,7 +53,7 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
|
|
53
53
|
refresh_token: Optional[Union[InterpolatedString, str]] = None
|
54
54
|
scopes: Optional[List[str]] = None
|
55
55
|
token_expiry_date: Optional[Union[InterpolatedString, str]] = None
|
56
|
-
_token_expiry_date: Optional[
|
56
|
+
_token_expiry_date: Optional[AirbyteDateTime] = field(init=False, repr=False, default=None)
|
57
57
|
token_expiry_date_format: Optional[str] = None
|
58
58
|
token_expiry_is_time_of_expiration: bool = False
|
59
59
|
access_token_name: Union[InterpolatedString, str] = "access_token"
|
@@ -122,15 +122,24 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
|
|
122
122
|
self._refresh_request_headers = InterpolatedMapping(
|
123
123
|
self.refresh_request_headers or {}, parameters=parameters
|
124
124
|
)
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
125
|
+
try:
|
126
|
+
if (
|
127
|
+
isinstance(self.token_expiry_date, (int, str))
|
128
|
+
and str(self.token_expiry_date).isdigit()
|
129
|
+
):
|
130
|
+
self._token_expiry_date = ab_datetime_parse(self.token_expiry_date)
|
131
|
+
else:
|
132
|
+
self._token_expiry_date = (
|
133
|
+
ab_datetime_parse(
|
134
|
+
InterpolatedString.create(
|
135
|
+
self.token_expiry_date, parameters=parameters
|
136
|
+
).eval(self.config)
|
137
|
+
)
|
138
|
+
if self.token_expiry_date
|
139
|
+
else ab_datetime_now() - timedelta(days=1)
|
129
140
|
)
|
130
|
-
|
131
|
-
|
132
|
-
else pendulum.now().subtract(days=1) # type: ignore # substract does not have type hints
|
133
|
-
)
|
141
|
+
except ValueError as e:
|
142
|
+
raise ValueError(f"Invalid token expiry date format: {e}")
|
134
143
|
self.use_profile_assertion = (
|
135
144
|
InterpolatedBoolean(self.use_profile_assertion, parameters=parameters)
|
136
145
|
if isinstance(self.use_profile_assertion, str)
|
@@ -222,8 +231,8 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
|
|
222
231
|
def get_refresh_request_headers(self) -> Mapping[str, Any]:
|
223
232
|
return self._refresh_request_headers.eval(self.config)
|
224
233
|
|
225
|
-
def get_token_expiry_date(self) ->
|
226
|
-
return self._token_expiry_date # type: ignore # _token_expiry_date is
|
234
|
+
def get_token_expiry_date(self) -> AirbyteDateTime:
|
235
|
+
return self._token_expiry_date # type: ignore # _token_expiry_date is an AirbyteDateTime. It is never None despite what mypy thinks
|
227
236
|
|
228
237
|
def set_token_expiry_date(self, value: Union[str, int]) -> None:
|
229
238
|
self._token_expiry_date = self._parse_token_expiration_date(value)
|
@@ -9,9 +9,7 @@ from dataclasses import InitVar, dataclass, field
|
|
9
9
|
from typing import Any, List, Mapping, Optional, Union
|
10
10
|
|
11
11
|
import dpath
|
12
|
-
import pendulum
|
13
12
|
from isodate import Duration
|
14
|
-
from pendulum import DateTime
|
15
13
|
|
16
14
|
from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
|
17
15
|
from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder
|
@@ -21,6 +19,7 @@ from airbyte_cdk.sources.declarative.requesters.requester import Requester
|
|
21
19
|
from airbyte_cdk.sources.http_logger import format_http_message
|
22
20
|
from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
|
23
21
|
from airbyte_cdk.sources.types import Config
|
22
|
+
from airbyte_cdk.utils.datetime_helpers import AirbyteDateTime, ab_datetime_now
|
24
23
|
|
25
24
|
|
26
25
|
class TokenProvider:
|
@@ -38,7 +37,7 @@ class SessionTokenProvider(TokenProvider):
|
|
38
37
|
message_repository: MessageRepository = NoopMessageRepository()
|
39
38
|
decoder: Decoder = field(default_factory=lambda: JsonDecoder(parameters={}))
|
40
39
|
|
41
|
-
_next_expiration_time: Optional[
|
40
|
+
_next_expiration_time: Optional[AirbyteDateTime] = None
|
42
41
|
_token: Optional[str] = None
|
43
42
|
|
44
43
|
def get_token(self) -> str:
|
@@ -48,7 +47,7 @@ class SessionTokenProvider(TokenProvider):
|
|
48
47
|
return self._token
|
49
48
|
|
50
49
|
def _refresh_if_necessary(self) -> None:
|
51
|
-
if self._next_expiration_time is None or self._next_expiration_time <
|
50
|
+
if self._next_expiration_time is None or self._next_expiration_time < ab_datetime_now():
|
52
51
|
self._refresh()
|
53
52
|
|
54
53
|
def _refresh(self) -> None:
|
@@ -65,7 +64,7 @@ class SessionTokenProvider(TokenProvider):
|
|
65
64
|
raise ReadException("Failed to get session token, response got ignored by requester")
|
66
65
|
session_token = dpath.get(next(self.decoder.decode(response)), self.session_token_path)
|
67
66
|
if self.expiration_duration is not None:
|
68
|
-
self._next_expiration_time =
|
67
|
+
self._next_expiration_time = ab_datetime_now() + self.expiration_duration
|
69
68
|
self._token = session_token # type: ignore # Returned decoded response will be Mapping and therefore session_token will be str or None
|
70
69
|
|
71
70
|
|
@@ -6,9 +6,6 @@ from abc import abstractmethod
|
|
6
6
|
from datetime import datetime, timedelta, timezone
|
7
7
|
from typing import Any, Callable, List, MutableMapping, Optional, Tuple
|
8
8
|
|
9
|
-
import pendulum
|
10
|
-
from pendulum.datetime import DateTime
|
11
|
-
|
12
9
|
# FIXME We would eventually like the Concurrent package do be agnostic of the declarative package. However, this is a breaking change and
|
13
10
|
# the goal in the short term is only to fix the issue we are seeing for source-declarative-manifest.
|
14
11
|
from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
|
@@ -17,6 +14,7 @@ from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_sta
|
|
17
14
|
AbstractStreamStateConverter,
|
18
15
|
ConcurrencyCompatibleStateType,
|
19
16
|
)
|
17
|
+
from airbyte_cdk.utils.datetime_helpers import AirbyteDateTime, ab_datetime_now, ab_datetime_parse
|
20
18
|
|
21
19
|
|
22
20
|
class DateTimeStreamStateConverter(AbstractStreamStateConverter):
|
@@ -36,7 +34,7 @@ class DateTimeStreamStateConverter(AbstractStreamStateConverter):
|
|
36
34
|
|
37
35
|
@classmethod
|
38
36
|
def get_end_provider(cls) -> Callable[[], datetime]:
|
39
|
-
return
|
37
|
+
return ab_datetime_now
|
40
38
|
|
41
39
|
@abstractmethod
|
42
40
|
def increment(self, timestamp: datetime) -> datetime: ...
|
@@ -136,10 +134,10 @@ class EpochValueConcurrentStreamStateConverter(DateTimeStreamStateConverter):
|
|
136
134
|
return int(timestamp.timestamp())
|
137
135
|
|
138
136
|
def parse_timestamp(self, timestamp: int) -> datetime:
|
139
|
-
dt_object =
|
140
|
-
if not isinstance(dt_object,
|
137
|
+
dt_object = AirbyteDateTime.fromtimestamp(timestamp, timezone.utc)
|
138
|
+
if not isinstance(dt_object, AirbyteDateTime):
|
141
139
|
raise ValueError(
|
142
|
-
f"
|
140
|
+
f"AirbyteDateTime object was expected but got {type(dt_object)} from AirbyteDateTime.fromtimestamp({timestamp})"
|
143
141
|
)
|
144
142
|
return dt_object
|
145
143
|
|
@@ -169,14 +167,25 @@ class IsoMillisConcurrentStreamStateConverter(DateTimeStreamStateConverter):
|
|
169
167
|
def increment(self, timestamp: datetime) -> datetime:
|
170
168
|
return timestamp + self._cursor_granularity
|
171
169
|
|
172
|
-
def output_format(self, timestamp: datetime) ->
|
173
|
-
|
170
|
+
def output_format(self, timestamp: datetime) -> str:
|
171
|
+
"""Format datetime with milliseconds always included.
|
172
|
+
|
173
|
+
Args:
|
174
|
+
timestamp: The datetime to format.
|
175
|
+
|
176
|
+
Returns:
|
177
|
+
str: ISO8601/RFC3339 formatted string with milliseconds.
|
178
|
+
"""
|
179
|
+
dt = AirbyteDateTime.from_datetime(timestamp)
|
180
|
+
# Always include milliseconds, even if zero
|
181
|
+
millis = dt.microsecond // 1000 if dt.microsecond else 0
|
182
|
+
return f"{dt.year:04d}-{dt.month:02d}-{dt.day:02d}T{dt.hour:02d}:{dt.minute:02d}:{dt.second:02d}.{millis:03d}Z"
|
174
183
|
|
175
184
|
def parse_timestamp(self, timestamp: str) -> datetime:
|
176
|
-
dt_object =
|
177
|
-
if not isinstance(dt_object,
|
185
|
+
dt_object = ab_datetime_parse(timestamp)
|
186
|
+
if not isinstance(dt_object, AirbyteDateTime):
|
178
187
|
raise ValueError(
|
179
|
-
f"
|
188
|
+
f"AirbyteDateTime object was expected but got {type(dt_object)} from parse({timestamp})"
|
180
189
|
)
|
181
190
|
return dt_object
|
182
191
|
|
@@ -184,7 +193,7 @@ class IsoMillisConcurrentStreamStateConverter(DateTimeStreamStateConverter):
|
|
184
193
|
class CustomFormatConcurrentStreamStateConverter(IsoMillisConcurrentStreamStateConverter):
|
185
194
|
"""
|
186
195
|
Datetime State converter that emits state according to the supplied datetime format. The converter supports reading
|
187
|
-
incoming state in any valid datetime format
|
196
|
+
incoming state in any valid datetime format using AirbyteDateTime parsing utilities.
|
188
197
|
"""
|
189
198
|
|
190
199
|
def __init__(
|
@@ -4,11 +4,11 @@
|
|
4
4
|
|
5
5
|
import logging
|
6
6
|
from abc import abstractmethod
|
7
|
+
from datetime import timedelta
|
7
8
|
from json import JSONDecodeError
|
8
9
|
from typing import Any, List, Mapping, MutableMapping, Optional, Tuple, Union
|
9
10
|
|
10
11
|
import backoff
|
11
|
-
import pendulum
|
12
12
|
import requests
|
13
13
|
from requests.auth import AuthBase
|
14
14
|
|
@@ -17,6 +17,7 @@ from airbyte_cdk.sources.http_logger import format_http_message
|
|
17
17
|
from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
|
18
18
|
from airbyte_cdk.utils import AirbyteTracedException
|
19
19
|
from airbyte_cdk.utils.airbyte_secrets_utils import add_to_secrets
|
20
|
+
from airbyte_cdk.utils.datetime_helpers import AirbyteDateTime, ab_datetime_now, ab_datetime_parse
|
20
21
|
|
21
22
|
from ..exceptions import DefaultBackoffException
|
22
23
|
|
@@ -72,7 +73,7 @@ class AbstractOauth2Authenticator(AuthBase):
|
|
72
73
|
|
73
74
|
def token_has_expired(self) -> bool:
|
74
75
|
"""Returns True if the token is expired"""
|
75
|
-
return
|
76
|
+
return ab_datetime_now() > self.get_token_expiry_date()
|
76
77
|
|
77
78
|
def build_refresh_request_body(self) -> Mapping[str, Any]:
|
78
79
|
"""
|
@@ -179,7 +180,7 @@ class AbstractOauth2Authenticator(AuthBase):
|
|
179
180
|
self.get_expires_in_name()
|
180
181
|
]
|
181
182
|
|
182
|
-
def _parse_token_expiration_date(self, value: Union[str, int]) ->
|
183
|
+
def _parse_token_expiration_date(self, value: Union[str, int]) -> AirbyteDateTime:
|
183
184
|
"""
|
184
185
|
Return the expiration datetime of the refresh token
|
185
186
|
|
@@ -191,9 +192,19 @@ class AbstractOauth2Authenticator(AuthBase):
|
|
191
192
|
raise ValueError(
|
192
193
|
f"Invalid token expiry date format {self.token_expiry_date_format}; a string representing the format is required."
|
193
194
|
)
|
194
|
-
|
195
|
+
try:
|
196
|
+
return ab_datetime_parse(str(value))
|
197
|
+
except ValueError as e:
|
198
|
+
raise ValueError(f"Invalid token expiry date format: {e}")
|
195
199
|
else:
|
196
|
-
|
200
|
+
try:
|
201
|
+
# Only accept numeric values (as int/float/string) when no format specified
|
202
|
+
seconds = int(float(str(value)))
|
203
|
+
return ab_datetime_now() + timedelta(seconds=seconds)
|
204
|
+
except (ValueError, TypeError):
|
205
|
+
raise ValueError(
|
206
|
+
f"Invalid expires_in value: {value}. Expected number of seconds when no format specified."
|
207
|
+
)
|
197
208
|
|
198
209
|
@property
|
199
210
|
def token_expiry_is_time_of_expiration(self) -> bool:
|
@@ -244,7 +255,7 @@ class AbstractOauth2Authenticator(AuthBase):
|
|
244
255
|
"""List of requested scopes"""
|
245
256
|
|
246
257
|
@abstractmethod
|
247
|
-
def get_token_expiry_date(self) ->
|
258
|
+
def get_token_expiry_date(self) -> AirbyteDateTime:
|
248
259
|
"""Expiration date of the access token"""
|
249
260
|
|
250
261
|
@abstractmethod
|
@@ -2,10 +2,10 @@
|
|
2
2
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
3
|
#
|
4
4
|
|
5
|
+
from datetime import timedelta
|
5
6
|
from typing import Any, List, Mapping, Optional, Sequence, Tuple, Union
|
6
7
|
|
7
8
|
import dpath
|
8
|
-
import pendulum
|
9
9
|
|
10
10
|
from airbyte_cdk.config_observation import (
|
11
11
|
create_connector_config_control_message,
|
@@ -15,6 +15,11 @@ from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
|
|
15
15
|
from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth import (
|
16
16
|
AbstractOauth2Authenticator,
|
17
17
|
)
|
18
|
+
from airbyte_cdk.utils.datetime_helpers import (
|
19
|
+
AirbyteDateTime,
|
20
|
+
ab_datetime_now,
|
21
|
+
ab_datetime_parse,
|
22
|
+
)
|
18
23
|
|
19
24
|
|
20
25
|
class Oauth2Authenticator(AbstractOauth2Authenticator):
|
@@ -34,7 +39,7 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
|
|
34
39
|
client_secret_name: str = "client_secret",
|
35
40
|
refresh_token_name: str = "refresh_token",
|
36
41
|
scopes: List[str] | None = None,
|
37
|
-
token_expiry_date:
|
42
|
+
token_expiry_date: AirbyteDateTime | None = None,
|
38
43
|
token_expiry_date_format: str | None = None,
|
39
44
|
access_token_name: str = "access_token",
|
40
45
|
expires_in_name: str = "expires_in",
|
@@ -62,7 +67,7 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
|
|
62
67
|
self._grant_type_name = grant_type_name
|
63
68
|
self._grant_type = grant_type
|
64
69
|
|
65
|
-
self._token_expiry_date = token_expiry_date or
|
70
|
+
self._token_expiry_date = token_expiry_date or (ab_datetime_now() - timedelta(days=1))
|
66
71
|
self._token_expiry_date_format = token_expiry_date_format
|
67
72
|
self._token_expiry_is_time_of_expiration = token_expiry_is_time_of_expiration
|
68
73
|
self._access_token = None
|
@@ -112,7 +117,7 @@ class Oauth2Authenticator(AbstractOauth2Authenticator):
|
|
112
117
|
def get_grant_type(self) -> str:
|
113
118
|
return self._grant_type
|
114
119
|
|
115
|
-
def get_token_expiry_date(self) ->
|
120
|
+
def get_token_expiry_date(self) -> AirbyteDateTime:
|
116
121
|
return self._token_expiry_date
|
117
122
|
|
118
123
|
def set_token_expiry_date(self, value: Union[str, int]) -> None:
|
@@ -276,17 +281,24 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
|
|
276
281
|
new_refresh_token,
|
277
282
|
)
|
278
283
|
|
279
|
-
def get_token_expiry_date(self) ->
|
284
|
+
def get_token_expiry_date(self) -> AirbyteDateTime:
|
280
285
|
expiry_date = dpath.get(
|
281
286
|
self._connector_config, # type: ignore[arg-type]
|
282
287
|
self._token_expiry_date_config_path,
|
283
288
|
default="",
|
284
289
|
)
|
285
|
-
|
290
|
+
result = (
|
291
|
+
ab_datetime_now() - timedelta(days=1)
|
292
|
+
if expiry_date == ""
|
293
|
+
else ab_datetime_parse(str(expiry_date))
|
294
|
+
)
|
295
|
+
if isinstance(result, AirbyteDateTime):
|
296
|
+
return result
|
297
|
+
raise TypeError("Invalid datetime conversion")
|
286
298
|
|
287
299
|
def set_token_expiry_date( # type: ignore[override]
|
288
300
|
self,
|
289
|
-
new_token_expiry_date:
|
301
|
+
new_token_expiry_date: AirbyteDateTime,
|
290
302
|
) -> None:
|
291
303
|
dpath.new(
|
292
304
|
self._connector_config, # type: ignore[arg-type]
|
@@ -296,17 +308,17 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
|
|
296
308
|
|
297
309
|
def token_has_expired(self) -> bool:
|
298
310
|
"""Returns True if the token is expired"""
|
299
|
-
return
|
311
|
+
return ab_datetime_now() > self.get_token_expiry_date()
|
300
312
|
|
301
313
|
@staticmethod
|
302
314
|
def get_new_token_expiry_date(
|
303
315
|
access_token_expires_in: str,
|
304
316
|
token_expiry_date_format: str | None = None,
|
305
|
-
) ->
|
317
|
+
) -> AirbyteDateTime:
|
306
318
|
if token_expiry_date_format:
|
307
|
-
return
|
319
|
+
return ab_datetime_parse(access_token_expires_in)
|
308
320
|
else:
|
309
|
-
return
|
321
|
+
return ab_datetime_now() + timedelta(seconds=int(access_token_expires_in))
|
310
322
|
|
311
323
|
def get_access_token(self) -> str:
|
312
324
|
"""Retrieve new access and refresh token if the access token has expired.
|
@@ -318,7 +330,7 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator):
|
|
318
330
|
new_access_token, access_token_expires_in, new_refresh_token = (
|
319
331
|
self.refresh_access_token()
|
320
332
|
)
|
321
|
-
new_token_expiry_date:
|
333
|
+
new_token_expiry_date: AirbyteDateTime = self.get_new_token_expiry_date(
|
322
334
|
access_token_expires_in, self._token_expiry_date_format
|
323
335
|
)
|
324
336
|
self.access_token = new_access_token
|
@@ -3,7 +3,6 @@
|
|
3
3
|
#
|
4
4
|
|
5
5
|
import logging
|
6
|
-
from distutils.util import strtobool
|
7
6
|
from enum import Flag, auto
|
8
7
|
from typing import Any, Callable, Dict, Generator, Mapping, Optional, cast
|
9
8
|
|
@@ -22,6 +21,28 @@ python_to_json = {v: k for k, v in json_to_python.items()}
|
|
22
21
|
|
23
22
|
logger = logging.getLogger("airbyte")
|
24
23
|
|
24
|
+
_TRUTHY_STRINGS = ("y", "yes", "t", "true", "on", "1")
|
25
|
+
_FALSEY_STRINGS = ("n", "no", "f", "false", "off", "0")
|
26
|
+
|
27
|
+
|
28
|
+
def _strtobool(value: str, /) -> int:
|
29
|
+
"""Mimic the behavior of distutils.util.strtobool.
|
30
|
+
|
31
|
+
From: https://docs.python.org/2/distutils/apiref.html#distutils.util.strtobool
|
32
|
+
|
33
|
+
> Convert a string representation of truth to true (1) or false (0).
|
34
|
+
> True values are y, yes, t, true, on and 1; false values are n, no, f, false, off and 0. Raises
|
35
|
+
> `ValueError` if val is anything else.
|
36
|
+
"""
|
37
|
+
normalized_str = value.lower().strip()
|
38
|
+
if normalized_str in _TRUTHY_STRINGS:
|
39
|
+
return 1
|
40
|
+
|
41
|
+
if normalized_str in _FALSEY_STRINGS:
|
42
|
+
return 0
|
43
|
+
|
44
|
+
raise ValueError(f"Invalid boolean value: {normalized_str}")
|
45
|
+
|
25
46
|
|
26
47
|
class TransformConfig(Flag):
|
27
48
|
"""
|
@@ -129,7 +150,7 @@ class TypeTransformer:
|
|
129
150
|
return int(original_item)
|
130
151
|
elif target_type == "boolean":
|
131
152
|
if isinstance(original_item, str):
|
132
|
-
return
|
153
|
+
return _strtobool(original_item) == 1
|
133
154
|
return bool(original_item)
|
134
155
|
elif target_type == "array":
|
135
156
|
item_types = set(subschema.get("items", {}).get("type", set()))
|
@@ -0,0 +1,517 @@
|
|
1
|
+
"""Provides consistent datetime handling across Airbyte with ISO8601/RFC3339 compliance.
|
2
|
+
|
3
|
+
Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
4
|
+
|
5
|
+
This module provides a custom datetime class (AirbyteDateTime) and helper functions that ensure
|
6
|
+
consistent datetime handling across Airbyte. All datetime strings are formatted according to
|
7
|
+
ISO8601/RFC3339 standards with 'T' delimiter and '+00:00' for UTC timezone.
|
8
|
+
|
9
|
+
Key Features:
|
10
|
+
- Timezone-aware datetime objects (defaults to UTC)
|
11
|
+
- ISO8601/RFC3339 compliant string formatting
|
12
|
+
- Consistent parsing of various datetime formats
|
13
|
+
- Support for Unix timestamps and milliseconds
|
14
|
+
- Type-safe datetime arithmetic with timedelta
|
15
|
+
|
16
|
+
## Basic Usage
|
17
|
+
|
18
|
+
```python
|
19
|
+
from airbyte_cdk.utils.datetime_helpers import AirbyteDateTime, ab_datetime_now, ab_datetime_parse
|
20
|
+
from datetime import timedelta, timezone
|
21
|
+
|
22
|
+
## Current time in UTC
|
23
|
+
now = ab_datetime_now()
|
24
|
+
print(now) # 2023-03-14T15:09:26.535897Z
|
25
|
+
|
26
|
+
# Parse various datetime formats
|
27
|
+
dt = ab_datetime_parse("2023-03-14T15:09:26Z") # ISO8601/RFC3339
|
28
|
+
dt = ab_datetime_parse("2023-03-14") # Date only (assumes midnight UTC)
|
29
|
+
dt = ab_datetime_parse(1678806566) # Unix timestamp
|
30
|
+
|
31
|
+
## Create with explicit timezone
|
32
|
+
dt = AirbyteDateTime(2023, 3, 14, 15, 9, 26, tzinfo=timezone.utc)
|
33
|
+
print(dt) # 2023-03-14T15:09:26+00:00
|
34
|
+
|
35
|
+
# Datetime arithmetic with timedelta
|
36
|
+
tomorrow = dt + timedelta(days=1)
|
37
|
+
yesterday = dt - timedelta(days=1)
|
38
|
+
time_diff = tomorrow - yesterday # timedelta object
|
39
|
+
```
|
40
|
+
|
41
|
+
## Millisecond Timestamp Handling
|
42
|
+
|
43
|
+
```python
|
44
|
+
# Convert to millisecond timestamp
|
45
|
+
dt = ab_datetime_parse("2023-03-14T15:09:26Z")
|
46
|
+
ms = dt.to_epoch_millis() # 1678806566000
|
47
|
+
|
48
|
+
# Create from millisecond timestamp
|
49
|
+
dt = AirbyteDateTime.from_epoch_millis(1678806566000)
|
50
|
+
print(dt) # 2023-03-14T15:09:26Z
|
51
|
+
```
|
52
|
+
|
53
|
+
## Timezone Handling
|
54
|
+
|
55
|
+
```python
|
56
|
+
# Create with non-UTC timezone
|
57
|
+
tz = timezone(timedelta(hours=-4)) # EDT
|
58
|
+
dt = AirbyteDateTime(2023, 3, 14, 15, 9, 26, tzinfo=tz)
|
59
|
+
print(dt) # 2023-03-14T15:09:26-04:00
|
60
|
+
|
61
|
+
## Parse with timezone
|
62
|
+
dt = ab_datetime_parse("2023-03-14T15:09:26-04:00")
|
63
|
+
print(dt) # 2023-03-14T15:09:26-04:00
|
64
|
+
|
65
|
+
## Naive datetimes are automatically converted to UTC
|
66
|
+
dt = ab_datetime_parse("2023-03-14T15:09:26")
|
67
|
+
print(dt) # 2023-03-14T15:09:26Z
|
68
|
+
```
|
69
|
+
|
70
|
+
# Format Validation
|
71
|
+
|
72
|
+
```python
|
73
|
+
from airbyte_cdk.utils.datetime_helpers import ab_datetime_try_parse
|
74
|
+
|
75
|
+
# Validate ISO8601/RFC3339 format
|
76
|
+
assert ab_datetime_try_parse("2023-03-14T15:09:26Z") # Basic UTC format
|
77
|
+
assert ab_datetime_try_parse("2023-03-14T15:09:26-04:00") # With timezone offset
|
78
|
+
assert ab_datetime_try_parse("2023-03-14T15:09:26+00:00") # With explicit UTC offset
|
79
|
+
assert not ab_datetime_try_parse("2023-03-14 15:09:26Z") # Invalid: missing T delimiter
|
80
|
+
assert not ab_datetime_try_parse("foo") # Invalid: not a datetime
|
81
|
+
```
|
82
|
+
"""
|
83
|
+
|
84
|
+
from datetime import datetime, timedelta, timezone
|
85
|
+
from typing import Any, Optional, Union, overload
|
86
|
+
|
87
|
+
from dateutil import parser
|
88
|
+
from typing_extensions import Never
|
89
|
+
from whenever import Instant, LocalDateTime, ZonedDateTime
|
90
|
+
|
91
|
+
|
92
|
+
class AirbyteDateTime(datetime):
|
93
|
+
"""A timezone-aware datetime class with ISO8601/RFC3339 string representation and operator overloading.
|
94
|
+
|
95
|
+
This class extends the standard datetime class to provide consistent timezone handling
|
96
|
+
(defaulting to UTC) and ISO8601/RFC3339 compliant string formatting. It also supports
|
97
|
+
operator overloading for datetime arithmetic with timedelta objects.
|
98
|
+
|
99
|
+
Example:
|
100
|
+
>>> dt = AirbyteDateTime(2023, 3, 14, 15, 9, 26, tzinfo=timezone.utc)
|
101
|
+
>>> str(dt)
|
102
|
+
'2023-03-14T15:09:26+00:00'
|
103
|
+
>>> dt + timedelta(hours=1)
|
104
|
+
'2023-03-14T16:09:26+00:00'
|
105
|
+
"""
|
106
|
+
|
107
|
+
def __new__(cls, *args: Any, **kwargs: Any) -> "AirbyteDateTime":
|
108
|
+
"""Creates a new timezone-aware AirbyteDateTime instance.
|
109
|
+
|
110
|
+
Ensures all instances are timezone-aware by defaulting to UTC if no timezone is provided.
|
111
|
+
|
112
|
+
Returns:
|
113
|
+
AirbyteDateTime: A new timezone-aware datetime instance.
|
114
|
+
"""
|
115
|
+
self = super().__new__(cls, *args, **kwargs)
|
116
|
+
if self.tzinfo is None:
|
117
|
+
return self.replace(tzinfo=timezone.utc)
|
118
|
+
return self
|
119
|
+
|
120
|
+
@classmethod
|
121
|
+
def from_datetime(cls, dt: datetime) -> "AirbyteDateTime":
|
122
|
+
"""Converts a standard datetime to AirbyteDateTime.
|
123
|
+
|
124
|
+
Args:
|
125
|
+
dt: A standard datetime object to convert.
|
126
|
+
|
127
|
+
Returns:
|
128
|
+
AirbyteDateTime: A new timezone-aware AirbyteDateTime instance.
|
129
|
+
"""
|
130
|
+
return cls(
|
131
|
+
dt.year,
|
132
|
+
dt.month,
|
133
|
+
dt.day,
|
134
|
+
dt.hour,
|
135
|
+
dt.minute,
|
136
|
+
dt.second,
|
137
|
+
dt.microsecond,
|
138
|
+
dt.tzinfo or timezone.utc,
|
139
|
+
)
|
140
|
+
|
141
|
+
def __str__(self) -> str:
|
142
|
+
"""Returns the datetime in ISO8601/RFC3339 format with 'T' delimiter.
|
143
|
+
|
144
|
+
Ensures consistent string representation with timezone, using '+00:00' for UTC.
|
145
|
+
Preserves full microsecond precision when present, omits when zero.
|
146
|
+
|
147
|
+
Returns:
|
148
|
+
str: ISO8601/RFC3339 formatted string.
|
149
|
+
"""
|
150
|
+
aware_self = self if self.tzinfo else self.replace(tzinfo=timezone.utc)
|
151
|
+
base = self.strftime("%Y-%m-%dT%H:%M:%S")
|
152
|
+
if self.microsecond:
|
153
|
+
base = f"{base}.{self.microsecond:06d}"
|
154
|
+
# Format timezone as ±HH:MM
|
155
|
+
offset = aware_self.strftime("%z")
|
156
|
+
return f"{base}{offset[:3]}:{offset[3:]}"
|
157
|
+
|
158
|
+
def __repr__(self) -> str:
|
159
|
+
"""Returns the same string representation as __str__ for consistency.
|
160
|
+
|
161
|
+
Returns:
|
162
|
+
str: ISO8601/RFC3339 formatted string.
|
163
|
+
"""
|
164
|
+
return self.__str__()
|
165
|
+
|
166
|
+
def add(self, delta: timedelta) -> "AirbyteDateTime":
|
167
|
+
"""Add a timedelta interval to this datetime.
|
168
|
+
|
169
|
+
This method provides a more explicit alternative to the + operator
|
170
|
+
for adding time intervals to datetimes.
|
171
|
+
|
172
|
+
Args:
|
173
|
+
delta: The timedelta interval to add.
|
174
|
+
|
175
|
+
Returns:
|
176
|
+
AirbyteDateTime: A new datetime with the interval added.
|
177
|
+
|
178
|
+
Example:
|
179
|
+
>>> dt = AirbyteDateTime(2023, 3, 14, tzinfo=timezone.utc)
|
180
|
+
>>> dt.add(timedelta(hours=1))
|
181
|
+
'2023-03-14T01:00:00Z'
|
182
|
+
"""
|
183
|
+
return self + delta
|
184
|
+
|
185
|
+
def subtract(self, delta: timedelta) -> "AirbyteDateTime":
|
186
|
+
"""Subtract a timedelta interval from this datetime.
|
187
|
+
|
188
|
+
This method provides a more explicit alternative to the - operator
|
189
|
+
for subtracting time intervals from datetimes.
|
190
|
+
|
191
|
+
Args:
|
192
|
+
delta: The timedelta interval to subtract.
|
193
|
+
|
194
|
+
Returns:
|
195
|
+
AirbyteDateTime: A new datetime with the interval subtracted.
|
196
|
+
|
197
|
+
Example:
|
198
|
+
>>> dt = AirbyteDateTime(2023, 3, 14, tzinfo=timezone.utc)
|
199
|
+
>>> dt.subtract(timedelta(hours=1))
|
200
|
+
'2023-03-13T23:00:00Z'
|
201
|
+
"""
|
202
|
+
result = super().__sub__(delta)
|
203
|
+
if isinstance(result, datetime):
|
204
|
+
return AirbyteDateTime.from_datetime(result)
|
205
|
+
raise TypeError("Invalid operation")
|
206
|
+
|
207
|
+
def __add__(self, other: timedelta) -> "AirbyteDateTime":
|
208
|
+
"""Adds a timedelta to this datetime.
|
209
|
+
|
210
|
+
Args:
|
211
|
+
other: A timedelta object to add.
|
212
|
+
|
213
|
+
Returns:
|
214
|
+
AirbyteDateTime: A new datetime with the timedelta added.
|
215
|
+
|
216
|
+
Raises:
|
217
|
+
TypeError: If other is not a timedelta.
|
218
|
+
"""
|
219
|
+
result = super().__add__(other)
|
220
|
+
if isinstance(result, datetime):
|
221
|
+
return AirbyteDateTime.from_datetime(result)
|
222
|
+
raise TypeError("Invalid operation")
|
223
|
+
|
224
|
+
def __radd__(self, other: timedelta) -> "AirbyteDateTime":
|
225
|
+
"""Supports timedelta + AirbyteDateTime operation.
|
226
|
+
|
227
|
+
Args:
|
228
|
+
other: A timedelta object to add.
|
229
|
+
|
230
|
+
Returns:
|
231
|
+
AirbyteDateTime: A new datetime with the timedelta added.
|
232
|
+
|
233
|
+
Raises:
|
234
|
+
TypeError: If other is not a timedelta.
|
235
|
+
"""
|
236
|
+
return self.__add__(other)
|
237
|
+
|
238
|
+
@overload # type: ignore[override]
|
239
|
+
def __sub__(self, other: timedelta) -> "AirbyteDateTime": ...
|
240
|
+
|
241
|
+
@overload # type: ignore[override]
|
242
|
+
def __sub__(self, other: Union[datetime, "AirbyteDateTime"]) -> timedelta: ...
|
243
|
+
|
244
|
+
def __sub__(
|
245
|
+
self, other: Union[datetime, "AirbyteDateTime", timedelta]
|
246
|
+
) -> Union[timedelta, "AirbyteDateTime"]: # type: ignore[override]
|
247
|
+
"""Subtracts a datetime, AirbyteDateTime, or timedelta from this datetime.
|
248
|
+
|
249
|
+
Args:
|
250
|
+
other: A datetime, AirbyteDateTime, or timedelta object to subtract.
|
251
|
+
|
252
|
+
Returns:
|
253
|
+
Union[timedelta, AirbyteDateTime]: A timedelta if subtracting datetime/AirbyteDateTime,
|
254
|
+
or a new datetime if subtracting timedelta.
|
255
|
+
|
256
|
+
Raises:
|
257
|
+
TypeError: If other is not a datetime, AirbyteDateTime, or timedelta.
|
258
|
+
"""
|
259
|
+
if isinstance(other, timedelta):
|
260
|
+
result = super().__sub__(other) # type: ignore[call-overload]
|
261
|
+
if isinstance(result, datetime):
|
262
|
+
return AirbyteDateTime.from_datetime(result)
|
263
|
+
elif isinstance(other, (datetime, AirbyteDateTime)):
|
264
|
+
result = super().__sub__(other) # type: ignore[call-overload]
|
265
|
+
if isinstance(result, timedelta):
|
266
|
+
return result
|
267
|
+
raise TypeError(
|
268
|
+
f"unsupported operand type(s) for -: '{type(self).__name__}' and '{type(other).__name__}'"
|
269
|
+
)
|
270
|
+
|
271
|
+
def __rsub__(self, other: datetime) -> timedelta:
|
272
|
+
"""Supports datetime - AirbyteDateTime operation.
|
273
|
+
|
274
|
+
Args:
|
275
|
+
other: A datetime object.
|
276
|
+
|
277
|
+
Returns:
|
278
|
+
timedelta: The time difference between the datetimes.
|
279
|
+
|
280
|
+
Raises:
|
281
|
+
TypeError: If other is not a datetime.
|
282
|
+
"""
|
283
|
+
if not isinstance(other, datetime):
|
284
|
+
return NotImplemented
|
285
|
+
result = other - datetime(
|
286
|
+
self.year,
|
287
|
+
self.month,
|
288
|
+
self.day,
|
289
|
+
self.hour,
|
290
|
+
self.minute,
|
291
|
+
self.second,
|
292
|
+
self.microsecond,
|
293
|
+
self.tzinfo,
|
294
|
+
)
|
295
|
+
if isinstance(result, timedelta):
|
296
|
+
return result
|
297
|
+
raise TypeError("Invalid operation")
|
298
|
+
|
299
|
+
def to_epoch_millis(self) -> int:
|
300
|
+
"""Return the Unix timestamp in milliseconds for this datetime.
|
301
|
+
|
302
|
+
Returns:
|
303
|
+
int: Number of milliseconds since Unix epoch (January 1, 1970).
|
304
|
+
|
305
|
+
Example:
|
306
|
+
>>> dt = AirbyteDateTime(2023, 3, 14, 15, 9, 26, tzinfo=timezone.utc)
|
307
|
+
>>> dt.to_epoch_millis()
|
308
|
+
1678806566000
|
309
|
+
"""
|
310
|
+
return int(self.timestamp() * 1000)
|
311
|
+
|
312
|
+
@classmethod
|
313
|
+
def from_epoch_millis(cls, milliseconds: int) -> "AirbyteDateTime":
|
314
|
+
"""Create an AirbyteDateTime from Unix timestamp in milliseconds.
|
315
|
+
|
316
|
+
Args:
|
317
|
+
milliseconds: Number of milliseconds since Unix epoch (January 1, 1970).
|
318
|
+
|
319
|
+
Returns:
|
320
|
+
AirbyteDateTime: A new timezone-aware datetime instance (UTC).
|
321
|
+
|
322
|
+
Example:
|
323
|
+
>>> dt = AirbyteDateTime.from_epoch_millis(1678806566000)
|
324
|
+
>>> str(dt)
|
325
|
+
'2023-03-14T15:09:26+00:00'
|
326
|
+
"""
|
327
|
+
return cls.fromtimestamp(milliseconds / 1000.0, timezone.utc)
|
328
|
+
|
329
|
+
@classmethod
|
330
|
+
def from_str(cls, dt_str: str) -> "AirbyteDateTime":
|
331
|
+
"""Thin convenience wrapper around `ab_datetime_parse()`.
|
332
|
+
|
333
|
+
This method attempts to create a new `AirbyteDateTime` using all available parsing
|
334
|
+
strategies.
|
335
|
+
|
336
|
+
Raises:
|
337
|
+
ValueError: If the value cannot be parsed into a valid datetime object.
|
338
|
+
"""
|
339
|
+
return ab_datetime_parse(dt_str)
|
340
|
+
|
341
|
+
|
342
|
+
def ab_datetime_now() -> AirbyteDateTime:
|
343
|
+
"""Returns the current time as an AirbyteDateTime in UTC timezone.
|
344
|
+
|
345
|
+
Previously named: now()
|
346
|
+
|
347
|
+
Returns:
|
348
|
+
AirbyteDateTime: Current UTC time.
|
349
|
+
|
350
|
+
Example:
|
351
|
+
>>> dt = ab_datetime_now()
|
352
|
+
>>> str(dt) # Returns current time in ISO8601/RFC3339
|
353
|
+
'2023-03-14T15:09:26.535897Z'
|
354
|
+
"""
|
355
|
+
return AirbyteDateTime.from_datetime(datetime.now(timezone.utc))
|
356
|
+
|
357
|
+
|
358
|
+
def ab_datetime_parse(dt_str: str | int) -> AirbyteDateTime:
|
359
|
+
"""Parses a datetime string or timestamp into an AirbyteDateTime with timezone awareness.
|
360
|
+
|
361
|
+
Previously named: parse()
|
362
|
+
|
363
|
+
Handles:
|
364
|
+
- ISO8601/RFC3339 format strings (with 'T' delimiter)
|
365
|
+
- Unix timestamps (as integers or strings)
|
366
|
+
- Date-only strings (YYYY-MM-DD)
|
367
|
+
- Timezone-aware formats (+00:00 for UTC, or ±HH:MM offset)
|
368
|
+
|
369
|
+
Always returns a timezone-aware datetime (defaults to UTC if no timezone specified).
|
370
|
+
|
371
|
+
Args:
|
372
|
+
dt_str: A datetime string in ISO8601/RFC3339 format, Unix timestamp (int/str),
|
373
|
+
or other recognizable datetime format.
|
374
|
+
|
375
|
+
Returns:
|
376
|
+
AirbyteDateTime: A timezone-aware datetime object.
|
377
|
+
|
378
|
+
Raises:
|
379
|
+
ValueError: If the input cannot be parsed as a valid datetime.
|
380
|
+
|
381
|
+
Example:
|
382
|
+
>>> ab_datetime_parse("2023-03-14T15:09:26+00:00")
|
383
|
+
'2023-03-14T15:09:26+00:00'
|
384
|
+
>>> ab_datetime_parse(1678806000) # Unix timestamp
|
385
|
+
'2023-03-14T15:00:00+00:00'
|
386
|
+
>>> ab_datetime_parse("2023-03-14") # Date-only
|
387
|
+
'2023-03-14T00:00:00+00:00'
|
388
|
+
"""
|
389
|
+
try:
|
390
|
+
# Handle numeric values as Unix timestamps (UTC)
|
391
|
+
if isinstance(dt_str, int) or (
|
392
|
+
isinstance(dt_str, str)
|
393
|
+
and (dt_str.isdigit() or (dt_str.startswith("-") and dt_str[1:].isdigit()))
|
394
|
+
):
|
395
|
+
timestamp = int(dt_str)
|
396
|
+
if timestamp < 0:
|
397
|
+
raise ValueError("Timestamp cannot be negative")
|
398
|
+
if len(str(abs(timestamp))) > 10:
|
399
|
+
raise ValueError("Timestamp value too large")
|
400
|
+
instant = Instant.from_timestamp(timestamp)
|
401
|
+
return AirbyteDateTime.from_datetime(instant.py_datetime())
|
402
|
+
|
403
|
+
if not isinstance(dt_str, str):
|
404
|
+
raise ValueError(
|
405
|
+
f"Could not parse datetime string: expected string or integer, got {type(dt_str)}"
|
406
|
+
)
|
407
|
+
|
408
|
+
# Handle date-only format first
|
409
|
+
if ":" not in dt_str and dt_str.count("-") == 2 and "/" not in dt_str:
|
410
|
+
try:
|
411
|
+
year, month, day = map(int, dt_str.split("-"))
|
412
|
+
if not (1 <= month <= 12 and 1 <= day <= 31):
|
413
|
+
raise ValueError(f"Invalid date format: {dt_str}")
|
414
|
+
instant = Instant.from_utc(year, month, day, 0, 0, 0)
|
415
|
+
return AirbyteDateTime.from_datetime(instant.py_datetime())
|
416
|
+
except (ValueError, TypeError):
|
417
|
+
raise ValueError(f"Invalid date format: {dt_str}")
|
418
|
+
|
419
|
+
# Validate datetime format
|
420
|
+
if "/" in dt_str or " " in dt_str or "GMT" in dt_str:
|
421
|
+
raise ValueError(f"Could not parse datetime string: {dt_str}")
|
422
|
+
|
423
|
+
# Try parsing with dateutil for timezone handling
|
424
|
+
try:
|
425
|
+
parsed = parser.parse(dt_str)
|
426
|
+
if parsed.tzinfo is None:
|
427
|
+
parsed = parsed.replace(tzinfo=timezone.utc)
|
428
|
+
return AirbyteDateTime.from_datetime(parsed)
|
429
|
+
except (ValueError, TypeError):
|
430
|
+
raise ValueError(f"Could not parse datetime string: {dt_str}")
|
431
|
+
except ValueError as e:
|
432
|
+
if "Invalid date format:" in str(e):
|
433
|
+
raise
|
434
|
+
if "Timestamp cannot be negative" in str(e):
|
435
|
+
raise
|
436
|
+
if "Timestamp value too large" in str(e):
|
437
|
+
raise
|
438
|
+
raise ValueError(f"Could not parse datetime string: {dt_str}")
|
439
|
+
|
440
|
+
|
441
|
+
def ab_datetime_format(dt: Union[datetime, AirbyteDateTime]) -> str:
|
442
|
+
"""Formats a datetime object as an ISO8601/RFC3339 string with 'T' delimiter and timezone.
|
443
|
+
|
444
|
+
Previously named: format()
|
445
|
+
|
446
|
+
Converts any datetime object to a string with 'T' delimiter and proper timezone.
|
447
|
+
If the datetime is naive (no timezone), UTC is assumed.
|
448
|
+
Uses '+00:00' for UTC timezone, otherwise keeps the original timezone offset.
|
449
|
+
|
450
|
+
Args:
|
451
|
+
dt: Any datetime object to format.
|
452
|
+
|
453
|
+
Returns:
|
454
|
+
str: ISO8601/RFC3339 formatted datetime string.
|
455
|
+
|
456
|
+
Example:
|
457
|
+
>>> dt = datetime(2023, 3, 14, 15, 9, 26, tzinfo=timezone.utc)
|
458
|
+
>>> ab_datetime_format(dt)
|
459
|
+
'2023-03-14T15:09:26+00:00'
|
460
|
+
"""
|
461
|
+
if isinstance(dt, AirbyteDateTime):
|
462
|
+
return str(dt)
|
463
|
+
|
464
|
+
if dt.tzinfo is None:
|
465
|
+
dt = dt.replace(tzinfo=timezone.utc)
|
466
|
+
|
467
|
+
# Format with consistent timezone representation
|
468
|
+
base = dt.strftime("%Y-%m-%dT%H:%M:%S")
|
469
|
+
if dt.microsecond:
|
470
|
+
base = f"{base}.{dt.microsecond:06d}"
|
471
|
+
offset = dt.strftime("%z")
|
472
|
+
return f"{base}{offset[:3]}:{offset[3:]}"
|
473
|
+
|
474
|
+
|
475
|
+
def ab_datetime_try_parse(dt_str: str) -> AirbyteDateTime | None:
|
476
|
+
"""Try to parse the input string as an ISO8601/RFC3339 datetime, failing gracefully instead of raising an exception.
|
477
|
+
|
478
|
+
Requires strict ISO8601/RFC3339 format with:
|
479
|
+
- 'T' delimiter between date and time components
|
480
|
+
- Valid timezone (Z for UTC or ±HH:MM offset)
|
481
|
+
- Complete datetime representation (date and time)
|
482
|
+
|
483
|
+
Returns None for any non-compliant formats including:
|
484
|
+
- Space-delimited datetimes
|
485
|
+
- Date-only strings
|
486
|
+
- Missing timezone
|
487
|
+
- Invalid timezone format
|
488
|
+
- Wrong date/time separators
|
489
|
+
|
490
|
+
Example:
|
491
|
+
>>> ab_datetime_try_parse("2023-03-14T15:09:26Z") # Returns AirbyteDateTime
|
492
|
+
>>> ab_datetime_try_parse("2023-03-14 15:09:26Z") # Returns None (invalid format)
|
493
|
+
>>> ab_datetime_try_parse("2023-03-14") # Returns None (missing time and timezone)
|
494
|
+
"""
|
495
|
+
if not isinstance(dt_str, str):
|
496
|
+
return None
|
497
|
+
try:
|
498
|
+
# Validate format before parsing
|
499
|
+
if "T" not in dt_str:
|
500
|
+
return None
|
501
|
+
if not any(x in dt_str for x in ["Z", "+", "-"]):
|
502
|
+
return None
|
503
|
+
if "/" in dt_str or " " in dt_str or "GMT" in dt_str:
|
504
|
+
return None
|
505
|
+
|
506
|
+
# Try parsing with dateutil
|
507
|
+
parsed = parser.parse(dt_str)
|
508
|
+
if parsed.tzinfo is None:
|
509
|
+
return None
|
510
|
+
|
511
|
+
# Validate time components
|
512
|
+
if not (0 <= parsed.hour <= 23 and 0 <= parsed.minute <= 59 and 0 <= parsed.second <= 59):
|
513
|
+
return None
|
514
|
+
|
515
|
+
return AirbyteDateTime.from_datetime(parsed)
|
516
|
+
except (ValueError, TypeError):
|
517
|
+
return None
|
@@ -0,0 +1 @@
|
|
1
|
+
Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
@@ -1,19 +1,20 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: airbyte-cdk
|
3
|
-
Version: 6.
|
3
|
+
Version: 6.28.0
|
4
4
|
Summary: A framework for writing Airbyte Connectors.
|
5
5
|
Home-page: https://airbyte.com
|
6
6
|
License: MIT
|
7
7
|
Keywords: airbyte,connector-development-kit,cdk
|
8
8
|
Author: Airbyte
|
9
9
|
Author-email: contact@airbyte.io
|
10
|
-
Requires-Python: >=3.10,<3.
|
10
|
+
Requires-Python: >=3.10,<3.13
|
11
11
|
Classifier: Development Status :: 3 - Alpha
|
12
12
|
Classifier: Intended Audience :: Developers
|
13
13
|
Classifier: License :: OSI Approved :: MIT License
|
14
14
|
Classifier: Programming Language :: Python :: 3
|
15
15
|
Classifier: Programming Language :: Python :: 3.10
|
16
16
|
Classifier: Programming Language :: Python :: 3.11
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
17
18
|
Classifier: Topic :: Scientific/Engineering
|
18
19
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
19
20
|
Provides-Extra: file-based
|
@@ -45,7 +46,6 @@ Requires-Dist: orjson (>=3.10.7,<4.0.0)
|
|
45
46
|
Requires-Dist: pandas (==2.2.2)
|
46
47
|
Requires-Dist: pdf2image (==1.16.3) ; extra == "file-based"
|
47
48
|
Requires-Dist: pdfminer.six (==20221105) ; extra == "file-based"
|
48
|
-
Requires-Dist: pendulum (<3.0.0)
|
49
49
|
Requires-Dist: psutil (==6.1.0)
|
50
50
|
Requires-Dist: pyarrow (>=15.0.0,<15.1.0) ; extra == "file-based"
|
51
51
|
Requires-Dist: pydantic (>=2.7,<3.0)
|
@@ -53,7 +53,7 @@ Requires-Dist: pyjwt (>=2.8.0,<3.0.0)
|
|
53
53
|
Requires-Dist: pyrate-limiter (>=3.1.0,<3.2.0)
|
54
54
|
Requires-Dist: pytesseract (==0.3.10) ; extra == "file-based"
|
55
55
|
Requires-Dist: python-calamine (==0.2.3) ; extra == "file-based"
|
56
|
-
Requires-Dist: python-dateutil
|
56
|
+
Requires-Dist: python-dateutil (>=2.9.0,<3.0.0)
|
57
57
|
Requires-Dist: python-snappy (==0.7.3) ; extra == "file-based"
|
58
58
|
Requires-Dist: python-ulid (>=3.0.0,<4.0.0)
|
59
59
|
Requires-Dist: pytz (==2024.2)
|
@@ -66,6 +66,7 @@ Requires-Dist: tiktoken (==0.8.0) ; extra == "vector-db-based"
|
|
66
66
|
Requires-Dist: unstructured.pytesseract (>=0.3.12) ; extra == "file-based"
|
67
67
|
Requires-Dist: unstructured[docx,pptx] (==0.10.27) ; extra == "file-based"
|
68
68
|
Requires-Dist: wcmatch (==10.0)
|
69
|
+
Requires-Dist: whenever (>=0.6.16,<0.7.0)
|
69
70
|
Requires-Dist: xmltodict (>=0.13,<0.15)
|
70
71
|
Project-URL: Documentation, https://docs.airbyte.io/
|
71
72
|
Project-URL: Repository, https://github.com/airbytehq/airbyte-python-cdk
|
@@ -1,13 +1,13 @@
|
|
1
1
|
airbyte_cdk/__init__.py,sha256=52uncJvDQNHvwKxaqzXgnMYTptIl65LDJr2fvlk8-DU,11707
|
2
2
|
airbyte_cdk/cli/__init__.py,sha256=Hu-1XT2KDoYjDF7-_ziDwv5bY3PueGjANOCbzeOegDg,57
|
3
3
|
airbyte_cdk/cli/source_declarative_manifest/__init__.py,sha256=-0ST722Nj65bgRokzpzPkD1NBBW5CytEHFUe38cB86Q,91
|
4
|
-
airbyte_cdk/cli/source_declarative_manifest/_run.py,sha256=
|
4
|
+
airbyte_cdk/cli/source_declarative_manifest/_run.py,sha256=9qtbjt-I_stGWzWX6yVUKO_eE-Ga7g-uTuibML9qLBs,8330
|
5
5
|
airbyte_cdk/cli/source_declarative_manifest/spec.json,sha256=Earc1L6ngcdIr514oFQlUoOxdF4RHqtUyStSIAquXdY,554
|
6
6
|
airbyte_cdk/config_observation.py,sha256=7SSPxtN0nXPkm4euGNcTTr1iLbwUL01jy-24V1Hzde0,3986
|
7
7
|
airbyte_cdk/connector.py,sha256=bO23kdGRkl8XKFytOgrrWFc_VagteTHVEF6IsbizVkM,4224
|
8
8
|
airbyte_cdk/connector_builder/README.md,sha256=Hw3wvVewuHG9-QgsAq1jDiKuLlStDxKBz52ftyNRnBw,1665
|
9
9
|
airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
10
|
-
airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=
|
10
|
+
airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=CZX1tdWYLdPE_2XtH2KnFNsHrqvWNxsotzTsKii0vrQ,4285
|
11
11
|
airbyte_cdk/connector_builder/main.py,sha256=ubAPE0Oo5gjZOa-KMtLLJQkc8_inUpFR3sIb2DEh2No,3722
|
12
12
|
airbyte_cdk/connector_builder/message_grouper.py,sha256=Xckskpqe9kbUByaKVmPsfTKxuyI2FHt8k4NZ4p8xo_I,19813
|
13
13
|
airbyte_cdk/connector_builder/models.py,sha256=uCHpOdJx2PyZtIqk-mt9eSVuFMQoEqrW-9sjCz0Z-AQ,1500
|
@@ -53,10 +53,10 @@ airbyte_cdk/sources/declarative/async_job/timer.py,sha256=Fb8P72CQ7jIzJyzMSSNuBf
|
|
53
53
|
airbyte_cdk/sources/declarative/auth/__init__.py,sha256=e2CRrcBWGhz3sQu3Oh34d1riEIwXipGS8hrSB1pu0Oo,284
|
54
54
|
airbyte_cdk/sources/declarative/auth/declarative_authenticator.py,sha256=nf-OmRUHYG4ORBwyb5CANzuHEssE-oNmL-Lccn41Td8,1099
|
55
55
|
airbyte_cdk/sources/declarative/auth/jwt.py,sha256=7r5q1zOekjw8kEmEk1oUyovzVt3cbD6BuFnRILeLZi8,8250
|
56
|
-
airbyte_cdk/sources/declarative/auth/oauth.py,sha256=
|
56
|
+
airbyte_cdk/sources/declarative/auth/oauth.py,sha256=fibXa-dqtM54jIUscWbz7DEA5uY6F2o1LfARjEeGRy0,13926
|
57
57
|
airbyte_cdk/sources/declarative/auth/selective_authenticator.py,sha256=qGwC6YsCldr1bIeKG6Qo-A9a5cTdHw-vcOn3OtQrS4c,1540
|
58
58
|
airbyte_cdk/sources/declarative/auth/token.py,sha256=r4u3WXyVa7WmiSZ9-eZXlrUI-pS0D4YWJnwjLzwV-Fk,11210
|
59
|
-
airbyte_cdk/sources/declarative/auth/token_provider.py,sha256=
|
59
|
+
airbyte_cdk/sources/declarative/auth/token_provider.py,sha256=9CuSsmOoHkvlc4k-oZ3Jx5luAgfTMm1I_5HOZxw7wMU,3075
|
60
60
|
airbyte_cdk/sources/declarative/checks/__init__.py,sha256=nsVV5Bo0E_tBNd8A4Xdsdb-75PpcLo5RQu2RQ_Gv-ME,806
|
61
61
|
airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py,sha256=aXKL1YSAB-0T_eZiavb7e5rprf-DdXG77Fy81FtlcWk,1843
|
62
62
|
airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=dAA-UhmMj0WLXCkRQrilWCfJmncBzXCZ18ptRNip3XA,2139
|
@@ -277,7 +277,7 @@ airbyte_cdk/sources/streams/concurrent/partitions/stream_slicer.py,sha256=nbdkkH
|
|
277
277
|
airbyte_cdk/sources/streams/concurrent/partitions/types.py,sha256=frPVvHtY7vLxpGEbMQzNvF1Y52ZVyct9f1DDhGoRjwY,1166
|
278
278
|
airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
279
279
|
airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py,sha256=CXHUMOhndu-LOKgsnNTItv5s5qrKpmJDeHOzlH1nBy8,6819
|
280
|
-
airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py,sha256=
|
280
|
+
airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py,sha256=x8MLm1pTMfLNHvMF3P1ixYkYt_xjpbaIwnvhY_ofdBo,8076
|
281
281
|
airbyte_cdk/sources/streams/core.py,sha256=jiYW6w8cjNjzXMd8U8Gt-02fYYU7b0ciXSSSnGvFRak,32219
|
282
282
|
airbyte_cdk/sources/streams/http/__init__.py,sha256=AGiEZ5B1Joi9ZnFpkJLT7F3QLpCAaBgAeVWy-1znmZw,311
|
283
283
|
airbyte_cdk/sources/streams/http/availability_strategy.py,sha256=sovoGFThZr-doMN9vJvTuJBrvkwQVIO0qTQO64pGZPY,2428
|
@@ -295,9 +295,9 @@ airbyte_cdk/sources/streams/http/http.py,sha256=JAMpiTdS9HFNOlwayWNvQdxoqs2rpW9w
|
|
295
295
|
airbyte_cdk/sources/streams/http/http_client.py,sha256=tDE0ROtxjGMVphvsw8INvGMtZ97hIF-v47pZ3jIyiwc,23011
|
296
296
|
airbyte_cdk/sources/streams/http/rate_limiting.py,sha256=IwdjrHKUnU97XO4qONgYRv4YYW51xQ8SJm4WLafXDB8,6351
|
297
297
|
airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py,sha256=RN0D3nOX1xLgwEwKWu6pkGy3XqBFzKSNZ8Lf6umU2eY,413
|
298
|
-
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256
|
298
|
+
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=DKv9oAHaEp80qHpy3JBaMhn8QLfXavoUpY4fIY77Fkk,12176
|
299
299
|
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py,sha256=Y3n7J-sk5yGjv_OxtY6Z6k0PEsFZmtIRi-x0KCbaHdA,1010
|
300
|
-
airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py,sha256=
|
300
|
+
airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py,sha256=jYlqj7wUs7z7son6mmKjbNzyizN7iWv9MeuRiYRpPHo,16902
|
301
301
|
airbyte_cdk/sources/streams/http/requests_native_auth/token.py,sha256=h5PTzcdH-RQLeCg7xZ45w_484OPUDSwNWl_iMJQmZoI,2526
|
302
302
|
airbyte_cdk/sources/streams/utils/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
303
303
|
airbyte_cdk/sources/types.py,sha256=aFPGI4t2K1vHz2oFSUIYUyDN7kw-vcYq4D7aD2zgfAU,5128
|
@@ -306,7 +306,7 @@ airbyte_cdk/sources/utils/casing.py,sha256=QC-gV1O4e8DR4-bhdXieUPKm_JamzslVyfABL
|
|
306
306
|
airbyte_cdk/sources/utils/record_helper.py,sha256=jeB0mucudzna7Zvj-pCBbwFrbLJ36SlAWZTh5O4Fb9Y,2168
|
307
307
|
airbyte_cdk/sources/utils/schema_helpers.py,sha256=bR3I70-e11S6B8r6VK-pthQXtcYrXojgXFvuK7lRrpg,8545
|
308
308
|
airbyte_cdk/sources/utils/slice_logger.py,sha256=qWWeFLAvigFz0b4O1_O3QDM1cy8PqZAMMgVPR2hEeb8,1778
|
309
|
-
airbyte_cdk/sources/utils/transform.py,sha256=
|
309
|
+
airbyte_cdk/sources/utils/transform.py,sha256=0LOvIJg1vmg_70AiAVe-YHMr-LHrqEuxg9cm1BnYPDM,11725
|
310
310
|
airbyte_cdk/sources/utils/types.py,sha256=41ZQR681t5TUnOScij58d088sb99klH_ZENFcaYro_g,175
|
311
311
|
airbyte_cdk/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
312
312
|
airbyte_cdk/sql/_util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -339,6 +339,7 @@ airbyte_cdk/utils/airbyte_secrets_utils.py,sha256=wEtRnl5KRhN6eLJwrDrC4FJjyqt_4v
|
|
339
339
|
airbyte_cdk/utils/analytics_message.py,sha256=bi3uugQ2NjecnwTnz63iD5D1M8ZR8mXPbdtt6w5cC4s,653
|
340
340
|
airbyte_cdk/utils/constants.py,sha256=QzCi7j5SqpI5I06uRvQ8FC73JVJi7rXaRnR3E_gro5c,108
|
341
341
|
airbyte_cdk/utils/datetime_format_inferrer.py,sha256=Ne2cpk7Tx3eZDEW2Q3O7jnNOY9g-w-AUMt3Ltvwg1tY,3989
|
342
|
+
airbyte_cdk/utils/datetime_helpers.py,sha256=PD47CAFDt0ZeCSH8HiVDyPKym0qAuGmaZrClPyVwO0U,18012
|
342
343
|
airbyte_cdk/utils/event_timing.py,sha256=aiuFmPU80buLlNdKq4fDTEqqhEIelHPF6AalFGwY8as,2557
|
343
344
|
airbyte_cdk/utils/is_cloud_environment.py,sha256=DayV32Irh-SdnJ0MnjvstwCJ66_l5oEsd8l85rZtHoc,574
|
344
345
|
airbyte_cdk/utils/mapping_helpers.py,sha256=H7BH-Yr8hSRG4W0zZcQ0ZzKOY5QFn8fNkGcEsE3xZN8,1736
|
@@ -350,8 +351,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
|
|
350
351
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
351
352
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
352
353
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
353
|
-
airbyte_cdk-6.
|
354
|
-
airbyte_cdk-6.
|
355
|
-
airbyte_cdk-6.
|
356
|
-
airbyte_cdk-6.
|
357
|
-
airbyte_cdk-6.
|
354
|
+
airbyte_cdk-6.28.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
355
|
+
airbyte_cdk-6.28.0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
356
|
+
airbyte_cdk-6.28.0.dist-info/METADATA,sha256=EYHArwfUWLBPhnr-8IYPz0Vhh8cA3Nx5BKznR26QB2g,6010
|
357
|
+
airbyte_cdk-6.28.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
358
|
+
airbyte_cdk-6.28.0.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
|
359
|
+
airbyte_cdk-6.28.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|