airbyte-cdk 6.8.1.dev1__py3-none-any.whl → 6.8.1rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/cli/source_declarative_manifest/_run.py +5 -11
- airbyte_cdk/config_observation.py +1 -1
- airbyte_cdk/connector_builder/main.py +1 -1
- airbyte_cdk/connector_builder/message_grouper.py +10 -10
- airbyte_cdk/destinations/destination.py +1 -1
- airbyte_cdk/destinations/vector_db_based/embedder.py +2 -2
- airbyte_cdk/destinations/vector_db_based/writer.py +4 -12
- airbyte_cdk/entrypoint.py +6 -7
- airbyte_cdk/logger.py +2 -2
- airbyte_cdk/sources/abstract_source.py +1 -1
- airbyte_cdk/sources/config.py +1 -1
- airbyte_cdk/sources/connector_state_manager.py +4 -9
- airbyte_cdk/sources/declarative/auth/oauth.py +1 -1
- airbyte_cdk/sources/declarative/auth/selective_authenticator.py +1 -6
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +1 -1
- airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +4 -10
- airbyte_cdk/sources/declarative/declarative_component_schema.yaml +17 -16
- airbyte_cdk/sources/declarative/decoders/noop_decoder.py +1 -4
- airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +6 -8
- airbyte_cdk/sources/declarative/interpolation/jinja.py +3 -3
- airbyte_cdk/sources/declarative/interpolation/macros.py +1 -1
- airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +6 -5
- airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +7 -13
- airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +1 -1
- airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +6 -8
- airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +1 -1
- airbyte_cdk/sources/declarative/requesters/request_options/datetime_based_request_options_provider.py +2 -2
- airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +1 -1
- airbyte_cdk/sources/declarative/retrievers/async_retriever.py +2 -5
- airbyte_cdk/sources/declarative/spec/spec.py +1 -1
- airbyte_cdk/sources/embedded/base_integration.py +2 -3
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +4 -12
- airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +7 -18
- airbyte_cdk/sources/file_based/file_types/avro_parser.py +11 -14
- airbyte_cdk/sources/file_based/file_types/csv_parser.py +3 -3
- airbyte_cdk/sources/file_based/file_types/excel_parser.py +5 -11
- airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +1 -1
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +2 -2
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +3 -6
- airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py +1 -1
- airbyte_cdk/sources/http_logger.py +3 -3
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +2 -5
- airbyte_cdk/sources/streams/concurrent/adapters.py +3 -6
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +3 -9
- airbyte_cdk/sources/streams/concurrent/cursor.py +1 -1
- airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py +2 -2
- airbyte_cdk/sources/streams/core.py +14 -17
- airbyte_cdk/sources/streams/http/http.py +19 -19
- airbyte_cdk/sources/streams/http/http_client.py +34 -3
- airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py +1 -2
- airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +33 -62
- airbyte_cdk/sources/utils/record_helper.py +1 -1
- airbyte_cdk/sources/utils/schema_helpers.py +1 -1
- airbyte_cdk/sources/utils/transform.py +15 -34
- airbyte_cdk/test/entrypoint_wrapper.py +6 -11
- airbyte_cdk/test/mock_http/response_builder.py +1 -1
- airbyte_cdk/utils/airbyte_secrets_utils.py +1 -1
- airbyte_cdk/utils/event_timing.py +10 -10
- airbyte_cdk/utils/message_utils.py +3 -4
- airbyte_cdk/utils/spec_schema_transformations.py +2 -3
- airbyte_cdk/utils/traced_exception.py +12 -14
- {airbyte_cdk-6.8.1.dev1.dist-info → airbyte_cdk-6.8.1rc1.dist-info}/METADATA +2 -1
- {airbyte_cdk-6.8.1.dev1.dist-info → airbyte_cdk-6.8.1rc1.dist-info}/RECORD +66 -67
- airbyte_cdk/test/utils/manifest_only_fixtures.py +0 -40
- {airbyte_cdk-6.8.1.dev1.dist-info → airbyte_cdk-6.8.1rc1.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.8.1.dev1.dist-info → airbyte_cdk-6.8.1rc1.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.8.1.dev1.dist-info → airbyte_cdk-6.8.1rc1.dist-info}/entry_points.txt +0 -0
@@ -25,7 +25,7 @@ from datetime import datetime
|
|
25
25
|
from pathlib import Path
|
26
26
|
from typing import Any, cast
|
27
27
|
|
28
|
-
import orjson
|
28
|
+
from orjson import orjson
|
29
29
|
|
30
30
|
from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
|
31
31
|
from airbyte_cdk.models import (
|
@@ -72,7 +72,7 @@ class SourceLocalYaml(YamlDeclarativeSource):
|
|
72
72
|
super().__init__(
|
73
73
|
catalog=catalog,
|
74
74
|
config=config,
|
75
|
-
state=state,
|
75
|
+
state=state,
|
76
76
|
path_to_yaml="manifest.yaml",
|
77
77
|
)
|
78
78
|
|
@@ -152,9 +152,7 @@ def handle_remote_manifest_command(args: list[str]) -> None:
|
|
152
152
|
)
|
153
153
|
|
154
154
|
|
155
|
-
def create_declarative_source(
|
156
|
-
args: list[str],
|
157
|
-
) -> ConcurrentDeclarativeSource: # type: ignore [type-arg]
|
155
|
+
def create_declarative_source(args: list[str]) -> ConcurrentDeclarativeSource:
|
158
156
|
"""Creates the source with the injected config.
|
159
157
|
|
160
158
|
This essentially does what other low-code sources do at build time, but at runtime,
|
@@ -162,14 +160,10 @@ def create_declarative_source(
|
|
162
160
|
connector builder.
|
163
161
|
"""
|
164
162
|
try:
|
165
|
-
config: Mapping[str, Any] | None
|
166
|
-
catalog: ConfiguredAirbyteCatalog | None
|
167
|
-
state: list[AirbyteStateMessage]
|
168
163
|
config, catalog, state = _parse_inputs_into_config_catalog_state(args)
|
169
|
-
if
|
164
|
+
if "__injected_declarative_manifest" not in config:
|
170
165
|
raise ValueError(
|
171
|
-
"Invalid config: `__injected_declarative_manifest` should be provided at the root "
|
172
|
-
f"of the config but config only has keys: {list(config.keys() if config else [])}"
|
166
|
+
f"Invalid config: `__injected_declarative_manifest` should be provided at the root of the config but config only has keys {list(config.keys())}"
|
173
167
|
)
|
174
168
|
return ConcurrentDeclarativeSource(
|
175
169
|
config=config,
|
@@ -71,7 +71,7 @@ class MessageGrouper:
|
|
71
71
|
|
72
72
|
is_nested_key = isinstance(field[0], str)
|
73
73
|
if is_nested_key:
|
74
|
-
return [field]
|
74
|
+
return [field] # type: ignore # the type of field is expected to be List[str] here
|
75
75
|
|
76
76
|
raise ValueError(f"Unknown type for cursor field `{field}")
|
77
77
|
|
@@ -232,9 +232,9 @@ class MessageGrouper:
|
|
232
232
|
current_slice_descriptor = self._parse_slice_description(message.log.message) # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message
|
233
233
|
current_slice_pages = []
|
234
234
|
at_least_one_page_in_group = False
|
235
|
-
elif message.type == MessageType.LOG and message.log.message.startswith(
|
235
|
+
elif message.type == MessageType.LOG and message.log.message.startswith(
|
236
236
|
SliceLogger.SLICE_LOG_PREFIX
|
237
|
-
):
|
237
|
+
): # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message
|
238
238
|
# parsing the first slice
|
239
239
|
current_slice_descriptor = self._parse_slice_description(message.log.message) # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message
|
240
240
|
elif message.type == MessageType.LOG:
|
@@ -274,14 +274,14 @@ class MessageGrouper:
|
|
274
274
|
if message.trace.type == TraceType.ERROR: # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has trace.type
|
275
275
|
yield message.trace
|
276
276
|
elif message.type == MessageType.RECORD:
|
277
|
-
current_page_records.append(message.record.data) # type: ignore[
|
277
|
+
current_page_records.append(message.record.data) # type: ignore[union-attr] # AirbyteMessage with MessageType.RECORD has record.data
|
278
278
|
records_count += 1
|
279
279
|
schema_inferrer.accumulate(message.record)
|
280
280
|
datetime_format_inferrer.accumulate(message.record)
|
281
281
|
elif (
|
282
282
|
message.type == MessageType.CONTROL
|
283
|
-
and message.control.type == OrchestratorType.CONNECTOR_CONFIG
|
284
|
-
):
|
283
|
+
and message.control.type == OrchestratorType.CONNECTOR_CONFIG
|
284
|
+
): # type: ignore[union-attr] # AirbyteMessage with MessageType.CONTROL has control.type
|
285
285
|
yield message.control
|
286
286
|
elif message.type == MessageType.STATE:
|
287
287
|
latest_state_message = message.state # type: ignore[assignment]
|
@@ -310,8 +310,8 @@ class MessageGrouper:
|
|
310
310
|
and message.type == MessageType.LOG
|
311
311
|
and (
|
312
312
|
MessageGrouper._is_page_http_request(json_message)
|
313
|
-
or message.log.message.startswith("slice:")
|
314
|
-
)
|
313
|
+
or message.log.message.startswith("slice:")
|
314
|
+
) # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message
|
315
315
|
)
|
316
316
|
|
317
317
|
@staticmethod
|
@@ -355,8 +355,8 @@ class MessageGrouper:
|
|
355
355
|
StreamReadPages(
|
356
356
|
request=current_page_request,
|
357
357
|
response=current_page_response,
|
358
|
-
records=deepcopy(current_page_records),
|
359
|
-
)
|
358
|
+
records=deepcopy(current_page_records),
|
359
|
+
) # type: ignore
|
360
360
|
)
|
361
361
|
current_page_records.clear()
|
362
362
|
|
@@ -9,7 +9,7 @@ import sys
|
|
9
9
|
from abc import ABC, abstractmethod
|
10
10
|
from typing import Any, Iterable, List, Mapping
|
11
11
|
|
12
|
-
import orjson
|
12
|
+
from orjson import orjson
|
13
13
|
|
14
14
|
from airbyte_cdk.connector import Connector
|
15
15
|
from airbyte_cdk.exception_handler import init_uncaught_exception_handler
|
@@ -107,7 +107,7 @@ class BaseOpenAIEmbedder(Embedder):
|
|
107
107
|
class OpenAIEmbedder(BaseOpenAIEmbedder):
|
108
108
|
def __init__(self, config: OpenAIEmbeddingConfigModel, chunk_size: int):
|
109
109
|
super().__init__(
|
110
|
-
OpenAIEmbeddings(
|
110
|
+
OpenAIEmbeddings(
|
111
111
|
openai_api_key=config.openai_key, max_retries=15, disallowed_special=()
|
112
112
|
),
|
113
113
|
chunk_size,
|
@@ -118,7 +118,7 @@ class AzureOpenAIEmbedder(BaseOpenAIEmbedder):
|
|
118
118
|
def __init__(self, config: AzureOpenAIEmbeddingConfigModel, chunk_size: int):
|
119
119
|
# Azure OpenAI API has — as of 20230927 — a limit of 16 documents per request
|
120
120
|
super().__init__(
|
121
|
-
OpenAIEmbeddings(
|
121
|
+
OpenAIEmbeddings(
|
122
122
|
openai_api_key=config.openai_key,
|
123
123
|
chunk_size=16,
|
124
124
|
max_retries=15,
|
@@ -83,19 +83,11 @@ class Writer:
|
|
83
83
|
yield message
|
84
84
|
elif message.type == Type.RECORD:
|
85
85
|
record_chunks, record_id_to_delete = self.processor.process(message.record)
|
86
|
-
self.chunks[
|
87
|
-
( # type: ignore [index] # expected "tuple[str, str]", got "tuple[str | Any | None, str | Any]"
|
88
|
-
message.record.namespace, # type: ignore [union-attr] # record not None
|
89
|
-
message.record.stream, # type: ignore [union-attr] # record not None
|
90
|
-
)
|
91
|
-
].extend(record_chunks)
|
86
|
+
self.chunks[(message.record.namespace, message.record.stream)].extend(record_chunks)
|
92
87
|
if record_id_to_delete is not None:
|
93
|
-
self.ids_to_delete[
|
94
|
-
|
95
|
-
|
96
|
-
message.record.stream, # type: ignore [union-attr] # record not None
|
97
|
-
)
|
98
|
-
].append(record_id_to_delete)
|
88
|
+
self.ids_to_delete[(message.record.namespace, message.record.stream)].append(
|
89
|
+
record_id_to_delete
|
90
|
+
)
|
99
91
|
self.number_of_chunks += len(record_chunks)
|
100
92
|
if self.number_of_chunks >= self.batch_size:
|
101
93
|
self._process_batch()
|
airbyte_cdk/entrypoint.py
CHANGED
@@ -22,7 +22,7 @@ from requests import PreparedRequest, Response, Session
|
|
22
22
|
from airbyte_cdk.connector import TConfig
|
23
23
|
from airbyte_cdk.exception_handler import init_uncaught_exception_handler
|
24
24
|
from airbyte_cdk.logger import init_logger
|
25
|
-
from airbyte_cdk.models import (
|
25
|
+
from airbyte_cdk.models import ( # type: ignore [attr-defined]
|
26
26
|
AirbyteConnectionStatus,
|
27
27
|
AirbyteMessage,
|
28
28
|
AirbyteMessageSerializer,
|
@@ -255,10 +255,9 @@ class AirbyteEntrypoint(object):
|
|
255
255
|
|
256
256
|
stream_message_count[
|
257
257
|
HashableStreamDescriptor(
|
258
|
-
name=message.record.stream,
|
259
|
-
namespace=message.record.namespace, # type: ignore[union-attr] # record has `namespace`
|
258
|
+
name=message.record.stream, namespace=message.record.namespace
|
260
259
|
)
|
261
|
-
] += 1.0
|
260
|
+
] += 1.0 # type: ignore[union-attr] # record has `stream` and `namespace`
|
262
261
|
case Type.STATE:
|
263
262
|
if message.state is None:
|
264
263
|
raise ValueError("State message must have a state attribute")
|
@@ -267,9 +266,9 @@ class AirbyteEntrypoint(object):
|
|
267
266
|
|
268
267
|
# Set record count from the counter onto the state message
|
269
268
|
message.state.sourceStats = message.state.sourceStats or AirbyteStateStats() # type: ignore[union-attr] # state has `sourceStats`
|
270
|
-
message.state.sourceStats.recordCount = stream_message_count.get(
|
269
|
+
message.state.sourceStats.recordCount = stream_message_count.get(
|
271
270
|
stream_descriptor, 0.0
|
272
|
-
)
|
271
|
+
) # type: ignore[union-attr] # state has `sourceStats`
|
273
272
|
|
274
273
|
# Reset the counter
|
275
274
|
stream_message_count[stream_descriptor] = 0.0
|
@@ -291,7 +290,7 @@ class AirbyteEntrypoint(object):
|
|
291
290
|
|
292
291
|
@staticmethod
|
293
292
|
def airbyte_message_to_string(airbyte_message: AirbyteMessage) -> str:
|
294
|
-
return orjson.dumps(AirbyteMessageSerializer.dump(airbyte_message)).decode()
|
293
|
+
return orjson.dumps(AirbyteMessageSerializer.dump(airbyte_message)).decode() # type: ignore[no-any-return] # orjson.dumps(message).decode() always returns string
|
295
294
|
|
296
295
|
@classmethod
|
297
296
|
def extract_state(cls, args: List[str]) -> Optional[Any]:
|
airbyte_cdk/logger.py
CHANGED
@@ -7,7 +7,7 @@ import logging
|
|
7
7
|
import logging.config
|
8
8
|
from typing import Any, Callable, Mapping, Optional, Tuple
|
9
9
|
|
10
|
-
import orjson
|
10
|
+
from orjson import orjson
|
11
11
|
|
12
12
|
from airbyte_cdk.models import (
|
13
13
|
AirbyteLogMessage,
|
@@ -78,7 +78,7 @@ class AirbyteLogFormatter(logging.Formatter):
|
|
78
78
|
log_message = AirbyteMessage(
|
79
79
|
type=Type.LOG, log=AirbyteLogMessage(level=airbyte_level, message=message)
|
80
80
|
)
|
81
|
-
return orjson.dumps(AirbyteMessageSerializer.dump(log_message)).decode()
|
81
|
+
return orjson.dumps(AirbyteMessageSerializer.dump(log_message)).decode() # type: ignore[no-any-return] # orjson.dumps(message).decode() always returns string
|
82
82
|
|
83
83
|
@staticmethod
|
84
84
|
def extract_extra_args_from_record(record: logging.LogRecord) -> Mapping[str, Any]:
|
@@ -200,7 +200,7 @@ class AbstractSource(Source, ABC):
|
|
200
200
|
if len(stream_name_to_exception) > 0:
|
201
201
|
error_message = generate_failed_streams_error_message(
|
202
202
|
{key: [value] for key, value in stream_name_to_exception.items()}
|
203
|
-
)
|
203
|
+
) # type: ignore # for some reason, mypy can't figure out the types for key and value
|
204
204
|
logger.info(error_message)
|
205
205
|
# We still raise at least one exception when a stream raises an exception because the platform currently relies
|
206
206
|
# on a non-zero exit code to determine if a sync attempt has failed. We also raise the exception as a config_error
|
airbyte_cdk/sources/config.py
CHANGED
@@ -24,4 +24,4 @@ class BaseConfig(BaseModel):
|
|
24
24
|
rename_key(schema, old_key="anyOf", new_key="oneOf") # UI supports only oneOf
|
25
25
|
expand_refs(schema)
|
26
26
|
schema.pop("description", None) # description added from the docstring
|
27
|
-
return schema
|
27
|
+
return schema # type: ignore[no-any-return]
|
@@ -4,7 +4,7 @@
|
|
4
4
|
|
5
5
|
import copy
|
6
6
|
from dataclasses import dataclass
|
7
|
-
from typing import Any, List, Mapping, MutableMapping, Optional, Tuple, Union
|
7
|
+
from typing import Any, List, Mapping, MutableMapping, Optional, Tuple, Union
|
8
8
|
|
9
9
|
from airbyte_cdk.models import (
|
10
10
|
AirbyteMessage,
|
@@ -15,7 +15,6 @@ from airbyte_cdk.models import (
|
|
15
15
|
StreamDescriptor,
|
16
16
|
)
|
17
17
|
from airbyte_cdk.models import Type as MessageType
|
18
|
-
from airbyte_cdk.models.airbyte_protocol import AirbyteGlobalState, AirbyteStateBlob
|
19
18
|
|
20
19
|
|
21
20
|
@dataclass(frozen=True)
|
@@ -119,12 +118,8 @@ class ConnectorStateManager:
|
|
119
118
|
is_global = cls._is_global_state(state)
|
120
119
|
|
121
120
|
if is_global:
|
122
|
-
#
|
123
|
-
|
124
|
-
# global_state has shared_state, also not None:
|
125
|
-
shared_state: AirbyteStateBlob = cast(
|
126
|
-
AirbyteStateBlob, copy.deepcopy(global_state.shared_state, {})
|
127
|
-
)
|
121
|
+
global_state = state[0].global_ # type: ignore # We verified state is a list in _is_global_state
|
122
|
+
shared_state = copy.deepcopy(global_state.shared_state, {}) # type: ignore[union-attr] # global_state has shared_state
|
128
123
|
streams = {
|
129
124
|
HashableStreamDescriptor(
|
130
125
|
name=per_stream_state.stream_descriptor.name,
|
@@ -136,7 +131,7 @@ class ConnectorStateManager:
|
|
136
131
|
else:
|
137
132
|
streams = {
|
138
133
|
HashableStreamDescriptor(
|
139
|
-
name=per_stream_state.stream.stream_descriptor.name,
|
134
|
+
name=per_stream_state.stream.stream_descriptor.name,
|
140
135
|
namespace=per_stream_state.stream.stream_descriptor.namespace, # type: ignore[union-attr] # stream has stream_descriptor
|
141
136
|
): per_stream_state.stream.stream_state # type: ignore[union-attr] # stream has stream_state
|
142
137
|
for per_stream_state in state
|
@@ -135,7 +135,7 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
|
|
135
135
|
return self.grant_type.eval(self.config) # type: ignore # eval returns a string in this context
|
136
136
|
|
137
137
|
def get_refresh_request_body(self) -> Mapping[str, Any]:
|
138
|
-
return self._refresh_request_body.eval(self.config)
|
138
|
+
return self._refresh_request_body.eval(self.config) # type: ignore # eval should return a Mapping in this context
|
139
139
|
|
140
140
|
def get_token_expiry_date(self) -> pendulum.DateTime:
|
141
141
|
return self._token_expiry_date # type: ignore # _token_expiry_date is a pendulum.DateTime. It is never None despite what mypy thinks
|
@@ -28,12 +28,7 @@ class SelectiveAuthenticator(DeclarativeAuthenticator):
|
|
28
28
|
**kwargs: Any,
|
29
29
|
) -> DeclarativeAuthenticator:
|
30
30
|
try:
|
31
|
-
selected_key = str(
|
32
|
-
dpath.get(
|
33
|
-
config, # type: ignore [arg-type] # Dpath wants mutable mapping but doesn't need it.
|
34
|
-
authenticator_selection_path,
|
35
|
-
)
|
36
|
-
)
|
31
|
+
selected_key = str(dpath.get(config, authenticator_selection_path))
|
37
32
|
except KeyError as err:
|
38
33
|
raise ValueError(
|
39
34
|
"The path from `authenticator_selection_path` is not found in the config."
|
@@ -128,7 +128,7 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
128
128
|
initial_number_of_partitions_to_generate=initial_number_of_partitions_to_generate,
|
129
129
|
logger=self.logger,
|
130
130
|
slice_logger=self._slice_logger,
|
131
|
-
message_repository=self.message_repository,
|
131
|
+
message_repository=self.message_repository, # type: ignore # message_repository is always instantiated with a value by factory
|
132
132
|
)
|
133
133
|
|
134
134
|
def read(
|
@@ -41,12 +41,12 @@ class MinMaxDatetime:
|
|
41
41
|
self.datetime = InterpolatedString.create(self.datetime, parameters=parameters or {})
|
42
42
|
self._parser = DatetimeParser()
|
43
43
|
self.min_datetime = (
|
44
|
-
InterpolatedString.create(self.min_datetime, parameters=parameters)
|
44
|
+
InterpolatedString.create(self.min_datetime, parameters=parameters)
|
45
45
|
if self.min_datetime
|
46
46
|
else None
|
47
47
|
) # type: ignore
|
48
48
|
self.max_datetime = (
|
49
|
-
InterpolatedString.create(self.max_datetime, parameters=parameters)
|
49
|
+
InterpolatedString.create(self.max_datetime, parameters=parameters)
|
50
50
|
if self.max_datetime
|
51
51
|
else None
|
52
52
|
) # type: ignore
|
@@ -66,13 +66,7 @@ class MinMaxDatetime:
|
|
66
66
|
datetime_format = "%Y-%m-%dT%H:%M:%S.%f%z"
|
67
67
|
|
68
68
|
time = self._parser.parse(
|
69
|
-
str(
|
70
|
-
self.datetime.eval( # type: ignore[union-attr] # str has no attribute "eval"
|
71
|
-
config,
|
72
|
-
**additional_parameters,
|
73
|
-
)
|
74
|
-
),
|
75
|
-
datetime_format,
|
69
|
+
str(self.datetime.eval(config, **additional_parameters)), datetime_format
|
76
70
|
) # type: ignore # datetime is always cast to an interpolated string
|
77
71
|
|
78
72
|
if self.min_datetime:
|
@@ -111,7 +105,7 @@ class MinMaxDatetime:
|
|
111
105
|
if isinstance(interpolated_string_or_min_max_datetime, InterpolatedString) or isinstance(
|
112
106
|
interpolated_string_or_min_max_datetime, str
|
113
107
|
):
|
114
|
-
return MinMaxDatetime(
|
108
|
+
return MinMaxDatetime(
|
115
109
|
datetime=interpolated_string_or_min_max_datetime, parameters=parameters
|
116
110
|
)
|
117
111
|
else:
|
@@ -2057,7 +2057,7 @@ definitions:
|
|
2057
2057
|
The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.
|
2058
2058
|
The placeholders are replaced during the processing to provide neccessary values.
|
2059
2059
|
examples:
|
2060
|
-
- access_token_url: https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}
|
2060
|
+
- access_token_url: https://auth.host.com/oauth2/token?{client_id_key}={{client_id_key}}&{client_secret_key}={{client_secret_key}}&{auth_code_key}={{auth_code_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}
|
2061
2061
|
access_token_headers:
|
2062
2062
|
title: (Optional) DeclarativeOAuth Access Token Headers
|
2063
2063
|
type: object
|
@@ -2065,10 +2065,9 @@ definitions:
|
|
2065
2065
|
description: |-
|
2066
2066
|
The DeclarativeOAuth Specific optional headers to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
|
2067
2067
|
examples:
|
2068
|
-
- access_token_headers:
|
2069
|
-
{
|
2070
|
-
|
2071
|
-
}
|
2068
|
+
- access_token_headers: {
|
2069
|
+
"Authorization": "Basic {base64Encoder:{client_id}:{client_secret}}"
|
2070
|
+
}
|
2072
2071
|
access_token_params:
|
2073
2072
|
title: (Optional) DeclarativeOAuth Access Token Query Params (Json Encoded)
|
2074
2073
|
type: object
|
@@ -2077,19 +2076,18 @@ definitions:
|
|
2077
2076
|
The DeclarativeOAuth Specific optional query parameters to inject while exchanging the `auth_code` to `access_token` during `completeOAuthFlow` step.
|
2078
2077
|
When this property is provided, the query params will be encoded as `Json` and included in the outgoing API request.
|
2079
2078
|
examples:
|
2080
|
-
- access_token_params:
|
2081
|
-
{
|
2082
|
-
|
2083
|
-
|
2084
|
-
|
2085
|
-
}
|
2079
|
+
- access_token_params: {
|
2080
|
+
"{auth_code_key}": "{{auth_code_key}}",
|
2081
|
+
"{client_id_key}": "{{client_id_key}}",
|
2082
|
+
"{client_secret_key}": "{{client_secret_key}}"
|
2083
|
+
}
|
2086
2084
|
extract_output:
|
2087
2085
|
title: DeclarativeOAuth Extract Output
|
2088
2086
|
type: array
|
2089
2087
|
items:
|
2090
2088
|
type: string
|
2091
2089
|
description: |-
|
2092
|
-
The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.
|
2090
|
+
The DeclarativeOAuth Specific list of strings to indicate which keys should be extracted and returned back to the input config.
|
2093
2091
|
examples:
|
2094
2092
|
- extract_output: ["access_token", "refresh_token", "other_field"]
|
2095
2093
|
state:
|
@@ -2101,14 +2099,17 @@ definitions:
|
|
2101
2099
|
- max
|
2102
2100
|
description: |-
|
2103
2101
|
The DeclarativeOAuth Specific object to provide the criteria of how the `state` query param should be constructed,
|
2104
|
-
including length and complexity.
|
2102
|
+
including length and complexity.
|
2105
2103
|
properties:
|
2106
2104
|
min:
|
2107
2105
|
type: integer
|
2108
2106
|
max:
|
2109
2107
|
type: integer
|
2110
2108
|
examples:
|
2111
|
-
- state: {
|
2109
|
+
- state: {
|
2110
|
+
"min": 7,
|
2111
|
+
"max": 128,
|
2112
|
+
}
|
2112
2113
|
client_id_key:
|
2113
2114
|
title: (Optional) DeclarativeOAuth Client ID Key Override
|
2114
2115
|
type: string
|
@@ -2134,14 +2135,14 @@ definitions:
|
|
2134
2135
|
title: (Optional) DeclarativeOAuth State Key Override
|
2135
2136
|
type: string
|
2136
2137
|
description: |-
|
2137
|
-
The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.
|
2138
|
+
The DeclarativeOAuth Specific optional override to provide the custom `state` key name, if required by data-provider.
|
2138
2139
|
examples:
|
2139
2140
|
- state_key: "my_custom_state_key_key_name"
|
2140
2141
|
auth_code_key:
|
2141
2142
|
title: (Optional) DeclarativeOAuth Auth Code Key Override
|
2142
2143
|
type: string
|
2143
2144
|
description: |-
|
2144
|
-
The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.
|
2145
|
+
The DeclarativeOAuth Specific optional override to provide the custom `code` key name to something like `auth_code` or `custom_auth_code`, if required by data-provider.
|
2145
2146
|
examples:
|
2146
2147
|
- auth_code_key: "my_custom_auth_code_key_name"
|
2147
2148
|
redirect_uri_key:
|
@@ -14,8 +14,5 @@ class NoopDecoder(Decoder):
|
|
14
14
|
def is_stream_response(self) -> bool:
|
15
15
|
return False
|
16
16
|
|
17
|
-
def decode(
|
18
|
-
self,
|
19
|
-
response: requests.Response,
|
20
|
-
) -> Generator[Mapping[str, Any], None, None]:
|
17
|
+
def decode(self, response: requests.Response) -> Generator[Mapping[str, Any], None, None]:
|
21
18
|
yield from [{}]
|
@@ -133,8 +133,8 @@ class DatetimeBasedCursor(DeclarativeCursor):
|
|
133
133
|
:param stream_state: The state of the stream as returned by get_stream_state
|
134
134
|
"""
|
135
135
|
self._cursor = (
|
136
|
-
stream_state.get(self.cursor_field.eval(self.config)) if stream_state else None
|
137
|
-
)
|
136
|
+
stream_state.get(self.cursor_field.eval(self.config)) if stream_state else None
|
137
|
+
) # type: ignore # cursor_field is converted to an InterpolatedString in __post_init__
|
138
138
|
|
139
139
|
def observe(self, stream_slice: StreamSlice, record: Record) -> None:
|
140
140
|
"""
|
@@ -158,10 +158,8 @@ class DatetimeBasedCursor(DeclarativeCursor):
|
|
158
158
|
)
|
159
159
|
if (
|
160
160
|
self._is_within_daterange_boundaries(
|
161
|
-
record,
|
162
|
-
|
163
|
-
stream_slice.get(end_field), # type: ignore [arg-type]
|
164
|
-
)
|
161
|
+
record, stream_slice.get(start_field), stream_slice.get(end_field)
|
162
|
+
) # type: ignore # we know that stream_slices for these cursors will use a string representing an unparsed date
|
165
163
|
and is_highest_observed_cursor_value
|
166
164
|
):
|
167
165
|
self._highest_observed_cursor_field_value = record_cursor_value
|
@@ -370,9 +368,9 @@ class DatetimeBasedCursor(DeclarativeCursor):
|
|
370
368
|
self._partition_field_start.eval(self.config)
|
371
369
|
)
|
372
370
|
if self.end_time_option and self.end_time_option.inject_into == option_type:
|
373
|
-
options[self.end_time_option.field_name.eval(config=self.config)] = stream_slice.get(
|
371
|
+
options[self.end_time_option.field_name.eval(config=self.config)] = stream_slice.get(
|
374
372
|
self._partition_field_end.eval(self.config)
|
375
|
-
)
|
373
|
+
) # type: ignore # field_name is always casted to an interpolated string
|
376
374
|
return options
|
377
375
|
|
378
376
|
def should_be_synced(self, record: Record) -> bool:
|
@@ -27,7 +27,7 @@ class StreamPartitionAccessEnvironment(SandboxedEnvironment):
|
|
27
27
|
def is_safe_attribute(self, obj: Any, attr: str, value: Any) -> bool:
|
28
28
|
if attr in ["_partition"]:
|
29
29
|
return True
|
30
|
-
return super().is_safe_attribute(obj, attr, value)
|
30
|
+
return super().is_safe_attribute(obj, attr, value) # type: ignore # for some reason, mypy says 'Returning Any from function declared to return "bool"'
|
31
31
|
|
32
32
|
|
33
33
|
class JinjaInterpolation(Interpolation):
|
@@ -132,7 +132,7 @@ class JinjaInterpolation(Interpolation):
|
|
132
132
|
return s
|
133
133
|
|
134
134
|
@cache
|
135
|
-
def _find_undeclared_variables(self, s: Optional[str]) ->
|
135
|
+
def _find_undeclared_variables(self, s: Optional[str]) -> Template:
|
136
136
|
"""
|
137
137
|
Find undeclared variables and cache them
|
138
138
|
"""
|
@@ -144,4 +144,4 @@ class JinjaInterpolation(Interpolation):
|
|
144
144
|
"""
|
145
145
|
We must cache the Jinja Template ourselves because we're using `from_string` instead of a template loader
|
146
146
|
"""
|
147
|
-
return self._environment.from_string(s)
|
147
|
+
return self._environment.from_string(s)
|
@@ -116,7 +116,7 @@ def duration(datestring: str) -> Union[datetime.timedelta, isodate.Duration]:
|
|
116
116
|
Usage:
|
117
117
|
`"{{ now_utc() - duration('P1D') }}"`
|
118
118
|
"""
|
119
|
-
return parse_duration(datestring)
|
119
|
+
return parse_duration(datestring) # type: ignore # mypy thinks this returns Any for some reason
|
120
120
|
|
121
121
|
|
122
122
|
def format_datetime(
|
@@ -396,7 +396,7 @@ class ModelToComponentFactory:
|
|
396
396
|
self._disable_retries = disable_retries
|
397
397
|
self._disable_cache = disable_cache
|
398
398
|
self._disable_resumable_full_refresh = disable_resumable_full_refresh
|
399
|
-
self._message_repository = message_repository or InMemoryMessageRepository(
|
399
|
+
self._message_repository = message_repository or InMemoryMessageRepository( # type: ignore
|
400
400
|
self._evaluate_log_level(emit_connector_builder_messages)
|
401
401
|
)
|
402
402
|
|
@@ -644,7 +644,7 @@ class ModelToComponentFactory:
|
|
644
644
|
declarative_stream.incremental_sync, # type: ignore # was already checked. Migration can be applied only to incremental streams.
|
645
645
|
config,
|
646
646
|
declarative_stream.parameters, # type: ignore # different type is expected here Mapping[str, Any], got Dict[str, Any]
|
647
|
-
)
|
647
|
+
) # type: ignore # The retriever type was already checked
|
648
648
|
|
649
649
|
def create_session_token_authenticator(
|
650
650
|
self, model: SessionTokenAuthenticatorModel, config: Config, name: str, **kwargs: Any
|
@@ -674,7 +674,7 @@ class ModelToComponentFactory:
|
|
674
674
|
return ModelToComponentFactory.create_bearer_authenticator(
|
675
675
|
BearerAuthenticatorModel(type="BearerAuthenticator", api_token=""), # type: ignore # $parameters has a default value
|
676
676
|
config,
|
677
|
-
token_provider=token_provider,
|
677
|
+
token_provider=token_provider, # type: ignore # $parameters defaults to None
|
678
678
|
)
|
679
679
|
else:
|
680
680
|
return ModelToComponentFactory.create_api_key_authenticator(
|
@@ -821,6 +821,7 @@ class ModelToComponentFactory:
|
|
821
821
|
input_datetime_formats=datetime_based_cursor_model.cursor_datetime_formats,
|
822
822
|
is_sequential_state=True,
|
823
823
|
cursor_granularity=cursor_granularity,
|
824
|
+
# type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice
|
824
825
|
)
|
825
826
|
|
826
827
|
start_date_runtime_value: Union[InterpolatedString, str, MinMaxDatetime]
|
@@ -893,7 +894,7 @@ class ModelToComponentFactory:
|
|
893
894
|
stream_name=stream_name,
|
894
895
|
stream_namespace=stream_namespace,
|
895
896
|
stream_state=stream_state,
|
896
|
-
message_repository=self._message_repository,
|
897
|
+
message_repository=self._message_repository, # type: ignore # message_repository is always instantiated with a value by factory
|
897
898
|
connector_state_manager=state_manager,
|
898
899
|
connector_state_converter=connector_state_converter,
|
899
900
|
cursor_field=cursor_field,
|
@@ -1704,7 +1705,7 @@ class ModelToComponentFactory:
|
|
1704
1705
|
refresh_token=model.refresh_token,
|
1705
1706
|
scopes=model.scopes,
|
1706
1707
|
token_expiry_date=model.token_expiry_date,
|
1707
|
-
token_expiry_date_format=model.token_expiry_date_format,
|
1708
|
+
token_expiry_date_format=model.token_expiry_date_format, # type: ignore
|
1708
1709
|
token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format),
|
1709
1710
|
token_refresh_endpoint=model.token_refresh_endpoint,
|
1710
1711
|
config=config,
|