airbyte-cdk 6.60.10__py3-none-any.whl → 6.60.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py +6 -9
- airbyte_cdk/sources/declarative/checks/check_stream.py +31 -12
- airbyte_cdk/sources/declarative/concurrent_declarative_source.py +0 -6
- airbyte_cdk/sources/file_based/availability_strategy/__init__.py +1 -5
- airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py +1 -27
- airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +0 -1
- airbyte_cdk/sources/file_based/stream/concurrent/adapters.py +0 -2
- airbyte_cdk/sources/streams/availability_strategy.py +1 -0
- airbyte_cdk/sources/streams/concurrent/abstract_stream.py +6 -6
- airbyte_cdk/sources/streams/concurrent/adapters.py +0 -43
- airbyte_cdk/sources/streams/concurrent/availability_strategy.py +19 -76
- airbyte_cdk/sources/streams/concurrent/default_stream.py +32 -9
- {airbyte_cdk-6.60.10.dist-info → airbyte_cdk-6.60.11.dist-info}/METADATA +1 -1
- {airbyte_cdk-6.60.10.dist-info → airbyte_cdk-6.60.11.dist-info}/RECORD +18 -18
- {airbyte_cdk-6.60.10.dist-info → airbyte_cdk-6.60.11.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.60.10.dist-info → airbyte_cdk-6.60.11.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.60.10.dist-info → airbyte_cdk-6.60.11.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.60.10.dist-info → airbyte_cdk-6.60.11.dist-info}/entry_points.txt +0 -0
@@ -3,12 +3,13 @@
|
|
3
3
|
#
|
4
4
|
|
5
5
|
import logging
|
6
|
-
import traceback
|
7
6
|
from dataclasses import InitVar, dataclass
|
8
|
-
from typing import Any, List, Mapping, Tuple
|
7
|
+
from typing import Any, List, Mapping, Tuple, Union
|
9
8
|
|
10
|
-
from airbyte_cdk import AbstractSource
|
9
|
+
from airbyte_cdk.sources.abstract_source import AbstractSource
|
10
|
+
from airbyte_cdk.sources.declarative.checks.check_stream import evaluate_availability
|
11
11
|
from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
|
12
|
+
from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
|
12
13
|
from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy
|
13
14
|
|
14
15
|
|
@@ -34,20 +35,16 @@ class CheckDynamicStream(ConnectionChecker):
|
|
34
35
|
def check_connection(
|
35
36
|
self, source: AbstractSource, logger: logging.Logger, config: Mapping[str, Any]
|
36
37
|
) -> Tuple[bool, Any]:
|
37
|
-
streams = source.streams(config=config)
|
38
|
+
streams: List[Union[Stream, AbstractStream]] = source.streams(config=config) # type: ignore # this is a migration step and we expect the declarative CDK to migrate off of ConnectionChecker
|
38
39
|
|
39
40
|
if len(streams) == 0:
|
40
41
|
return False, f"No streams to connect to from source {source}"
|
41
42
|
if not self.use_check_availability:
|
42
43
|
return True, None
|
43
44
|
|
44
|
-
availability_strategy = HttpAvailabilityStrategy()
|
45
|
-
|
46
45
|
try:
|
47
46
|
for stream in streams[: min(self.stream_count, len(streams))]:
|
48
|
-
stream_is_available, reason =
|
49
|
-
stream, logger
|
50
|
-
)
|
47
|
+
stream_is_available, reason = evaluate_availability(stream, logger)
|
51
48
|
if not stream_is_available:
|
52
49
|
logger.warning(f"Stream {stream.name} is not available: {reason}")
|
53
50
|
return False, reason
|
@@ -5,13 +5,30 @@
|
|
5
5
|
import logging
|
6
6
|
import traceback
|
7
7
|
from dataclasses import InitVar, dataclass
|
8
|
-
from typing import Any, Dict, List, Mapping, Optional, Tuple
|
8
|
+
from typing import Any, Dict, List, Mapping, Optional, Tuple, Union
|
9
9
|
|
10
|
-
from airbyte_cdk import AbstractSource
|
10
|
+
from airbyte_cdk.sources.abstract_source import AbstractSource
|
11
11
|
from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
|
12
|
+
from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
|
13
|
+
from airbyte_cdk.sources.streams.core import Stream
|
12
14
|
from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy
|
13
15
|
|
14
16
|
|
17
|
+
def evaluate_availability(
|
18
|
+
stream: Union[Stream, AbstractStream], logger: logging.Logger
|
19
|
+
) -> Tuple[bool, Optional[str]]:
|
20
|
+
"""
|
21
|
+
As a transition period, we want to support both Stream and AbstractStream until we migrate everything to AbstractStream.
|
22
|
+
"""
|
23
|
+
if isinstance(stream, Stream):
|
24
|
+
return HttpAvailabilityStrategy().check_availability(stream, logger)
|
25
|
+
elif isinstance(stream, AbstractStream):
|
26
|
+
availability = stream.check_availability()
|
27
|
+
return availability.is_available, availability.reason
|
28
|
+
else:
|
29
|
+
raise ValueError(f"Unsupported stream type {type(stream)}")
|
30
|
+
|
31
|
+
|
15
32
|
@dataclass(frozen=True)
|
16
33
|
class DynamicStreamCheckConfig:
|
17
34
|
"""Defines the configuration for dynamic stream during connection checking. This class specifies
|
@@ -51,7 +68,7 @@ class CheckStream(ConnectionChecker):
|
|
51
68
|
) -> Tuple[bool, Any]:
|
52
69
|
"""Checks the connection to the source and its streams."""
|
53
70
|
try:
|
54
|
-
streams = source.streams(config=config)
|
71
|
+
streams: List[Union[Stream, AbstractStream]] = source.streams(config=config) # type: ignore # this is a migration step and we expect the declarative CDK to migrate off of ConnectionChecker
|
55
72
|
if not streams:
|
56
73
|
return False, f"No streams to connect to from source {source}"
|
57
74
|
except Exception as error:
|
@@ -82,13 +99,15 @@ class CheckStream(ConnectionChecker):
|
|
82
99
|
return True, None
|
83
100
|
|
84
101
|
def _check_stream_availability(
|
85
|
-
self,
|
102
|
+
self,
|
103
|
+
stream_name_to_stream: Dict[str, Union[Stream, AbstractStream]],
|
104
|
+
stream_name: str,
|
105
|
+
logger: logging.Logger,
|
86
106
|
) -> Tuple[bool, Any]:
|
87
107
|
"""Checks if streams are available."""
|
88
|
-
availability_strategy = HttpAvailabilityStrategy()
|
89
108
|
try:
|
90
109
|
stream = stream_name_to_stream[stream_name]
|
91
|
-
stream_is_available, reason =
|
110
|
+
stream_is_available, reason = evaluate_availability(stream, logger)
|
92
111
|
if not stream_is_available:
|
93
112
|
message = f"Stream {stream_name} is not available: {reason}"
|
94
113
|
logger.warning(message)
|
@@ -98,7 +117,10 @@ class CheckStream(ConnectionChecker):
|
|
98
117
|
return True, None
|
99
118
|
|
100
119
|
def _check_dynamic_streams_availability(
|
101
|
-
self,
|
120
|
+
self,
|
121
|
+
source: AbstractSource,
|
122
|
+
stream_name_to_stream: Dict[str, Union[Stream, AbstractStream]],
|
123
|
+
logger: logging.Logger,
|
102
124
|
) -> Tuple[bool, Any]:
|
103
125
|
"""Checks the availability of dynamic streams."""
|
104
126
|
dynamic_streams = source.resolved_manifest.get("dynamic_streams", []) # type: ignore[attr-defined] # The source's resolved_manifest manifest is checked before calling this method
|
@@ -135,18 +157,15 @@ class CheckStream(ConnectionChecker):
|
|
135
157
|
def _check_generated_streams_availability(
|
136
158
|
self,
|
137
159
|
generated_streams: List[Dict[str, Any]],
|
138
|
-
stream_name_to_stream: Dict[str,
|
160
|
+
stream_name_to_stream: Dict[str, Union[Stream, AbstractStream]],
|
139
161
|
logger: logging.Logger,
|
140
162
|
max_count: int,
|
141
163
|
) -> Tuple[bool, Any]:
|
142
164
|
"""Checks availability of generated dynamic streams."""
|
143
|
-
availability_strategy = HttpAvailabilityStrategy()
|
144
165
|
for declarative_stream in generated_streams[: min(max_count, len(generated_streams))]:
|
145
166
|
stream = stream_name_to_stream[declarative_stream["name"]]
|
146
167
|
try:
|
147
|
-
stream_is_available, reason =
|
148
|
-
stream, logger
|
149
|
-
)
|
168
|
+
stream_is_available, reason = evaluate_availability(stream, logger)
|
150
169
|
if not stream_is_available:
|
151
170
|
message = f"Dynamic Stream {stream.name} is not available: {reason}"
|
152
171
|
logger.warning(message)
|
@@ -52,9 +52,6 @@ from airbyte_cdk.sources.source import TState
|
|
52
52
|
from airbyte_cdk.sources.streams import Stream
|
53
53
|
from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
|
54
54
|
from airbyte_cdk.sources.streams.concurrent.abstract_stream_facade import AbstractStreamFacade
|
55
|
-
from airbyte_cdk.sources.streams.concurrent.availability_strategy import (
|
56
|
-
AlwaysAvailableAvailabilityStrategy,
|
57
|
-
)
|
58
55
|
from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor, FinalStateCursor
|
59
56
|
from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
|
60
57
|
from airbyte_cdk.sources.streams.concurrent.helpers import get_primary_key_from_stream
|
@@ -325,7 +322,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
325
322
|
partition_generator=partition_generator,
|
326
323
|
name=declarative_stream.name,
|
327
324
|
json_schema=declarative_stream.get_json_schema(),
|
328
|
-
availability_strategy=AlwaysAvailableAvailabilityStrategy(),
|
329
325
|
primary_key=get_primary_key_from_stream(declarative_stream.primary_key),
|
330
326
|
cursor_field=cursor.cursor_field.cursor_field_key
|
331
327
|
if hasattr(cursor, "cursor_field")
|
@@ -362,7 +358,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
362
358
|
partition_generator=partition_generator,
|
363
359
|
name=declarative_stream.name,
|
364
360
|
json_schema=declarative_stream.get_json_schema(),
|
365
|
-
availability_strategy=AlwaysAvailableAvailabilityStrategy(),
|
366
361
|
primary_key=get_primary_key_from_stream(declarative_stream.primary_key),
|
367
362
|
cursor_field=None,
|
368
363
|
logger=self.logger,
|
@@ -417,7 +412,6 @@ class ConcurrentDeclarativeSource(ManifestDeclarativeSource, Generic[TState]):
|
|
417
412
|
partition_generator=partition_generator,
|
418
413
|
name=declarative_stream.name,
|
419
414
|
json_schema=declarative_stream.get_json_schema(),
|
420
|
-
availability_strategy=AlwaysAvailableAvailabilityStrategy(),
|
421
415
|
primary_key=get_primary_key_from_stream(declarative_stream.primary_key),
|
422
416
|
cursor_field=perpartition_cursor.cursor_field.cursor_field_key,
|
423
417
|
logger=self.logger,
|
@@ -1,11 +1,7 @@
|
|
1
|
-
from .abstract_file_based_availability_strategy import
|
2
|
-
AbstractFileBasedAvailabilityStrategy,
|
3
|
-
AbstractFileBasedAvailabilityStrategyWrapper,
|
4
|
-
)
|
1
|
+
from .abstract_file_based_availability_strategy import AbstractFileBasedAvailabilityStrategy
|
5
2
|
from .default_file_based_availability_strategy import DefaultFileBasedAvailabilityStrategy
|
6
3
|
|
7
4
|
__all__ = [
|
8
5
|
"AbstractFileBasedAvailabilityStrategy",
|
9
|
-
"AbstractFileBasedAvailabilityStrategyWrapper",
|
10
6
|
"DefaultFileBasedAvailabilityStrategy",
|
11
7
|
]
|
airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py
CHANGED
@@ -10,12 +10,6 @@ from typing import TYPE_CHECKING, Optional, Tuple
|
|
10
10
|
|
11
11
|
from airbyte_cdk.sources import Source
|
12
12
|
from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy
|
13
|
-
from airbyte_cdk.sources.streams.concurrent.availability_strategy import (
|
14
|
-
AbstractAvailabilityStrategy,
|
15
|
-
StreamAvailability,
|
16
|
-
StreamAvailable,
|
17
|
-
StreamUnavailable,
|
18
|
-
)
|
19
13
|
from airbyte_cdk.sources.streams.core import Stream
|
20
14
|
|
21
15
|
if TYPE_CHECKING:
|
@@ -28,7 +22,7 @@ class AbstractFileBasedAvailabilityStrategy(AvailabilityStrategy):
|
|
28
22
|
self,
|
29
23
|
stream: Stream,
|
30
24
|
logger: logging.Logger,
|
31
|
-
|
25
|
+
source: Optional[Source] = None,
|
32
26
|
) -> Tuple[bool, Optional[str]]:
|
33
27
|
"""
|
34
28
|
Perform a connection check for the stream.
|
@@ -51,23 +45,3 @@ class AbstractFileBasedAvailabilityStrategy(AvailabilityStrategy):
|
|
51
45
|
Returns (True, None) if successful, otherwise (False, <error message>).
|
52
46
|
"""
|
53
47
|
...
|
54
|
-
|
55
|
-
|
56
|
-
class AbstractFileBasedAvailabilityStrategyWrapper(AbstractAvailabilityStrategy):
|
57
|
-
def __init__(self, stream: AbstractFileBasedStream) -> None:
|
58
|
-
self.stream = stream
|
59
|
-
|
60
|
-
def check_availability(self, logger: logging.Logger) -> StreamAvailability:
|
61
|
-
is_available, reason = self.stream.availability_strategy.check_availability(
|
62
|
-
self.stream, logger, None
|
63
|
-
)
|
64
|
-
if is_available:
|
65
|
-
return StreamAvailable()
|
66
|
-
return StreamUnavailable(reason or "")
|
67
|
-
|
68
|
-
def check_availability_and_parsability(
|
69
|
-
self, logger: logging.Logger
|
70
|
-
) -> Tuple[bool, Optional[str]]:
|
71
|
-
return self.stream.availability_strategy.check_availability_and_parsability(
|
72
|
-
self.stream, logger, None
|
73
|
-
)
|
@@ -21,7 +21,6 @@ from airbyte_cdk.sources import AbstractSource
|
|
21
21
|
from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
|
22
22
|
from airbyte_cdk.sources.file_based.availability_strategy import (
|
23
23
|
AbstractFileBasedAvailabilityStrategy,
|
24
|
-
AbstractFileBasedAvailabilityStrategyWrapper,
|
25
24
|
)
|
26
25
|
from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
|
27
26
|
from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser
|
@@ -97,7 +96,6 @@ class FileBasedStreamFacade(AbstractStreamFacade[DefaultStream], AbstractFileBas
|
|
97
96
|
),
|
98
97
|
name=stream.name,
|
99
98
|
json_schema=stream.get_json_schema(),
|
100
|
-
availability_strategy=AbstractFileBasedAvailabilityStrategyWrapper(stream),
|
101
99
|
primary_key=pk,
|
102
100
|
cursor_field=cursor_field,
|
103
101
|
logger=logger,
|
@@ -64,12 +64,6 @@ class AbstractStream(ABC):
|
|
64
64
|
:return: The name of the field used as a cursor. Nested cursor fields are not supported.
|
65
65
|
"""
|
66
66
|
|
67
|
-
@abstractmethod
|
68
|
-
def check_availability(self) -> StreamAvailability:
|
69
|
-
"""
|
70
|
-
:return: The stream's availability
|
71
|
-
"""
|
72
|
-
|
73
67
|
@abstractmethod
|
74
68
|
def get_json_schema(self) -> Mapping[str, Any]:
|
75
69
|
"""
|
@@ -94,3 +88,9 @@ class AbstractStream(ABC):
|
|
94
88
|
"""
|
95
89
|
:return: The cursor associated with this stream.
|
96
90
|
"""
|
91
|
+
|
92
|
+
@abstractmethod
|
93
|
+
def check_availability(self) -> StreamAvailability:
|
94
|
+
"""
|
95
|
+
:return: If the stream is available and if not, why
|
96
|
+
"""
|
@@ -24,12 +24,7 @@ from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager
|
|
24
24
|
from airbyte_cdk.sources.message import MessageRepository
|
25
25
|
from airbyte_cdk.sources.source import ExperimentalClassWarning
|
26
26
|
from airbyte_cdk.sources.streams import Stream
|
27
|
-
from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy
|
28
27
|
from airbyte_cdk.sources.streams.concurrent.abstract_stream_facade import AbstractStreamFacade
|
29
|
-
from airbyte_cdk.sources.streams.concurrent.availability_strategy import (
|
30
|
-
AbstractAvailabilityStrategy,
|
31
|
-
AlwaysAvailableAvailabilityStrategy,
|
32
|
-
)
|
33
28
|
from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, FinalStateCursor
|
34
29
|
from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream
|
35
30
|
from airbyte_cdk.sources.streams.concurrent.exceptions import ExceptionWithDisplayMessage
|
@@ -101,7 +96,6 @@ class StreamFacade(AbstractStreamFacade[DefaultStream], Stream):
|
|
101
96
|
name=stream.name,
|
102
97
|
namespace=stream.namespace,
|
103
98
|
json_schema=stream.get_json_schema(),
|
104
|
-
availability_strategy=AlwaysAvailableAvailabilityStrategy(),
|
105
99
|
primary_key=pk,
|
106
100
|
cursor_field=cursor_field,
|
107
101
|
logger=logger,
|
@@ -210,18 +204,6 @@ class StreamFacade(AbstractStreamFacade[DefaultStream], Stream):
|
|
210
204
|
def supports_incremental(self) -> bool:
|
211
205
|
return self._legacy_stream.supports_incremental
|
212
206
|
|
213
|
-
def check_availability(
|
214
|
-
self, logger: logging.Logger, source: Optional["Source"] = None
|
215
|
-
) -> Tuple[bool, Optional[str]]:
|
216
|
-
"""
|
217
|
-
Verifies the stream is available. Delegates to the underlying AbstractStream and ignores the parameters
|
218
|
-
:param logger: (ignored)
|
219
|
-
:param source: (ignored)
|
220
|
-
:return:
|
221
|
-
"""
|
222
|
-
availability = self._abstract_stream.check_availability()
|
223
|
-
return availability.is_available(), availability.message()
|
224
|
-
|
225
207
|
def as_airbyte_stream(self) -> AirbyteStream:
|
226
208
|
return self._abstract_stream.as_airbyte_stream()
|
227
209
|
|
@@ -370,28 +352,3 @@ class StreamPartitionGenerator(PartitionGenerator):
|
|
370
352
|
self._cursor_field,
|
371
353
|
self._state,
|
372
354
|
)
|
373
|
-
|
374
|
-
|
375
|
-
@deprecated(
|
376
|
-
"Availability strategy has been soft deprecated. Do not use. Class is subject to removal",
|
377
|
-
category=ExperimentalClassWarning,
|
378
|
-
)
|
379
|
-
class AvailabilityStrategyFacade(AvailabilityStrategy):
|
380
|
-
def __init__(self, abstract_availability_strategy: AbstractAvailabilityStrategy):
|
381
|
-
self._abstract_availability_strategy = abstract_availability_strategy
|
382
|
-
|
383
|
-
def check_availability(
|
384
|
-
self, stream: Stream, logger: logging.Logger, source: Optional["Source"] = None
|
385
|
-
) -> Tuple[bool, Optional[str]]:
|
386
|
-
"""
|
387
|
-
Checks stream availability.
|
388
|
-
|
389
|
-
Important to note that the stream and source parameters are not used by the underlying AbstractAvailabilityStrategy.
|
390
|
-
|
391
|
-
:param stream: (unused)
|
392
|
-
:param logger: logger object to use
|
393
|
-
:param source: (unused)
|
394
|
-
:return: A tuple of (boolean, str). If boolean is true, then the stream
|
395
|
-
"""
|
396
|
-
stream_availability = self._abstract_availability_strategy.check_availability(logger)
|
397
|
-
return stream_availability.is_available(), stream_availability.message()
|
@@ -2,93 +2,36 @@
|
|
2
2
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
3
3
|
#
|
4
4
|
|
5
|
-
import logging
|
6
5
|
from abc import ABC, abstractmethod
|
7
6
|
from typing import Optional
|
8
7
|
|
9
|
-
from typing_extensions import deprecated
|
10
8
|
|
11
|
-
|
9
|
+
class StreamAvailability:
|
10
|
+
@classmethod
|
11
|
+
def available(cls) -> "StreamAvailability":
|
12
|
+
return cls(True)
|
12
13
|
|
14
|
+
@classmethod
|
15
|
+
def unavailable(cls, reason: str) -> "StreamAvailability":
|
16
|
+
return cls(False, reason)
|
13
17
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
"""
|
18
|
-
:return: True if the stream is available. False if the stream is not
|
19
|
-
"""
|
20
|
-
|
21
|
-
@abstractmethod
|
22
|
-
def message(self) -> Optional[str]:
|
23
|
-
"""
|
24
|
-
:return: A message describing why the stream is not available. If the stream is available, this should return None.
|
25
|
-
"""
|
18
|
+
def __init__(self, available: bool, reason: Optional[str] = None) -> None:
|
19
|
+
self._available = available
|
20
|
+
self._reason = reason
|
26
21
|
|
22
|
+
if not available:
|
23
|
+
assert reason, "A reason needs to be provided if the stream is not available"
|
27
24
|
|
28
|
-
|
25
|
+
@property
|
29
26
|
def is_available(self) -> bool:
|
30
|
-
return True
|
31
|
-
|
32
|
-
def message(self) -> Optional[str]:
|
33
|
-
return None
|
34
|
-
|
35
|
-
|
36
|
-
class StreamUnavailable(StreamAvailability):
|
37
|
-
def __init__(self, message: str):
|
38
|
-
self._message = message
|
39
|
-
|
40
|
-
def is_available(self) -> bool:
|
41
|
-
return False
|
42
|
-
|
43
|
-
def message(self) -> Optional[str]:
|
44
|
-
return self._message
|
45
|
-
|
46
|
-
|
47
|
-
# Singleton instances of StreamAvailability to avoid the overhead of creating new dummy objects
|
48
|
-
STREAM_AVAILABLE = StreamAvailable()
|
49
|
-
|
50
|
-
|
51
|
-
@deprecated(
|
52
|
-
"This class is experimental. Use at your own risk.",
|
53
|
-
category=ExperimentalClassWarning,
|
54
|
-
)
|
55
|
-
class AbstractAvailabilityStrategy(ABC):
|
56
|
-
"""
|
57
|
-
AbstractAvailabilityStrategy is an experimental interface developed as part of the Concurrent CDK.
|
58
|
-
This interface is not yet stable and may change in the future. Use at your own risk.
|
59
|
-
|
60
|
-
Why create a new interface instead of using the existing AvailabilityStrategy?
|
61
|
-
The existing AvailabilityStrategy is tightly coupled with Stream and Source, which yields to circular dependencies and makes it difficult to move away from the Stream interface to AbstractStream.
|
62
|
-
"""
|
63
|
-
|
64
|
-
@abstractmethod
|
65
|
-
def check_availability(self, logger: logging.Logger) -> StreamAvailability:
|
66
27
|
"""
|
67
|
-
|
68
|
-
|
69
|
-
:param logger: logger object to use
|
70
|
-
:return: A StreamAvailability object describing the stream's availability
|
28
|
+
:return: True if the stream is available. False if the stream is not
|
71
29
|
"""
|
30
|
+
return self._available
|
72
31
|
|
73
|
-
|
74
|
-
|
75
|
-
"This class is experimental. Use at your own risk.",
|
76
|
-
category=ExperimentalClassWarning,
|
77
|
-
)
|
78
|
-
class AlwaysAvailableAvailabilityStrategy(AbstractAvailabilityStrategy):
|
79
|
-
"""
|
80
|
-
An availability strategy that always indicates a stream is available.
|
81
|
-
|
82
|
-
This strategy is used to avoid breaking changes and serves as a soft
|
83
|
-
deprecation of the availability strategy, allowing a smoother transition
|
84
|
-
without disrupting existing functionality.
|
85
|
-
"""
|
86
|
-
|
87
|
-
def check_availability(self, logger: logging.Logger) -> StreamAvailability:
|
32
|
+
@property
|
33
|
+
def reason(self) -> Optional[str]:
|
88
34
|
"""
|
89
|
-
|
90
|
-
|
91
|
-
:param logger: logger object to use
|
92
|
-
:return: A StreamAvailability object describing the stream's availability
|
35
|
+
:return: A message describing why the stream is not available. If the stream is available, this should return None.
|
93
36
|
"""
|
94
|
-
return
|
37
|
+
return self._reason
|
@@ -8,13 +8,11 @@ from typing import Any, Iterable, List, Mapping, Optional
|
|
8
8
|
|
9
9
|
from airbyte_cdk.models import AirbyteStream, SyncMode
|
10
10
|
from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream
|
11
|
-
from airbyte_cdk.sources.streams.concurrent.availability_strategy import
|
12
|
-
AbstractAvailabilityStrategy,
|
13
|
-
StreamAvailability,
|
14
|
-
)
|
11
|
+
from airbyte_cdk.sources.streams.concurrent.availability_strategy import StreamAvailability
|
15
12
|
from airbyte_cdk.sources.streams.concurrent.cursor import Cursor
|
16
13
|
from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition
|
17
14
|
from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator
|
15
|
+
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
18
16
|
|
19
17
|
|
20
18
|
class DefaultStream(AbstractStream):
|
@@ -23,7 +21,6 @@ class DefaultStream(AbstractStream):
|
|
23
21
|
partition_generator: PartitionGenerator,
|
24
22
|
name: str,
|
25
23
|
json_schema: Mapping[str, Any],
|
26
|
-
availability_strategy: AbstractAvailabilityStrategy,
|
27
24
|
primary_key: List[str],
|
28
25
|
cursor_field: Optional[str],
|
29
26
|
logger: Logger,
|
@@ -34,7 +31,6 @@ class DefaultStream(AbstractStream):
|
|
34
31
|
self._stream_partition_generator = partition_generator
|
35
32
|
self._name = name
|
36
33
|
self._json_schema = json_schema
|
37
|
-
self._availability_strategy = availability_strategy
|
38
34
|
self._primary_key = primary_key
|
39
35
|
self._cursor_field = cursor_field
|
40
36
|
self._logger = logger
|
@@ -53,9 +49,6 @@ class DefaultStream(AbstractStream):
|
|
53
49
|
def namespace(self) -> Optional[str]:
|
54
50
|
return self._namespace
|
55
51
|
|
56
|
-
def check_availability(self) -> StreamAvailability:
|
57
|
-
return self._availability_strategy.check_availability(self._logger)
|
58
|
-
|
59
52
|
@property
|
60
53
|
def cursor_field(self) -> Optional[str]:
|
61
54
|
return self._cursor_field
|
@@ -100,3 +93,33 @@ class DefaultStream(AbstractStream):
|
|
100
93
|
@property
|
101
94
|
def cursor(self) -> Cursor:
|
102
95
|
return self._cursor
|
96
|
+
|
97
|
+
def check_availability(self) -> StreamAvailability:
|
98
|
+
"""
|
99
|
+
Check stream availability by attempting to read the first record of the stream.
|
100
|
+
"""
|
101
|
+
try:
|
102
|
+
partition = next(iter(self.generate_partitions()))
|
103
|
+
except StopIteration:
|
104
|
+
# NOTE: The following comment was copied from legacy stuff and I don't know how relevant it is:
|
105
|
+
# If stream_slices has no `next()` item (Note - this is different from stream_slices returning [None]!)
|
106
|
+
# This can happen when a substream's `stream_slices` method does a `for record in parent_records: yield <something>`
|
107
|
+
# without accounting for the case in which the parent stream is empty.
|
108
|
+
return StreamAvailability.unavailable(
|
109
|
+
f"Cannot attempt to connect to stream {self.name} - no stream slices were found"
|
110
|
+
)
|
111
|
+
except AirbyteTracedException as error:
|
112
|
+
return StreamAvailability.unavailable(
|
113
|
+
error.message or error.internal_message or "<no error message>"
|
114
|
+
)
|
115
|
+
|
116
|
+
try:
|
117
|
+
next(iter(partition.read()))
|
118
|
+
return StreamAvailability.available()
|
119
|
+
except StopIteration:
|
120
|
+
self._logger.info(f"Successfully connected to stream {self.name}, but got 0 records.")
|
121
|
+
return StreamAvailability.available()
|
122
|
+
except AirbyteTracedException as error:
|
123
|
+
return StreamAvailability.unavailable(
|
124
|
+
error.message or error.internal_message or "<no error message>"
|
125
|
+
)
|
@@ -81,12 +81,12 @@ airbyte_cdk/sources/declarative/auth/selective_authenticator.py,sha256=qGwC6YsCl
|
|
81
81
|
airbyte_cdk/sources/declarative/auth/token.py,sha256=2EnE78EhBOY9hbeZnQJ9AuFaM-G7dccU-oKo_LThRQk,11070
|
82
82
|
airbyte_cdk/sources/declarative/auth/token_provider.py,sha256=Jzuxlmt1_-_aFC_n0OmP8L1nDOacLzbEVVx3kjdX_W8,3104
|
83
83
|
airbyte_cdk/sources/declarative/checks/__init__.py,sha256=cpoBwEbRNcMi7Rmi5pD63ppUvAOZsAWzasmc57ob9rc,873
|
84
|
-
airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py,sha256=
|
85
|
-
airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=
|
84
|
+
airbyte_cdk/sources/declarative/checks/check_dynamic_stream.py,sha256=AF3qzwvvln5Q7NgmdCUu5TBVLNhMLDNPIHop7Cq7ad0,2354
|
85
|
+
airbyte_cdk/sources/declarative/checks/check_stream.py,sha256=sV-ZY7dZ03V8GdAxPYyYJoGfYBSJZ6qxu_9lF1rGbpE,8012
|
86
86
|
airbyte_cdk/sources/declarative/checks/connection_checker.py,sha256=MBRJo6WJlZQHpIfOGaNOkkHUmgUl_4wDM6VPo41z5Ss,1383
|
87
87
|
airbyte_cdk/sources/declarative/concurrency_level/__init__.py,sha256=5XUqrmlstYlMM0j6crktlKQwALek0uiz2D3WdM46MyA,191
|
88
88
|
airbyte_cdk/sources/declarative/concurrency_level/concurrency_level.py,sha256=YIwCTCpOr_QSNW4ltQK0yUGWInI8PKNY216HOOegYLk,2101
|
89
|
-
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=
|
89
|
+
airbyte_cdk/sources/declarative/concurrent_declarative_source.py,sha256=_C0IKcPlumc3HAGxccmeyS8lSQo7HxPr4jcNFok_9oU,26637
|
90
90
|
airbyte_cdk/sources/declarative/datetime/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
91
91
|
airbyte_cdk/sources/declarative/datetime/datetime_parser.py,sha256=_zGNGq31RNy_0QBLt_EcTvgPyhj7urPdx6oA3M5-r3o,3150
|
92
92
|
airbyte_cdk/sources/declarative/datetime/min_max_datetime.py,sha256=0BHBtDNQZfvwM45-tY5pNlTcKAFSGGNxemoi0Jic-0E,5785
|
@@ -247,8 +247,8 @@ airbyte_cdk/sources/declarative/validators/validator.py,sha256=MAwo8OievUsuzBuPx
|
|
247
247
|
airbyte_cdk/sources/declarative/yaml_declarative_source.py,sha256=-pHwGO7ZW-x8lmsqSpbrN0pOgIyjJhFDGUNwB3kQWWc,2794
|
248
248
|
airbyte_cdk/sources/file_based/README.md,sha256=iMqww4VZ882jfNQIdljjDgqreKs-mkdtSrRKA94iX6A,11085
|
249
249
|
airbyte_cdk/sources/file_based/__init__.py,sha256=EaxHv_9ot-eRlUCR47ZMZ0IOtB-n0HH24om7Bfn-uuQ,868
|
250
|
-
airbyte_cdk/sources/file_based/availability_strategy/__init__.py,sha256=
|
251
|
-
airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py,sha256=
|
250
|
+
airbyte_cdk/sources/file_based/availability_strategy/__init__.py,sha256=VY-0PcTaPttXU_mGf-7w7u5VKQD7iQWJbKMvIYAOhcQ,288
|
251
|
+
airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py,sha256=VA3_qymdY74VRLcu7-waj7MAZaCCmlKDmvMT13lO9nc,1443
|
252
252
|
airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py,sha256=j9T5TimfWFUz7nqsaj-83G3xWmDpsmeSbDnaUNmz0UM,5849
|
253
253
|
airbyte_cdk/sources/file_based/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
254
254
|
airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py,sha256=purspcxMNWLhGunaqVJa9TISMZK7vlQSwdz4zykFWNA,6989
|
@@ -283,9 +283,9 @@ airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py,sha256=FkB
|
|
283
283
|
airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py,sha256=kjvX7nOmUALYd7HuZHilUzgJPZ-MnZ08mtvuBnt2tQ0,618
|
284
284
|
airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py,sha256=vjTlmYT_nqzY3DbT5xem7X-bwgA9RyXHoKFqiMO2URk,1728
|
285
285
|
airbyte_cdk/sources/file_based/stream/__init__.py,sha256=q_zmeOHHg0JK5j1YNSOIsyXGz-wlTl_0E8z5GKVAcVM,543
|
286
|
-
airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py,sha256=
|
286
|
+
airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py,sha256=fCF6lBKmPAFS2ZpJb9I0GIGwMxfCkA-41Yhs_fyLTxs,7454
|
287
287
|
airbyte_cdk/sources/file_based/stream/concurrent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
288
|
-
airbyte_cdk/sources/file_based/stream/concurrent/adapters.py,sha256=
|
288
|
+
airbyte_cdk/sources/file_based/stream/concurrent/adapters.py,sha256=MxK7urU9s4YeYA-4Kjx4gnJak9AyyojqNsW1bYtfbp8,13383
|
289
289
|
airbyte_cdk/sources/file_based/stream/concurrent/cursor/__init__.py,sha256=Rx7TwjH8B7e0eee83Tlqxv1bWn-BVXOmlUAH7auM1uM,344
|
290
290
|
airbyte_cdk/sources/file_based/stream/concurrent/cursor/abstract_concurrent_file_based_cursor.py,sha256=5dYZMLBEbvCyrCT89lCYdm2FdrLPLuxjdpQSVGP5o0w,1856
|
291
291
|
airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_concurrent_cursor.py,sha256=lubzER11aRWGureKhdvZqWraM7hOWiv-WrGQofnmcJU,14957
|
@@ -304,7 +304,7 @@ airbyte_cdk/sources/message/repository.py,sha256=SG7avgti_-dj8FcRHTTrhgLLGJbElv1
|
|
304
304
|
airbyte_cdk/sources/source.py,sha256=KIBBH5VLEb8BZ8B9aROlfaI6OLoJqKDPMJ10jkAR7nk,3611
|
305
305
|
airbyte_cdk/sources/specs/transfer_modes.py,sha256=sfSVO0yT6SaGKN5_TP0Nl_ftG0yPhecaBv0WkhAEXA8,932
|
306
306
|
airbyte_cdk/sources/streams/__init__.py,sha256=8fzTKpRTnSx5PggXgQPKJzHNZUV2BCA40N-dI6JM1xI,256
|
307
|
-
airbyte_cdk/sources/streams/availability_strategy.py,sha256
|
307
|
+
airbyte_cdk/sources/streams/availability_strategy.py,sha256=-WORUv8btPo9WI85M9sU43OCN9WE_DfdT9zMdS13sUo,3260
|
308
308
|
airbyte_cdk/sources/streams/call_rate.py,sha256=jRsGp1PDZBCDQNxzcGVnVmVzLk0wLHxS1JnJwMAgy9U,27568
|
309
309
|
airbyte_cdk/sources/streams/checkpoint/__init__.py,sha256=3oy7Hd4ivVWTZlN6dKAf4Fv_G7U5iZrvhO9hT871UIo,712
|
310
310
|
airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py,sha256=6HMT2NI-FQuaW0nt95NcyWrt5rZN4gF-Arx0sxdgbv4,15221
|
@@ -314,14 +314,14 @@ airbyte_cdk/sources/streams/checkpoint/resumable_full_refresh_cursor.py,sha256=G
|
|
314
314
|
airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py,sha256=qydclrks-hHJ9B9wOKvbu_FZXJOFiB3yD5U4KwsX80Y,4569
|
315
315
|
airbyte_cdk/sources/streams/concurrent/README.md,sha256=0nvgnlCBfZJiPDAofT8yFmUhGc4L99RCb3fL_PI4sSY,1070
|
316
316
|
airbyte_cdk/sources/streams/concurrent/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
317
|
-
airbyte_cdk/sources/streams/concurrent/abstract_stream.py,sha256=
|
317
|
+
airbyte_cdk/sources/streams/concurrent/abstract_stream.py,sha256=aalik3FvyEjoeA1S3wUYEV3bgQLGrTnhYKPvT-rgy9E,3919
|
318
318
|
airbyte_cdk/sources/streams/concurrent/abstract_stream_facade.py,sha256=QTry1QCBUwJDw1QSCEvz23s7zIEx_7QMxkPq9j-oPIQ,1358
|
319
|
-
airbyte_cdk/sources/streams/concurrent/adapters.py,sha256=
|
320
|
-
airbyte_cdk/sources/streams/concurrent/availability_strategy.py,sha256=
|
319
|
+
airbyte_cdk/sources/streams/concurrent/adapters.py,sha256=R9EZza1jF0e_HaLgN9Q_VREjvmlk0p9UfBLsnHB2y48,13936
|
320
|
+
airbyte_cdk/sources/streams/concurrent/availability_strategy.py,sha256=M0XmvF3vjlr4GbCM0XH1hAj7udiAONM9SnmXjqufzLM,1035
|
321
321
|
airbyte_cdk/sources/streams/concurrent/clamping.py,sha256=i26GVyui2ScEXSP-IP_61K2HaTp1-6lTlYHsZVYpuZA,3240
|
322
322
|
airbyte_cdk/sources/streams/concurrent/cursor.py,sha256=xFFB8eEbtjGUdb42vkyWT5JB-WTUsaJlZ0gjKoVEycc,22307
|
323
323
|
airbyte_cdk/sources/streams/concurrent/cursor_types.py,sha256=ZyWLPpeLX1qXcP5MwS-wxK11IBMsnVPCw9zx8gA2_Ro,843
|
324
|
-
airbyte_cdk/sources/streams/concurrent/default_stream.py,sha256=
|
324
|
+
airbyte_cdk/sources/streams/concurrent/default_stream.py,sha256=cVBMwCM8OVlsLA39e1YN_eGb7qApqH-GQtnWMs19sGU,4687
|
325
325
|
airbyte_cdk/sources/streams/concurrent/exceptions.py,sha256=JOZ446MCLpmF26r9KfS6OO_6rGjcjgJNZdcw6jccjEI,468
|
326
326
|
airbyte_cdk/sources/streams/concurrent/helpers.py,sha256=S6AW8TgIASCZ2UuUcQLE8OzgYUHWt2-KPOvNPwnQf-Q,1596
|
327
327
|
airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py,sha256=2t64b_z9cEPmlHZnjSiMTO8PEtEdiAJDG0JcYOtUqAE,3363
|
@@ -424,9 +424,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
|
|
424
424
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
425
425
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
426
426
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
427
|
-
airbyte_cdk-6.60.
|
428
|
-
airbyte_cdk-6.60.
|
429
|
-
airbyte_cdk-6.60.
|
430
|
-
airbyte_cdk-6.60.
|
431
|
-
airbyte_cdk-6.60.
|
432
|
-
airbyte_cdk-6.60.
|
427
|
+
airbyte_cdk-6.60.11.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
428
|
+
airbyte_cdk-6.60.11.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
429
|
+
airbyte_cdk-6.60.11.dist-info/METADATA,sha256=ThGpN0itMBbE8stLZwHsuUHiuHa_Zh5eUvybXg-wtTk,6478
|
430
|
+
airbyte_cdk-6.60.11.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
431
|
+
airbyte_cdk-6.60.11.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
|
432
|
+
airbyte_cdk-6.60.11.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|