airbyte-cdk 6.26.0.dev4108__py3-none-any.whl → 6.26.0.dev4109__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +1 -17
- airbyte_cdk/sources/file_based/config/validate_config_transfer_modes.py +20 -9
- airbyte_cdk/sources/file_based/file_based_source.py +4 -5
- airbyte_cdk/sources/file_based/file_based_stream_reader.py +24 -2
- airbyte_cdk/sources/file_based/schema_helpers.py +0 -25
- airbyte_cdk/sources/file_based/stream/__init__.py +2 -2
- airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +2 -3
- airbyte_cdk/sources/file_based/stream/identities_stream.py +6 -58
- airbyte_cdk/sources/specs/transfer_modes.py +26 -0
- airbyte_cdk/sources/streams/permissions/identities.py +77 -0
- {airbyte_cdk-6.26.0.dev4108.dist-info → airbyte_cdk-6.26.0.dev4109.dist-info}/METADATA +1 -1
- {airbyte_cdk-6.26.0.dev4108.dist-info → airbyte_cdk-6.26.0.dev4109.dist-info}/RECORD +16 -14
- {airbyte_cdk-6.26.0.dev4108.dist-info → airbyte_cdk-6.26.0.dev4109.dist-info}/LICENSE.txt +0 -0
- {airbyte_cdk-6.26.0.dev4108.dist-info → airbyte_cdk-6.26.0.dev4109.dist-info}/LICENSE_SHORT +0 -0
- {airbyte_cdk-6.26.0.dev4108.dist-info → airbyte_cdk-6.26.0.dev4109.dist-info}/WHEEL +0 -0
- {airbyte_cdk-6.26.0.dev4108.dist-info → airbyte_cdk-6.26.0.dev4109.dist-info}/entry_points.txt +0 -0
@@ -11,26 +11,10 @@ from pydantic.v1 import AnyUrl, BaseModel, Field
|
|
11
11
|
|
12
12
|
from airbyte_cdk import OneOfOptionConfig
|
13
13
|
from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig
|
14
|
+
from airbyte_cdk.sources.specs.transfer_modes import DeliverPermissions
|
14
15
|
from airbyte_cdk.sources.utils import schema_helpers
|
15
16
|
|
16
17
|
|
17
|
-
class DeliverPermissions(BaseModel):
|
18
|
-
class Config(OneOfOptionConfig):
|
19
|
-
title = "Replicate Permissions ACL"
|
20
|
-
description = "Sends one identity stream and one for more permissions (ACL) streams to the destination. This data can be used in downstream systems to recreate permission restrictions mirroring the original source."
|
21
|
-
discriminator = "delivery_type"
|
22
|
-
|
23
|
-
delivery_type: Literal["use_permissions_transfer"] = Field(
|
24
|
-
"use_permissions_transfer", const=True
|
25
|
-
)
|
26
|
-
|
27
|
-
include_identities_stream: bool = Field(
|
28
|
-
title="Include Identity Stream",
|
29
|
-
description="This data can be used in downstream systems to recreate permission restrictions mirroring the original source",
|
30
|
-
default=True,
|
31
|
-
)
|
32
|
-
|
33
|
-
|
34
18
|
class DeliverRecords(BaseModel):
|
35
19
|
class Config(OneOfOptionConfig):
|
36
20
|
title = "Replicate Records"
|
@@ -2,13 +2,23 @@
|
|
2
2
|
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
3
|
#
|
4
4
|
|
5
|
-
from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import
|
5
|
+
from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import (
|
6
|
+
AbstractFileBasedSpec,
|
7
|
+
DeliverRawFiles,
|
8
|
+
)
|
9
|
+
from airbyte_cdk.sources.specs.transfer_modes import DeliverPermissions
|
10
|
+
|
11
|
+
DELIVERY_TYPE_KEY = "delivery_type"
|
12
|
+
DELIVERY_TYPE_PERMISSION_TRANSFER_MODE_VALUE = "use_permissions_transfer"
|
13
|
+
DELIVERY_TYPE_FILES_TRANSFER_MODE_VALUE = "use_file_transfer"
|
14
|
+
PRESERVE_DIRECTORY_STRUCTURE_KEY = "preserve_directory_structure"
|
15
|
+
INCLUDE_IDENTITIES_STREAM_KEY = "include_identities_stream"
|
6
16
|
|
7
17
|
|
8
18
|
def use_file_transfer(parsed_config: AbstractFileBasedSpec) -> bool:
|
9
19
|
return (
|
10
|
-
hasattr(parsed_config.delivery_method,
|
11
|
-
and parsed_config.delivery_method.delivery_type ==
|
20
|
+
hasattr(parsed_config.delivery_method, DELIVERY_TYPE_KEY)
|
21
|
+
and parsed_config.delivery_method.delivery_type == DELIVERY_TYPE_FILES_TRANSFER_MODE_VALUE
|
12
22
|
)
|
13
23
|
|
14
24
|
|
@@ -27,8 +37,8 @@ def preserve_directory_structure(parsed_config: AbstractFileBasedSpec) -> bool:
|
|
27
37
|
"""
|
28
38
|
if (
|
29
39
|
use_file_transfer(parsed_config)
|
30
|
-
and hasattr(parsed_config.delivery_method,
|
31
|
-
and parsed_config.delivery_method
|
40
|
+
and hasattr(parsed_config.delivery_method, PRESERVE_DIRECTORY_STRUCTURE_KEY)
|
41
|
+
and isinstance(parsed_config.delivery_method, DeliverRawFiles)
|
32
42
|
):
|
33
43
|
return parsed_config.delivery_method.preserve_directory_structure
|
34
44
|
return True
|
@@ -36,16 +46,17 @@ def preserve_directory_structure(parsed_config: AbstractFileBasedSpec) -> bool:
|
|
36
46
|
|
37
47
|
def use_permissions_transfer(parsed_config: AbstractFileBasedSpec) -> bool:
|
38
48
|
return (
|
39
|
-
hasattr(parsed_config.delivery_method,
|
40
|
-
and parsed_config.delivery_method.delivery_type
|
49
|
+
hasattr(parsed_config.delivery_method, DELIVERY_TYPE_KEY)
|
50
|
+
and parsed_config.delivery_method.delivery_type
|
51
|
+
== DELIVERY_TYPE_PERMISSION_TRANSFER_MODE_VALUE
|
41
52
|
)
|
42
53
|
|
43
54
|
|
44
55
|
def include_identities_stream(parsed_config: AbstractFileBasedSpec) -> bool:
|
45
56
|
if (
|
46
57
|
use_permissions_transfer(parsed_config)
|
47
|
-
and hasattr(parsed_config.delivery_method,
|
48
|
-
and parsed_config.delivery_method
|
58
|
+
and hasattr(parsed_config.delivery_method, INCLUDE_IDENTITIES_STREAM_KEY)
|
59
|
+
and isinstance(parsed_config.delivery_method, DeliverPermissions)
|
49
60
|
):
|
50
61
|
return parsed_config.delivery_method.include_identities_stream
|
51
62
|
return False
|
@@ -58,7 +58,7 @@ from airbyte_cdk.sources.file_based.schema_validation_policies import (
|
|
58
58
|
from airbyte_cdk.sources.file_based.stream import (
|
59
59
|
AbstractFileBasedStream,
|
60
60
|
DefaultFileBasedStream,
|
61
|
-
|
61
|
+
FileIdentities,
|
62
62
|
)
|
63
63
|
from airbyte_cdk.sources.file_based.stream.concurrent.adapters import FileBasedStreamFacade
|
64
64
|
from airbyte_cdk.sources.file_based.stream.concurrent.cursor import (
|
@@ -67,7 +67,6 @@ from airbyte_cdk.sources.file_based.stream.concurrent.cursor import (
|
|
67
67
|
FileBasedFinalStateCursor,
|
68
68
|
)
|
69
69
|
from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor
|
70
|
-
from airbyte_cdk.sources.file_based.stream.identities_stream import IDENTITIES_STREAM_NAME
|
71
70
|
from airbyte_cdk.sources.message.repository import InMemoryMessageRepository, MessageRepository
|
72
71
|
from airbyte_cdk.sources.streams import Stream
|
73
72
|
from airbyte_cdk.sources.streams.concurrent.cursor import CursorField
|
@@ -169,7 +168,7 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
|
|
169
168
|
errors = []
|
170
169
|
tracebacks = []
|
171
170
|
for stream in streams:
|
172
|
-
if isinstance(stream,
|
171
|
+
if isinstance(stream, FileIdentities):
|
173
172
|
identity = next(iter(stream.load_identity_groups()))
|
174
173
|
if not identity:
|
175
174
|
errors.append(
|
@@ -341,8 +340,8 @@ class FileBasedSource(ConcurrentSourceAdapter, ABC):
|
|
341
340
|
def _make_identities_stream(
|
342
341
|
self,
|
343
342
|
) -> Stream:
|
344
|
-
return
|
345
|
-
catalog_schema=self.stream_schemas.get(IDENTITIES_STREAM_NAME),
|
343
|
+
return FileIdentities(
|
344
|
+
catalog_schema=self.stream_schemas.get(FileIdentities.IDENTITIES_STREAM_NAME),
|
346
345
|
stream_reader=self.stream_reader,
|
347
346
|
discovery_policy=self.discovery_policy,
|
348
347
|
errors_collector=self.errors_collector,
|
@@ -192,7 +192,7 @@ class AbstractFileBasedStreamReader(ABC):
|
|
192
192
|
ACL Permissions from files.
|
193
193
|
"""
|
194
194
|
raise NotImplementedError(
|
195
|
-
f"{self.__class__.__name__}
|
195
|
+
f"{self.__class__.__name__} does not implement get_file_acl_permissions(). To support ACL permissions, implement this method and update file_permissions_schema."
|
196
196
|
)
|
197
197
|
|
198
198
|
@abstractmethod
|
@@ -202,5 +202,27 @@ class AbstractFileBasedStreamReader(ABC):
|
|
202
202
|
identities.
|
203
203
|
"""
|
204
204
|
raise NotImplementedError(
|
205
|
-
f"{self.__class__.__name__}
|
205
|
+
f"{self.__class__.__name__} does not implement load_identity_groups(). To support identities, implement this method and update identities_schema."
|
206
|
+
)
|
207
|
+
|
208
|
+
@property
|
209
|
+
@abstractmethod
|
210
|
+
def file_permissions_schema(self) -> Dict[str, Any]:
|
211
|
+
"""
|
212
|
+
This is required for connectors that will support syncing
|
213
|
+
ACL Permissions from files.
|
214
|
+
"""
|
215
|
+
raise NotImplementedError(
|
216
|
+
f"{self.__class__.__name__} does not implement file_permissions_schema, please return json schema for your permissions streams."
|
217
|
+
)
|
218
|
+
|
219
|
+
@property
|
220
|
+
@abstractmethod
|
221
|
+
def identities_schema(self) -> Dict[str, Any]:
|
222
|
+
"""
|
223
|
+
This is required for connectors that will support syncing
|
224
|
+
identities.
|
225
|
+
"""
|
226
|
+
raise NotImplementedError(
|
227
|
+
f"{self.__class__.__name__} does not implement identities_schema, please return json schema for your identities stream."
|
206
228
|
)
|
@@ -23,31 +23,6 @@ file_transfer_schema = {
|
|
23
23
|
"properties": {"data": {"type": "object"}, "file": {"type": "object"}},
|
24
24
|
}
|
25
25
|
|
26
|
-
remote_file_permissions_schema = {
|
27
|
-
"type": "object",
|
28
|
-
"properties": {
|
29
|
-
"id": {"type": "string"},
|
30
|
-
"file_path": {"type": "string"},
|
31
|
-
"allowed_identity_remote_ids": {"type": "array", "items": {"type": "string"}},
|
32
|
-
"publicly_accessible": {"type": "boolean"},
|
33
|
-
},
|
34
|
-
}
|
35
|
-
|
36
|
-
remote_file_identity_schema = {
|
37
|
-
"type": "object",
|
38
|
-
"properties": {
|
39
|
-
"id": {"type": "string"},
|
40
|
-
"remote_id": {"type": "string"},
|
41
|
-
"parent_id": {"type": ["null", "string"]},
|
42
|
-
"name": {"type": ["null", "string"]},
|
43
|
-
"description": {"type": ["null", "string"]},
|
44
|
-
"email_address": {"type": ["null", "string"]},
|
45
|
-
"member_email_addresses": {"type": ["null", "array"]},
|
46
|
-
"type": {"type": "string"},
|
47
|
-
"modified_at": {"type": "string"},
|
48
|
-
},
|
49
|
-
}
|
50
|
-
|
51
26
|
|
52
27
|
@total_ordering
|
53
28
|
class ComparableType(Enum):
|
@@ -1,5 +1,5 @@
|
|
1
1
|
from airbyte_cdk.sources.file_based.stream.abstract_file_based_stream import AbstractFileBasedStream
|
2
2
|
from airbyte_cdk.sources.file_based.stream.default_file_based_stream import DefaultFileBasedStream
|
3
|
-
from airbyte_cdk.sources.file_based.stream.identities_stream import
|
3
|
+
from airbyte_cdk.sources.file_based.stream.identities_stream import FileIdentities
|
4
4
|
|
5
|
-
__all__ = ["AbstractFileBasedStream", "DefaultFileBasedStream", "
|
5
|
+
__all__ = ["AbstractFileBasedStream", "DefaultFileBasedStream", "FileIdentities"]
|
@@ -29,7 +29,6 @@ from airbyte_cdk.sources.file_based.schema_helpers import (
|
|
29
29
|
SchemaType,
|
30
30
|
file_transfer_schema,
|
31
31
|
merge_schemas,
|
32
|
-
remote_file_permissions_schema,
|
33
32
|
schemaless_schema,
|
34
33
|
)
|
35
34
|
from airbyte_cdk.sources.file_based.stream import AbstractFileBasedStream
|
@@ -111,7 +110,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
|
|
111
110
|
},
|
112
111
|
}
|
113
112
|
elif self.use_permissions_transfer:
|
114
|
-
return
|
113
|
+
return self.stream_reader.file_permissions_schema
|
115
114
|
else:
|
116
115
|
return super()._filter_schema_invalid_properties(configured_catalog_json_schema)
|
117
116
|
|
@@ -315,7 +314,7 @@ class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin):
|
|
315
314
|
if self.use_file_transfer:
|
316
315
|
return file_transfer_schema
|
317
316
|
elif self.use_permissions_transfer:
|
318
|
-
return
|
317
|
+
return self.stream_reader.file_permissions_schema
|
319
318
|
elif self.config.input_schema:
|
320
319
|
return self.config.get_input_schema() # type: ignore
|
321
320
|
elif self.config.schemaless:
|
@@ -2,30 +2,18 @@
|
|
2
2
|
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
3
3
|
#
|
4
4
|
|
5
|
-
import traceback
|
6
5
|
from functools import cache
|
7
|
-
from typing import Any, Dict, Iterable,
|
6
|
+
from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional
|
8
7
|
|
9
|
-
from airbyte_protocol_dataclasses.models import SyncMode
|
10
|
-
|
11
|
-
from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level
|
12
|
-
from airbyte_cdk.models import Type as MessageType
|
13
8
|
from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType
|
14
9
|
from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy
|
15
|
-
from airbyte_cdk.sources.file_based.exceptions import FileBasedErrorsCollector
|
10
|
+
from airbyte_cdk.sources.file_based.exceptions import FileBasedErrorsCollector
|
16
11
|
from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
|
17
|
-
from airbyte_cdk.sources.file_based.schema_helpers import remote_file_identity_schema
|
18
|
-
from airbyte_cdk.sources.file_based.types import StreamSlice
|
19
|
-
from airbyte_cdk.sources.streams import Stream
|
20
|
-
from airbyte_cdk.sources.streams.checkpoint import Cursor
|
21
12
|
from airbyte_cdk.sources.streams.core import JsonSchema
|
22
|
-
from airbyte_cdk.sources.
|
23
|
-
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
24
|
-
|
25
|
-
IDENTITIES_STREAM_NAME = "identities"
|
13
|
+
from airbyte_cdk.sources.streams.permissions.identities import Identities
|
26
14
|
|
27
15
|
|
28
|
-
class
|
16
|
+
class FileIdentities(Identities):
|
29
17
|
"""
|
30
18
|
The identities stream. A full refresh stream to sync identities from a certain domain.
|
31
19
|
The stream reader manage the logic to get such data, which is implemented on connector side.
|
@@ -39,7 +27,7 @@ class IdentitiesStream(Stream):
|
|
39
27
|
stream_reader: AbstractFileBasedStreamReader,
|
40
28
|
discovery_policy: AbstractDiscoveryPolicy,
|
41
29
|
errors_collector: FileBasedErrorsCollector,
|
42
|
-
):
|
30
|
+
) -> None:
|
43
31
|
super().__init__()
|
44
32
|
self.catalog_schema = catalog_schema
|
45
33
|
self.stream_reader = stream_reader
|
@@ -47,53 +35,13 @@ class IdentitiesStream(Stream):
|
|
47
35
|
self.errors_collector = errors_collector
|
48
36
|
self._cursor: MutableMapping[str, Any] = {}
|
49
37
|
|
50
|
-
@property
|
51
|
-
def state(self) -> MutableMapping[str, Any]:
|
52
|
-
return self._cursor
|
53
|
-
|
54
|
-
@state.setter
|
55
|
-
def state(self, value: MutableMapping[str, Any]) -> None:
|
56
|
-
"""State setter, accept state serialized by state getter."""
|
57
|
-
self._cursor = value
|
58
|
-
|
59
38
|
@property
|
60
39
|
def primary_key(self) -> PrimaryKeyType:
|
61
40
|
return None
|
62
41
|
|
63
|
-
def read_records(
|
64
|
-
self,
|
65
|
-
sync_mode: SyncMode,
|
66
|
-
cursor_field: Optional[List[str]] = None,
|
67
|
-
stream_slice: Optional[StreamSlice] = None,
|
68
|
-
stream_state: Optional[Mapping[str, Any]] = None,
|
69
|
-
) -> Iterable[Mapping[str, Any] | AirbyteMessage]:
|
70
|
-
try:
|
71
|
-
identity_groups = self.load_identity_groups()
|
72
|
-
for record in identity_groups:
|
73
|
-
yield stream_data_to_airbyte_message(self.name, record)
|
74
|
-
except AirbyteTracedException as exc:
|
75
|
-
# Re-raise the exception to stop the whole sync immediately as this is a fatal error
|
76
|
-
raise exc
|
77
|
-
except Exception:
|
78
|
-
yield AirbyteMessage(
|
79
|
-
type=MessageType.LOG,
|
80
|
-
log=AirbyteLogMessage(
|
81
|
-
level=Level.ERROR,
|
82
|
-
message=f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream={self.name}",
|
83
|
-
stack_trace=traceback.format_exc(),
|
84
|
-
),
|
85
|
-
)
|
86
|
-
|
87
42
|
def load_identity_groups(self) -> Iterable[Dict[str, Any]]:
|
88
43
|
return self.stream_reader.load_identity_groups(logger=self.logger)
|
89
44
|
|
90
45
|
@cache
|
91
46
|
def get_json_schema(self) -> JsonSchema:
|
92
|
-
return
|
93
|
-
|
94
|
-
@property
|
95
|
-
def name(self) -> str:
|
96
|
-
return IDENTITIES_STREAM_NAME
|
97
|
-
|
98
|
-
def get_cursor(self) -> Optional[Cursor]:
|
99
|
-
return None
|
47
|
+
return self.stream_reader.identities_schema
|
@@ -0,0 +1,26 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
|
5
|
+
from typing import Literal
|
6
|
+
|
7
|
+
from pydantic.v1 import AnyUrl, BaseModel, Field
|
8
|
+
|
9
|
+
from airbyte_cdk import OneOfOptionConfig
|
10
|
+
|
11
|
+
|
12
|
+
class DeliverPermissions(BaseModel):
|
13
|
+
class Config(OneOfOptionConfig):
|
14
|
+
title = "Replicate Permissions ACL"
|
15
|
+
description = "Sends one identity stream and one for more permissions (ACL) streams to the destination. This data can be used in downstream systems to recreate permission restrictions mirroring the original source."
|
16
|
+
discriminator = "delivery_type"
|
17
|
+
|
18
|
+
delivery_type: Literal["use_permissions_transfer"] = Field(
|
19
|
+
"use_permissions_transfer", const=True
|
20
|
+
)
|
21
|
+
|
22
|
+
include_identities_stream: bool = Field(
|
23
|
+
title="Include Identity Stream",
|
24
|
+
description="This data can be used in downstream systems to recreate permission restrictions mirroring the original source",
|
25
|
+
default=True,
|
26
|
+
)
|
@@ -0,0 +1,77 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
|
5
|
+
import traceback
|
6
|
+
from abc import ABC, abstractmethod
|
7
|
+
from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional
|
8
|
+
|
9
|
+
from airbyte_protocol_dataclasses.models import SyncMode
|
10
|
+
|
11
|
+
from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level
|
12
|
+
from airbyte_cdk.models import Type as MessageType
|
13
|
+
from airbyte_cdk.sources.streams import Stream
|
14
|
+
from airbyte_cdk.sources.streams.checkpoint import Cursor
|
15
|
+
from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message
|
16
|
+
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
|
17
|
+
|
18
|
+
DEFAULT_IDENTITIES_STREAM_NAME = "identities"
|
19
|
+
|
20
|
+
|
21
|
+
class Identities(Stream, ABC):
|
22
|
+
"""
|
23
|
+
The identities stream. A full refresh stream to sync identities from a certain domain.
|
24
|
+
The load_identity_groups method manage the logic to get such data.
|
25
|
+
"""
|
26
|
+
|
27
|
+
IDENTITIES_STREAM_NAME = DEFAULT_IDENTITIES_STREAM_NAME
|
28
|
+
|
29
|
+
is_resumable = False
|
30
|
+
|
31
|
+
def __init__(self) -> None:
|
32
|
+
super().__init__()
|
33
|
+
self._cursor: MutableMapping[str, Any] = {}
|
34
|
+
|
35
|
+
@property
|
36
|
+
def state(self) -> MutableMapping[str, Any]:
|
37
|
+
return self._cursor
|
38
|
+
|
39
|
+
@state.setter
|
40
|
+
def state(self, value: MutableMapping[str, Any]) -> None:
|
41
|
+
"""State setter, accept state serialized by state getter."""
|
42
|
+
self._cursor = value
|
43
|
+
|
44
|
+
def read_records(
|
45
|
+
self,
|
46
|
+
sync_mode: SyncMode,
|
47
|
+
cursor_field: Optional[List[str]] = None,
|
48
|
+
stream_slice: Optional[Mapping[str, Any]] = None,
|
49
|
+
stream_state: Optional[Mapping[str, Any]] = None,
|
50
|
+
) -> Iterable[Mapping[str, Any] | AirbyteMessage]:
|
51
|
+
try:
|
52
|
+
identity_groups = self.load_identity_groups()
|
53
|
+
for record in identity_groups:
|
54
|
+
yield stream_data_to_airbyte_message(self.name, record)
|
55
|
+
except AirbyteTracedException as exc:
|
56
|
+
# Re-raise the exception to stop the whole sync immediately as this is a fatal error
|
57
|
+
raise exc
|
58
|
+
except Exception as e:
|
59
|
+
yield AirbyteMessage(
|
60
|
+
type=MessageType.LOG,
|
61
|
+
log=AirbyteLogMessage(
|
62
|
+
level=Level.ERROR,
|
63
|
+
message=f"Error trying to read identities: {e} stream={self.name}",
|
64
|
+
stack_trace=traceback.format_exc(),
|
65
|
+
),
|
66
|
+
)
|
67
|
+
|
68
|
+
@abstractmethod
|
69
|
+
def load_identity_groups(self) -> Iterable[Dict[str, Any]]:
|
70
|
+
raise NotImplementedError("Implement this method to read identity records")
|
71
|
+
|
72
|
+
@property
|
73
|
+
def name(self) -> str:
|
74
|
+
return self.IDENTITIES_STREAM_NAME
|
75
|
+
|
76
|
+
def get_cursor(self) -> Optional[Cursor]:
|
77
|
+
return None
|
@@ -201,7 +201,7 @@ airbyte_cdk/sources/file_based/availability_strategy/__init__.py,sha256=ddKQfUmk
|
|
201
201
|
airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py,sha256=01Nd4b7ERAbp-OZo_8rrAzFXWPTMwr02SnWiN17nx8Q,2363
|
202
202
|
airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py,sha256=j9T5TimfWFUz7nqsaj-83G3xWmDpsmeSbDnaUNmz0UM,5849
|
203
203
|
airbyte_cdk/sources/file_based/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
204
|
-
airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py,sha256=
|
204
|
+
airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py,sha256=purspcxMNWLhGunaqVJa9TISMZK7vlQSwdz4zykFWNA,6989
|
205
205
|
airbyte_cdk/sources/file_based/config/avro_format.py,sha256=NxTF96ewzn6HuhgodsY7Rpb-ybr1ZEWW5d4Vid64g5A,716
|
206
206
|
airbyte_cdk/sources/file_based/config/csv_format.py,sha256=NWekkyT8dTwiVK0mwa_krQD4FJPHSDfILo8kPAg3-Vs,8006
|
207
207
|
airbyte_cdk/sources/file_based/config/excel_format.py,sha256=9qAmTsT6SoVzNfNv0oBVkVCmiyqQuVAbfRKajjoa7Js,378
|
@@ -209,13 +209,13 @@ airbyte_cdk/sources/file_based/config/file_based_stream_config.py,sha256=rkTuHpz
|
|
209
209
|
airbyte_cdk/sources/file_based/config/jsonl_format.py,sha256=cxtpz4t9_ERQyj_1Bx4DjOxuYLykWt0B02S4dWW5BgM,378
|
210
210
|
airbyte_cdk/sources/file_based/config/parquet_format.py,sha256=XOp-7nmm_WcbGI8SjKH2fs3Mkf1H4RAOYSWeUFYAz3w,741
|
211
211
|
airbyte_cdk/sources/file_based/config/unstructured_format.py,sha256=tIbB9Pn1HqU67ju7hEZ9dBstRrb2eojUNMsdckzbj58,3565
|
212
|
-
airbyte_cdk/sources/file_based/config/validate_config_transfer_modes.py,sha256=
|
212
|
+
airbyte_cdk/sources/file_based/config/validate_config_transfer_modes.py,sha256=pZDWOZj5USMsW0Y1Srq6FyO9PRzdtyeugZrK1Nhkxnw,2297
|
213
213
|
airbyte_cdk/sources/file_based/discovery_policy/__init__.py,sha256=gl3ey6mZbyfraB9P3pFhf9UJp2JeTZ1SUFAopy2iBvY,301
|
214
214
|
airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py,sha256=dCfXX529Rd5rtopg4VeEgTPJjFtqjtjzPq6LCw18Wt0,605
|
215
215
|
airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py,sha256=-xujTidtrq6HC00WKbjQh1CZdT5LMuzkp5BLjqDmfTY,1007
|
216
216
|
airbyte_cdk/sources/file_based/exceptions.py,sha256=WP0qkG6fpWoBpOyyicgp5YNE393VWyegq5qSy0v4QtM,7362
|
217
|
-
airbyte_cdk/sources/file_based/file_based_source.py,sha256=
|
218
|
-
airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=
|
217
|
+
airbyte_cdk/sources/file_based/file_based_source.py,sha256=zXZbLaQ-XW9D2l-k5N3466bOLYvwfRzCEDpih03NA2s,17603
|
218
|
+
airbyte_cdk/sources/file_based/file_based_stream_reader.py,sha256=z_DNWGiKUQCgYO9vsOy40aNbTsU_QbOuSHMCGegNLuw,8409
|
219
219
|
airbyte_cdk/sources/file_based/file_types/__init__.py,sha256=blCLn0-2LC-ZdgcNyDEhqM2RiUvEjEBh-G4-t32ZtuM,1268
|
220
220
|
airbyte_cdk/sources/file_based/file_types/avro_parser.py,sha256=XNx-JC-sgzH9u3nOJ2M59FxBXvtig8LN6BIkeDOavZA,10858
|
221
221
|
airbyte_cdk/sources/file_based/file_types/csv_parser.py,sha256=QlCXB-ry3np67Q_VerQEPoWDOTcPTB6Go4ydZxY9ae4,20445
|
@@ -226,11 +226,11 @@ airbyte_cdk/sources/file_based/file_types/jsonl_parser.py,sha256=GwyNyxmST4RX-Xp
|
|
226
226
|
airbyte_cdk/sources/file_based/file_types/parquet_parser.py,sha256=XenFg5sJ-UBnIkSmsiNJRou11NO0zZXx-RXgPHMT2NA,10487
|
227
227
|
airbyte_cdk/sources/file_based/file_types/unstructured_parser.py,sha256=2TYOQl62FQPCa8otLbkDIk_j01EP3oWaKSfXGhCjCHg,19492
|
228
228
|
airbyte_cdk/sources/file_based/remote_file.py,sha256=yqRz93vPe8PBXLIMJ5W5u2JRlZRhg6sBrAjn3pPjJ8A,315
|
229
|
-
airbyte_cdk/sources/file_based/schema_helpers.py,sha256=
|
229
|
+
airbyte_cdk/sources/file_based/schema_helpers.py,sha256=Cf8FH1bDFP0qCDDfEYir_WjP4exXUnikz8hZ40y1Ek0,9601
|
230
230
|
airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py,sha256=FkByIyEy56x2_awYnxGPqGaOp7zAzpAoRkPZHKySI9M,536
|
231
231
|
airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py,sha256=kjvX7nOmUALYd7HuZHilUzgJPZ-MnZ08mtvuBnt2tQ0,618
|
232
232
|
airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py,sha256=vjTlmYT_nqzY3DbT5xem7X-bwgA9RyXHoKFqiMO2URk,1728
|
233
|
-
airbyte_cdk/sources/file_based/stream/__init__.py,sha256=
|
233
|
+
airbyte_cdk/sources/file_based/stream/__init__.py,sha256=gATNUUwhzUXBxi029_GRxBt7n7CFNtdK959Jlu2mUW4,366
|
234
234
|
airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py,sha256=9pQh3BHYcxm8CRC8XawfmBxL8O9HggpWwCCbX_ncINE,7509
|
235
235
|
airbyte_cdk/sources/file_based/stream/concurrent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
236
236
|
airbyte_cdk/sources/file_based/stream/concurrent/adapters.py,sha256=WZ5q2uovgohauJgwfxq_LFeZ92WMZd0LoH6c5QQURPo,13931
|
@@ -241,14 +241,15 @@ airbyte_cdk/sources/file_based/stream/concurrent/cursor/file_based_final_state_c
|
|
241
241
|
airbyte_cdk/sources/file_based/stream/cursor/__init__.py,sha256=MhFB5hOo8sjwvCh8gangaymdg3EJWYt_72brFOZt068,191
|
242
242
|
airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py,sha256=om-x3gZFPgWDpi15S9RxZmR36VHnk8sytgN6LlBQhAw,1934
|
243
243
|
airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py,sha256=VGV7xLyBribuBMVrXtO1xqkWJD86bl7yhXtjnwLMohM,7051
|
244
|
-
airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=
|
245
|
-
airbyte_cdk/sources/file_based/stream/identities_stream.py,sha256=
|
244
|
+
airbyte_cdk/sources/file_based/stream/default_file_based_stream.py,sha256=hqAbm61DZzXn39BwJReiT5l8Kcd6MkOTOTXzW-4j624,19728
|
245
|
+
airbyte_cdk/sources/file_based/stream/identities_stream.py,sha256=KuhIVV0NFc6RhtzrAvswYvAO_dgPSYkWgOfqr4NAgQw,1765
|
246
246
|
airbyte_cdk/sources/file_based/types.py,sha256=INxG7OPnkdUP69oYNKMAbwhvV1AGvLRHs1J6pIia2FI,218
|
247
247
|
airbyte_cdk/sources/http_config.py,sha256=OBZeuyFilm6NlDlBhFQvHhTWabEvZww6OHDIlZujIS0,730
|
248
248
|
airbyte_cdk/sources/http_logger.py,sha256=l_1fk5YwdonZ1wvAsTwjj6d36fj2WrVraIAMj5jTQdM,1575
|
249
249
|
airbyte_cdk/sources/message/__init__.py,sha256=y98fzHsQBwXwp2zEa4K5mxGFqjnx9lDn9O0pTk-VS4U,395
|
250
250
|
airbyte_cdk/sources/message/repository.py,sha256=SG7avgti_-dj8FcRHTTrhgLLGJbElv14_zIB0SH8AIc,4763
|
251
251
|
airbyte_cdk/sources/source.py,sha256=KIBBH5VLEb8BZ8B9aROlfaI6OLoJqKDPMJ10jkAR7nk,3611
|
252
|
+
airbyte_cdk/sources/specs/transfer_modes.py,sha256=sfSVO0yT6SaGKN5_TP0Nl_ftG0yPhecaBv0WkhAEXA8,932
|
252
253
|
airbyte_cdk/sources/streams/__init__.py,sha256=8fzTKpRTnSx5PggXgQPKJzHNZUV2BCA40N-dI6JM1xI,256
|
253
254
|
airbyte_cdk/sources/streams/availability_strategy.py,sha256=_RU4JITrxMEN36g1RDHMu0iSw0I_3yWGfo5N8_YRvOg,3247
|
254
255
|
airbyte_cdk/sources/streams/call_rate.py,sha256=Um_Ny8R7WZ2B0PWoxr-wrWPsgc5we7HrHalaMcozuVs,21052
|
@@ -301,6 +302,7 @@ airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py,sha256=n
|
|
301
302
|
airbyte_cdk/sources/streams/http/requests_native_auth/abstract_token.py,sha256=Y3n7J-sk5yGjv_OxtY6Z6k0PEsFZmtIRi-x0KCbaHdA,1010
|
302
303
|
airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py,sha256=C2j2uVfi9d-3KgHO3NGxIiFdfASjHOtsd6g_LWPYOAs,20311
|
303
304
|
airbyte_cdk/sources/streams/http/requests_native_auth/token.py,sha256=h5PTzcdH-RQLeCg7xZ45w_484OPUDSwNWl_iMJQmZoI,2526
|
305
|
+
airbyte_cdk/sources/streams/permissions/identities.py,sha256=q1TeQbgiI04dVAAw1A9dlVKJRbQvBheZx95rFBVd140,2680
|
304
306
|
airbyte_cdk/sources/streams/utils/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
|
305
307
|
airbyte_cdk/sources/types.py,sha256=aFPGI4t2K1vHz2oFSUIYUyDN7kw-vcYq4D7aD2zgfAU,5128
|
306
308
|
airbyte_cdk/sources/utils/__init__.py,sha256=TTN6VUxVy6Is8BhYQZR5pxJGQh8yH4duXh4O1TiMiEY,118
|
@@ -353,9 +355,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
|
|
353
355
|
airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
|
354
356
|
airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
|
355
357
|
airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
|
356
|
-
airbyte_cdk-6.26.0.
|
357
|
-
airbyte_cdk-6.26.0.
|
358
|
-
airbyte_cdk-6.26.0.
|
359
|
-
airbyte_cdk-6.26.0.
|
360
|
-
airbyte_cdk-6.26.0.
|
361
|
-
airbyte_cdk-6.26.0.
|
358
|
+
airbyte_cdk-6.26.0.dev4109.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
|
359
|
+
airbyte_cdk-6.26.0.dev4109.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
|
360
|
+
airbyte_cdk-6.26.0.dev4109.dist-info/METADATA,sha256=OY2eDa-8oZYg5jjBx3kWiDnDXAdqkWkLEsUryerMYpA,6018
|
361
|
+
airbyte_cdk-6.26.0.dev4109.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
362
|
+
airbyte_cdk-6.26.0.dev4109.dist-info/entry_points.txt,sha256=fj-e3PAQvsxsQzyyq8UkG1k8spunWnD4BAH2AwlR6NM,95
|
363
|
+
airbyte_cdk-6.26.0.dev4109.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
{airbyte_cdk-6.26.0.dev4108.dist-info → airbyte_cdk-6.26.0.dev4109.dist-info}/entry_points.txt
RENAMED
File without changes
|