airbyte-source-gitlab 3.0.0.dev202403072311__py3-none-any.whl → 4.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {airbyte_source_gitlab-3.0.0.dev202403072311.dist-info → airbyte_source_gitlab-4.0.1.dist-info}/METADATA +3 -3
- {airbyte_source_gitlab-3.0.0.dev202403072311.dist-info → airbyte_source_gitlab-4.0.1.dist-info}/RECORD +7 -6
- source_gitlab/components/partition_routers.py +43 -0
- source_gitlab/manifest.yaml +613 -0
- source_gitlab/source.py +9 -224
- source_gitlab/streams.py +0 -431
- {airbyte_source_gitlab-3.0.0.dev202403072311.dist-info → airbyte_source_gitlab-4.0.1.dist-info}/WHEEL +0 -0
- {airbyte_source_gitlab-3.0.0.dev202403072311.dist-info → airbyte_source_gitlab-4.0.1.dist-info}/entry_points.txt +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: airbyte-source-gitlab
|
3
|
-
Version:
|
3
|
+
Version: 4.0.1
|
4
4
|
Summary: Source implementation for GitLab.
|
5
5
|
Home-page: https://airbyte.com
|
6
6
|
License: MIT
|
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.9
|
13
13
|
Classifier: Programming Language :: Python :: 3.10
|
14
14
|
Classifier: Programming Language :: Python :: 3.11
|
15
|
-
Requires-Dist: airbyte-cdk (
|
15
|
+
Requires-Dist: airbyte-cdk (>=0,<1)
|
16
16
|
Requires-Dist: vcrpy (==4.1.1)
|
17
17
|
Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/gitlab
|
18
18
|
Project-URL: Repository, https://github.com/airbytehq/airbyte
|
@@ -50,7 +50,7 @@ See `sample_files/sample_config.json` for a sample config file.
|
|
50
50
|
poetry run source-gitlab spec
|
51
51
|
poetry run source-gitlab check --config secrets/config.json
|
52
52
|
poetry run source-gitlab discover --config secrets/config.json
|
53
|
-
poetry run source-gitlab read --config secrets/config.json --catalog
|
53
|
+
poetry run source-gitlab read --config secrets/config.json --catalog integration_tests/configured_catalog.json
|
54
54
|
```
|
55
55
|
|
56
56
|
### Running unit tests
|
@@ -1,5 +1,7 @@
|
|
1
1
|
source_gitlab/__init__.py,sha256=Bip0P3F1yGxbb0nNwdTE8i98KtkuTT_MzZbMeSxziBM,1134
|
2
|
+
source_gitlab/components/partition_routers.py,sha256=tu8gl2FoBCic6xohSbZr1YTdOAEyczQyZqMSBomhEG8,2054
|
2
3
|
source_gitlab/config_migrations.py,sha256=K6Bg8zh4Zz1U9QlilRyAC1oy8RQJOOsVhh62jVIPTrY,3816
|
4
|
+
source_gitlab/manifest.yaml,sha256=n0a-CfKB9OmsKD9fCYYGW8S8tt5nfg2VfebMPQbw9fM,20615
|
3
5
|
source_gitlab/run.py,sha256=opjAlKxINNpsekbyOYAAxh25DxI7gTKg5ohJmDwe0q0,403
|
4
6
|
source_gitlab/schemas/branches.json,sha256=iQMHidNwdx2g4UgfkeOZPhpG67K--S2jPM-XLBPhJb8,800
|
5
7
|
source_gitlab/schemas/commits.json,sha256=2VgI6Ih5lrzzeuIacVNqigM8RwR93u2cEwD21LmEpvU,1676
|
@@ -24,11 +26,10 @@ source_gitlab/schemas/projects.json,sha256=V9Plxh9SB-zPxH8wOTSgOhHm4BPb4XS87RqUY
|
|
24
26
|
source_gitlab/schemas/releases.json,sha256=NfDIhLN2DBZhXjhn_5Xbw8PUO2e5KRgeiU8HgjsSoXU,1781
|
25
27
|
source_gitlab/schemas/tags.json,sha256=uyRFTJF2d9z2tbIp69NhkJmrRMmaEuHEdSagP99iqjw,771
|
26
28
|
source_gitlab/schemas/users.json,sha256=h89E6ZHQWbwU_MhErTZTtfiIZjF6aLAbHZQJ8TCTLY8,496
|
27
|
-
source_gitlab/source.py,sha256=
|
29
|
+
source_gitlab/source.py,sha256=Gc17C7D_cazqOdnVEIUNwRYYucDzNXrsGXqXnZy8CN8,1093
|
28
30
|
source_gitlab/spec.json,sha256=0eit0zD-6kdcaNhJ1sAf0ABgIUwnlPV87Sbu3A4JvG0,6132
|
29
|
-
source_gitlab/streams.py,sha256=2vjqwHYECUbTj9GQq2WpSK2m9Dy3o_KE0xs6bbRESNw,16122
|
30
31
|
source_gitlab/utils.py,sha256=idtKZiT-ILRRtpO0rxPJ1wim-mWN6ewzN_KgLCmogBs,456
|
31
|
-
airbyte_source_gitlab-
|
32
|
-
airbyte_source_gitlab-
|
33
|
-
airbyte_source_gitlab-
|
34
|
-
airbyte_source_gitlab-
|
32
|
+
airbyte_source_gitlab-4.0.1.dist-info/METADATA,sha256=HYXjnOyqR16tnvzGxD1Wpi6qtNc87480IJ_4QjosROc,5234
|
33
|
+
airbyte_source_gitlab-4.0.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
34
|
+
airbyte_source_gitlab-4.0.1.dist-info/entry_points.txt,sha256=sbp4881P3oluHtnFDy4ZCVHDfL5fIkVQhJ7vka_qzTc,55
|
35
|
+
airbyte_source_gitlab-4.0.1.dist-info/RECORD,,
|
@@ -0,0 +1,43 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
|
3
|
+
#
|
4
|
+
|
5
|
+
|
6
|
+
from dataclasses import dataclass
|
7
|
+
from typing import Iterable
|
8
|
+
|
9
|
+
from airbyte_cdk.models import SyncMode
|
10
|
+
from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import SubstreamPartitionRouter
|
11
|
+
from airbyte_cdk.sources.declarative.types import StreamSlice
|
12
|
+
|
13
|
+
|
14
|
+
@dataclass
|
15
|
+
class GroupStreamsPartitionRouter(SubstreamPartitionRouter):
|
16
|
+
def stream_slices(self) -> Iterable[StreamSlice]:
|
17
|
+
parent_streams = {stream.stream.name: stream.stream for stream in self.parent_stream_configs}
|
18
|
+
groups_list = self.config.get("groups_list")
|
19
|
+
selected_parent = parent_streams["include_descendant_groups"] if groups_list else parent_streams["groups_list"]
|
20
|
+
|
21
|
+
for stream_slice in selected_parent.stream_slices(sync_mode=SyncMode.full_refresh):
|
22
|
+
for record in selected_parent.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice):
|
23
|
+
yield StreamSlice(partition={"id": record["id"]}, cursor_slice={})
|
24
|
+
|
25
|
+
|
26
|
+
@dataclass
|
27
|
+
class ProjectStreamsPartitionRouter(SubstreamPartitionRouter):
|
28
|
+
def stream_slices(self) -> Iterable[StreamSlice]:
|
29
|
+
parent_stream = self.parent_stream_configs[0].stream
|
30
|
+
projects_list = self.config.get("projects_list", [])
|
31
|
+
|
32
|
+
group_project_ids = []
|
33
|
+
for stream_slice in parent_stream.stream_slices(sync_mode=SyncMode.full_refresh):
|
34
|
+
for record in parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice):
|
35
|
+
group_project_ids.extend([i["path_with_namespace"] for i in record["projects"]])
|
36
|
+
|
37
|
+
if group_project_ids:
|
38
|
+
for project_id in group_project_ids:
|
39
|
+
if not projects_list or projects_list and project_id in projects_list:
|
40
|
+
yield StreamSlice(partition={"id": project_id.replace("/", "%2F")}, cursor_slice={})
|
41
|
+
else:
|
42
|
+
for project_id in projects_list:
|
43
|
+
yield StreamSlice(partition={"id": project_id.replace("/", "%2F")}, cursor_slice={})
|