airbyte-source-gitlab 3.0.0.dev202403072311__py3-none-any.whl → 4.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: airbyte-source-gitlab
3
- Version: 3.0.0.dev202403072311
3
+ Version: 4.0.1
4
4
  Summary: Source implementation for GitLab.
5
5
  Home-page: https://airbyte.com
6
6
  License: MIT
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3
12
12
  Classifier: Programming Language :: Python :: 3.9
13
13
  Classifier: Programming Language :: Python :: 3.10
14
14
  Classifier: Programming Language :: Python :: 3.11
15
- Requires-Dist: airbyte-cdk (==0.58.8)
15
+ Requires-Dist: airbyte-cdk (>=0,<1)
16
16
  Requires-Dist: vcrpy (==4.1.1)
17
17
  Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/gitlab
18
18
  Project-URL: Repository, https://github.com/airbytehq/airbyte
@@ -50,7 +50,7 @@ See `sample_files/sample_config.json` for a sample config file.
50
50
  poetry run source-gitlab spec
51
51
  poetry run source-gitlab check --config secrets/config.json
52
52
  poetry run source-gitlab discover --config secrets/config.json
53
- poetry run source-gitlab read --config secrets/config.json --catalog sample_files/configured_catalog.json
53
+ poetry run source-gitlab read --config secrets/config.json --catalog integration_tests/configured_catalog.json
54
54
  ```
55
55
 
56
56
  ### Running unit tests
@@ -1,5 +1,7 @@
1
1
  source_gitlab/__init__.py,sha256=Bip0P3F1yGxbb0nNwdTE8i98KtkuTT_MzZbMeSxziBM,1134
2
+ source_gitlab/components/partition_routers.py,sha256=tu8gl2FoBCic6xohSbZr1YTdOAEyczQyZqMSBomhEG8,2054
2
3
  source_gitlab/config_migrations.py,sha256=K6Bg8zh4Zz1U9QlilRyAC1oy8RQJOOsVhh62jVIPTrY,3816
4
+ source_gitlab/manifest.yaml,sha256=n0a-CfKB9OmsKD9fCYYGW8S8tt5nfg2VfebMPQbw9fM,20615
3
5
  source_gitlab/run.py,sha256=opjAlKxINNpsekbyOYAAxh25DxI7gTKg5ohJmDwe0q0,403
4
6
  source_gitlab/schemas/branches.json,sha256=iQMHidNwdx2g4UgfkeOZPhpG67K--S2jPM-XLBPhJb8,800
5
7
  source_gitlab/schemas/commits.json,sha256=2VgI6Ih5lrzzeuIacVNqigM8RwR93u2cEwD21LmEpvU,1676
@@ -24,11 +26,10 @@ source_gitlab/schemas/projects.json,sha256=V9Plxh9SB-zPxH8wOTSgOhHm4BPb4XS87RqUY
24
26
  source_gitlab/schemas/releases.json,sha256=NfDIhLN2DBZhXjhn_5Xbw8PUO2e5KRgeiU8HgjsSoXU,1781
25
27
  source_gitlab/schemas/tags.json,sha256=uyRFTJF2d9z2tbIp69NhkJmrRMmaEuHEdSagP99iqjw,771
26
28
  source_gitlab/schemas/users.json,sha256=h89E6ZHQWbwU_MhErTZTtfiIZjF6aLAbHZQJ8TCTLY8,496
27
- source_gitlab/source.py,sha256=_ovomkDxlb62M4PqNczTRk29grlNEsx9Scoy-79eebM,12072
29
+ source_gitlab/source.py,sha256=Gc17C7D_cazqOdnVEIUNwRYYucDzNXrsGXqXnZy8CN8,1093
28
30
  source_gitlab/spec.json,sha256=0eit0zD-6kdcaNhJ1sAf0ABgIUwnlPV87Sbu3A4JvG0,6132
29
- source_gitlab/streams.py,sha256=2vjqwHYECUbTj9GQq2WpSK2m9Dy3o_KE0xs6bbRESNw,16122
30
31
  source_gitlab/utils.py,sha256=idtKZiT-ILRRtpO0rxPJ1wim-mWN6ewzN_KgLCmogBs,456
31
- airbyte_source_gitlab-3.0.0.dev202403072311.dist-info/METADATA,sha256=_ubb4Z-8sAaZ3WkQCWLGOgRqfu3NY7eLfqcJxtqmYsk,5247
32
- airbyte_source_gitlab-3.0.0.dev202403072311.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
33
- airbyte_source_gitlab-3.0.0.dev202403072311.dist-info/entry_points.txt,sha256=sbp4881P3oluHtnFDy4ZCVHDfL5fIkVQhJ7vka_qzTc,55
34
- airbyte_source_gitlab-3.0.0.dev202403072311.dist-info/RECORD,,
32
+ airbyte_source_gitlab-4.0.1.dist-info/METADATA,sha256=HYXjnOyqR16tnvzGxD1Wpi6qtNc87480IJ_4QjosROc,5234
33
+ airbyte_source_gitlab-4.0.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
34
+ airbyte_source_gitlab-4.0.1.dist-info/entry_points.txt,sha256=sbp4881P3oluHtnFDy4ZCVHDfL5fIkVQhJ7vka_qzTc,55
35
+ airbyte_source_gitlab-4.0.1.dist-info/RECORD,,
@@ -0,0 +1,43 @@
1
+ #
2
+ # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+
6
+ from dataclasses import dataclass
7
+ from typing import Iterable
8
+
9
+ from airbyte_cdk.models import SyncMode
10
+ from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import SubstreamPartitionRouter
11
+ from airbyte_cdk.sources.declarative.types import StreamSlice
12
+
13
+
14
+ @dataclass
15
+ class GroupStreamsPartitionRouter(SubstreamPartitionRouter):
16
+ def stream_slices(self) -> Iterable[StreamSlice]:
17
+ parent_streams = {stream.stream.name: stream.stream for stream in self.parent_stream_configs}
18
+ groups_list = self.config.get("groups_list")
19
+ selected_parent = parent_streams["include_descendant_groups"] if groups_list else parent_streams["groups_list"]
20
+
21
+ for stream_slice in selected_parent.stream_slices(sync_mode=SyncMode.full_refresh):
22
+ for record in selected_parent.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice):
23
+ yield StreamSlice(partition={"id": record["id"]}, cursor_slice={})
24
+
25
+
26
+ @dataclass
27
+ class ProjectStreamsPartitionRouter(SubstreamPartitionRouter):
28
+ def stream_slices(self) -> Iterable[StreamSlice]:
29
+ parent_stream = self.parent_stream_configs[0].stream
30
+ projects_list = self.config.get("projects_list", [])
31
+
32
+ group_project_ids = []
33
+ for stream_slice in parent_stream.stream_slices(sync_mode=SyncMode.full_refresh):
34
+ for record in parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice):
35
+ group_project_ids.extend([i["path_with_namespace"] for i in record["projects"]])
36
+
37
+ if group_project_ids:
38
+ for project_id in group_project_ids:
39
+ if not projects_list or projects_list and project_id in projects_list:
40
+ yield StreamSlice(partition={"id": project_id.replace("/", "%2F")}, cursor_slice={})
41
+ else:
42
+ for project_id in projects_list:
43
+ yield StreamSlice(partition={"id": project_id.replace("/", "%2F")}, cursor_slice={})