airbyte-source-gitlab 3.0.0.dev202403062351__py3-none-any.whl → 4.0.0__py3-none-any.whl

{airbyte_source_gitlab-3.0.0.dev202403062351.dist-info → airbyte_source_gitlab-4.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-gitlab
-Version: 3.0.0.dev202403062351
+Version: 4.0.0
 Summary: Source implementation for GitLab.
 Home-page: https://airbyte.com
 License: MIT
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk (==0.58.8)
+Requires-Dist: airbyte-cdk (>=0,<1)
 Requires-Dist: vcrpy (==4.1.1)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/gitlab
 Project-URL: Repository, https://github.com/airbytehq/airbyte
@@ -50,7 +50,7 @@ See `sample_files/sample_config.json` for a sample config file.
 poetry run source-gitlab spec
 poetry run source-gitlab check --config secrets/config.json
 poetry run source-gitlab discover --config secrets/config.json
-poetry run source-gitlab read --config secrets/config.json --catalog sample_files/configured_catalog.json
+poetry run source-gitlab read --config secrets/config.json --catalog integration_tests/configured_catalog.json
 ```
 
 ### Running unit tests

{airbyte_source_gitlab-3.0.0.dev202403062351.dist-info → airbyte_source_gitlab-4.0.0.dist-info}/RECORD

@@ -1,5 +1,7 @@
 source_gitlab/__init__.py,sha256=Bip0P3F1yGxbb0nNwdTE8i98KtkuTT_MzZbMeSxziBM,1134
+source_gitlab/components/partition_routers.py,sha256=tu8gl2FoBCic6xohSbZr1YTdOAEyczQyZqMSBomhEG8,2054
 source_gitlab/config_migrations.py,sha256=K6Bg8zh4Zz1U9QlilRyAC1oy8RQJOOsVhh62jVIPTrY,3816
+source_gitlab/manifest.yaml,sha256=GztDIgezKHPryerxFotu6U1DZDpertlk6yOLsW6kOmM,20701
 source_gitlab/run.py,sha256=opjAlKxINNpsekbyOYAAxh25DxI7gTKg5ohJmDwe0q0,403
 source_gitlab/schemas/branches.json,sha256=iQMHidNwdx2g4UgfkeOZPhpG67K--S2jPM-XLBPhJb8,800
 source_gitlab/schemas/commits.json,sha256=2VgI6Ih5lrzzeuIacVNqigM8RwR93u2cEwD21LmEpvU,1676
@@ -24,11 +26,10 @@ source_gitlab/schemas/projects.json,sha256=V9Plxh9SB-zPxH8wOTSgOhHm4BPb4XS87RqUY
 source_gitlab/schemas/releases.json,sha256=NfDIhLN2DBZhXjhn_5Xbw8PUO2e5KRgeiU8HgjsSoXU,1781
 source_gitlab/schemas/tags.json,sha256=uyRFTJF2d9z2tbIp69NhkJmrRMmaEuHEdSagP99iqjw,771
 source_gitlab/schemas/users.json,sha256=h89E6ZHQWbwU_MhErTZTtfiIZjF6aLAbHZQJ8TCTLY8,496
-source_gitlab/source.py,sha256=_ovomkDxlb62M4PqNczTRk29grlNEsx9Scoy-79eebM,12072
+source_gitlab/source.py,sha256=Gc17C7D_cazqOdnVEIUNwRYYucDzNXrsGXqXnZy8CN8,1093
 source_gitlab/spec.json,sha256=0eit0zD-6kdcaNhJ1sAf0ABgIUwnlPV87Sbu3A4JvG0,6132
-source_gitlab/streams.py,sha256=2vjqwHYECUbTj9GQq2WpSK2m9Dy3o_KE0xs6bbRESNw,16122
 source_gitlab/utils.py,sha256=idtKZiT-ILRRtpO0rxPJ1wim-mWN6ewzN_KgLCmogBs,456
-airbyte_source_gitlab-3.0.0.dev202403062351.dist-info/METADATA,sha256=zdyl5g3Hwsgd45khiw3oF-bycTXMFNEGpaMmLg84Jfk,5247
-airbyte_source_gitlab-3.0.0.dev202403062351.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-airbyte_source_gitlab-3.0.0.dev202403062351.dist-info/entry_points.txt,sha256=sbp4881P3oluHtnFDy4ZCVHDfL5fIkVQhJ7vka_qzTc,55
-airbyte_source_gitlab-3.0.0.dev202403062351.dist-info/RECORD,,
+airbyte_source_gitlab-4.0.0.dist-info/METADATA,sha256=asNgVphsQaU72h5JTog65drshNXIH166yimWLEhRhZE,5234
+airbyte_source_gitlab-4.0.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+airbyte_source_gitlab-4.0.0.dist-info/entry_points.txt,sha256=sbp4881P3oluHtnFDy4ZCVHDfL5fIkVQhJ7vka_qzTc,55
+airbyte_source_gitlab-4.0.0.dist-info/RECORD,,

source_gitlab/components/partition_routers.py

@@ -0,0 +1,43 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+
+from dataclasses import dataclass
+from typing import Iterable
+
+from airbyte_cdk.models import SyncMode
+from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import SubstreamPartitionRouter
+from airbyte_cdk.sources.declarative.types import StreamSlice
+
+
+@dataclass
+class GroupStreamsPartitionRouter(SubstreamPartitionRouter):
+    def stream_slices(self) -> Iterable[StreamSlice]:
+        parent_streams = {stream.stream.name: stream.stream for stream in self.parent_stream_configs}
+        groups_list = self.config.get("groups_list")
+        selected_parent = parent_streams["include_descendant_groups"] if groups_list else parent_streams["groups_list"]
+
+        for stream_slice in selected_parent.stream_slices(sync_mode=SyncMode.full_refresh):
+            for record in selected_parent.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice):
+                yield StreamSlice(partition={"id": record["id"]}, cursor_slice={})
+
+
+@dataclass
+class ProjectStreamsPartitionRouter(SubstreamPartitionRouter):
+    def stream_slices(self) -> Iterable[StreamSlice]:
+        parent_stream = self.parent_stream_configs[0].stream
+        projects_list = self.config.get("projects_list", [])
+
+        group_project_ids = []
+        for stream_slice in parent_stream.stream_slices(sync_mode=SyncMode.full_refresh):
+            for record in parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice):
+                group_project_ids.extend([i["path_with_namespace"] for i in record["projects"]])
+
+        if group_project_ids:
+            for project_id in group_project_ids:
+                if not projects_list or projects_list and project_id in projects_list:
+                    yield StreamSlice(partition={"id": project_id.replace("/", "%2F")}, cursor_slice={})
+        else:
+            for project_id in projects_list:
+                yield StreamSlice(partition={"id": project_id.replace("/", "%2F")}, cursor_slice={})