airbyte-source-gitlab 3.0.0.dev202403072311__py3-none-any.whl → 4.0.1__py3-none-any.whl

source_gitlab/source.py

@@ -1,245 +1,30 @@
 #
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 #
 
 
-import os
-from typing import Any, List, Mapping, MutableMapping, Optional, Tuple, Union
+from typing import Any, MutableMapping, Tuple
 
-import pendulum
-from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message
-from airbyte_cdk.models import SyncMode
-from airbyte_cdk.sources import AbstractSource
-from airbyte_cdk.sources.streams import Stream
-from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import SingleUseRefreshTokenOauth2Authenticator
-from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator
-from airbyte_cdk.utils import AirbyteTracedException
-from requests.auth import AuthBase
-from requests.exceptions import HTTPError
+from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
+from airbyte_cdk.utils import is_cloud_environment
 
-from .streams import (
-    Branches,
-    Commits,
-    Deployments,
-    EpicIssues,
-    Epics,
-    GitlabStream,
-    GroupIssueBoards,
-    GroupLabels,
-    GroupMembers,
-    GroupMilestones,
-    GroupProjects,
-    Groups,
-    GroupsList,
-    IncludeDescendantGroups,
-    Issues,
-    Jobs,
-    MergeRequestCommits,
-    MergeRequests,
-    Pipelines,
-    PipelinesExtended,
-    ProjectLabels,
-    ProjectMembers,
-    ProjectMilestones,
-    Projects,
-    Releases,
-    Tags,
-    Users,
-)
 from .utils import parse_url
 
 
-class SingleUseRefreshTokenGitlabOAuth2Authenticator(SingleUseRefreshTokenOauth2Authenticator):
-    def __init__(self, *args, created_at_name: str = "created_at", **kwargs):
-        super().__init__(*args, **kwargs)
-        self._created_at_name = created_at_name
-
-    def get_created_at_name(self) -> str:
-        return self._created_at_name
-
-    def get_access_token(self) -> str:
-        if self.token_has_expired():
-            new_access_token, access_token_expires_in, access_token_created_at, new_refresh_token = self.refresh_access_token()
-            new_token_expiry_date = self.get_new_token_expiry_date(access_token_expires_in, access_token_created_at)
-            self.access_token = new_access_token
-            self.set_refresh_token(new_refresh_token)
-            self.set_token_expiry_date(new_token_expiry_date)
-            emit_configuration_as_airbyte_control_message(self._connector_config)
-        return self.access_token
-
-    @staticmethod
-    def get_new_token_expiry_date(access_token_expires_in: int, access_token_created_at: int) -> pendulum.DateTime:
-        return pendulum.from_timestamp(access_token_created_at + access_token_expires_in)
-
-    def refresh_access_token(self) -> Tuple[str, int, int, str]:
-        response_json = self._get_refresh_access_token_response()
-        return (
-            response_json[self.get_access_token_name()],
-            response_json[self.get_expires_in_name()],
-            response_json[self.get_created_at_name()],
-            response_json[self.get_refresh_token_name()],
-        )
-
-
-def get_authenticator(config: MutableMapping) -> AuthBase:
-    if config["credentials"]["auth_type"] == "access_token":
-        return TokenAuthenticator(token=config["credentials"]["access_token"])
-    return SingleUseRefreshTokenGitlabOAuth2Authenticator(
-        config,
-        token_refresh_endpoint=f"https://{config['api_url']}/oauth/token",
-        refresh_token_error_status_codes=(400,),
-        refresh_token_error_key="error",
-        refresh_token_error_values="invalid_grant",
-    )
-
-
-class SourceGitlab(AbstractSource):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.__auth_params: Mapping[str, Any] = {}
-        self.__groups_stream: Optional[GitlabStream] = None
-        self.__projects_stream: Optional[GitlabStream] = None
+class SourceGitlab(YamlDeclarativeSource):
+    def __init__(self):
+        super().__init__(**{"path_to_yaml": "manifest.yaml"})
 
     @staticmethod
     def _ensure_default_values(config: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
         config["api_url"] = config.get("api_url") or "gitlab.com"
         return config
 
-    def _groups_stream(self, config: MutableMapping[str, Any]) -> Groups:
-        if not self.__groups_stream:
-            auth_params = self._auth_params(config)
-            group_ids = list(map(lambda x: x["id"], self._get_group_list(config)))
-            self.__groups_stream = Groups(group_ids=group_ids, **auth_params)
-        return self.__groups_stream
-
-    def _projects_stream(self, config: MutableMapping[str, Any]) -> Union[Projects, GroupProjects]:
-        if not self.__projects_stream:
-            auth_params = self._auth_params(config)
-            project_ids = config.get("projects_list", [])
-            groups_stream = self._groups_stream(config)
-            if groups_stream.group_ids:
-                self.__projects_stream = GroupProjects(project_ids=project_ids, parent_stream=groups_stream, **auth_params)
-                return self.__projects_stream
-            self.__projects_stream = Projects(project_ids=project_ids, **auth_params)
-        return self.__projects_stream
-
-    def _auth_params(self, config: MutableMapping[str, Any]) -> Mapping[str, Any]:
-        if not self.__auth_params:
-            auth = get_authenticator(config)
-            self.__auth_params = dict(authenticator=auth, api_url=config["api_url"])
-        return self.__auth_params
-
-    def _get_group_list(self, config: MutableMapping[str, Any]) -> List[str]:
-        group_ids = config.get("groups_list")
-        # Gitlab exposes different APIs to get a list of groups.
-        # We use https://docs.gitlab.com/ee/api/groups.html#list-groups in case there's no group IDs in the input config.
-        # This API provides full information about all available groups, including subgroups.
-        #
-        # In case there is a definitive list of groups IDs in the input config, the above API can not be used since
-        # it does not support filtering by group ID, so we use
-        # https://docs.gitlab.com/ee/api/groups.html#details-of-a-group and
-        # https: //docs.gitlab.com/ee/api/groups.html#list-a-groups-descendant-groups for each group ID. The latter one does not
-        # provide full group info so can only be used to retrieve  alist of group IDs and pass it further to init a corresponding stream.
-        auth_params = self._auth_params(config)
-        stream = GroupsList(**auth_params) if not group_ids else IncludeDescendantGroups(group_ids=group_ids, **auth_params)
-        for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh):
-            yield from stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)
-
-    @staticmethod
-    def _is_http_allowed() -> bool:
-        return os.environ.get("DEPLOYMENT_MODE", "").upper() != "CLOUD"
-
-    def _try_refresh_access_token(self, logger, config: Mapping[str, Any]) -> Mapping[str, Any]:
-        """
-        This method attempts to refresh the expired `access_token`, while `refresh_token` is still valid.
-        In order to obtain the new `refresh_token`, the Customer should `re-auth` in the source settings.
-        """
-        # get current authenticator
-        authenticator: Union[SingleUseRefreshTokenOauth2Authenticator, TokenAuthenticator] = self.__auth_params.get("authenticator")
-        if isinstance(authenticator, SingleUseRefreshTokenOauth2Authenticator):
-            try:
-                creds = authenticator.refresh_access_token()
-                # update the actual config values
-                config["credentials"]["access_token"] = creds[0]
-                config["credentials"]["refresh_token"] = creds[3]
-                config["credentials"]["token_expiry_date"] = authenticator.get_new_token_expiry_date(creds[1], creds[2]).to_rfc3339_string()
-                # update the config
-                emit_configuration_as_airbyte_control_message(config)
-                logger.info("The `access_token` was successfully refreshed.")
-                return config
-            except (AirbyteTracedException, HTTPError) as http_error:
-                raise http_error
-            except Exception as e:
-                raise Exception(f"Unknown error occurred while refreshing the `access_token`, details: {e}")
-
-    def _handle_expired_access_token_error(self, logger, config: Mapping[str, Any]) -> Tuple[bool, Any]:
-        try:
-            return self.check_connection(logger, self._try_refresh_access_token(logger, config))
-        except HTTPError as http_error:
-            return False, f"Unable to refresh the `access_token`, please re-authenticate in Sources > Settings. Details: {http_error}"
-
     def check_connection(self, logger, config) -> Tuple[bool, Any]:
         config = self._ensure_default_values(config)
         is_valid, scheme, _ = parse_url(config["api_url"])
         if not is_valid:
             return False, "Invalid API resource locator."
-        if scheme == "http" and not self._is_http_allowed():
+        if scheme == "http" and is_cloud_environment():
             return False, "Http scheme is not allowed in this environment. Please use `https` instead."
-        try:
-            projects = self._projects_stream(config)
-            for stream_slice in projects.stream_slices(sync_mode=SyncMode.full_refresh):
-                try:
-                    next(projects.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice))
-                    return True, None
-                except StopIteration:
-                    # in case groups/projects provided and 404 occurs
-                    return False, "Groups and/or projects that you provide are invalid or you don't have permission to view it."
-            return True, None  # in case there's no projects
-        except HTTPError as http_error:
-            if config["credentials"]["auth_type"] == "oauth2.0":
-                if http_error.response.status_code == 401:
-                    return self._handle_expired_access_token_error(logger, config)
-                elif http_error.response.status_code == 500:
-                    return False, f"Unable to connect to Gitlab API with the provided credentials - {repr(http_error)}"
-            else:
-                return False, f"Unable to connect to Gitlab API with the provided Private Access Token - {repr(http_error)}"
-        except Exception as error:
-            return False, f"Unknown error occurred while checking the connection - {repr(error)}"
-
-    def streams(self, config: MutableMapping[str, Any]) -> List[Stream]:
-        config = self._ensure_default_values(config)
-        auth_params = self._auth_params(config)
-        start_date = config.get("start_date")
-
-        groups, projects = self._groups_stream(config), self._projects_stream(config)
-        pipelines = Pipelines(parent_stream=projects, start_date=start_date, **auth_params)
-        merge_requests = MergeRequests(parent_stream=projects, start_date=start_date, **auth_params)
-        epics = Epics(parent_stream=groups, **auth_params)
-
-        streams = [
-            groups,
-            projects,
-            Branches(parent_stream=projects, repository_part=True, **auth_params),
-            Commits(parent_stream=projects, repository_part=True, start_date=start_date, **auth_params),
-            epics,
-            Deployments(parent_stream=projects, **auth_params),
-            EpicIssues(parent_stream=epics, **auth_params),
-            GroupIssueBoards(parent_stream=groups, **auth_params),
-            Issues(parent_stream=projects, start_date=start_date, **auth_params),
-            Jobs(parent_stream=pipelines, **auth_params),
-            ProjectMilestones(parent_stream=projects, **auth_params),
-            GroupMilestones(parent_stream=groups, **auth_params),
-            ProjectMembers(parent_stream=projects, **auth_params),
-            GroupMembers(parent_stream=groups, **auth_params),
-            ProjectLabels(parent_stream=projects, **auth_params),
-            GroupLabels(parent_stream=groups, **auth_params),
-            merge_requests,
-            MergeRequestCommits(parent_stream=merge_requests, **auth_params),
-            Releases(parent_stream=projects, **auth_params),
-            Tags(parent_stream=projects, repository_part=True, **auth_params),
-            pipelines,
-            PipelinesExtended(parent_stream=pipelines, **auth_params),
-            Users(parent_stream=projects, **auth_params),
-        ]
-
-        return streams
+        return super().check_connection(logger, config)

source_gitlab/streams.py

@@ -1,431 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import datetime
-from abc import ABC
-from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple
-
-import pendulum
-import requests
-from airbyte_cdk.models import SyncMode
-from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy
-from airbyte_cdk.sources.streams.core import StreamData
-from airbyte_cdk.sources.streams.http import HttpStream
-
-from .utils import parse_url
-
-
-class GitlabStream(HttpStream, ABC):
-    primary_key = "id"
-    raise_on_http_errors = True
-    stream_base_params = {}
-    flatten_id_keys = []
-    flatten_list_keys = []
-    per_page = 50
-    non_retriable_codes: List[int] = (403, 404)
-
-    def __init__(self, api_url: str, **kwargs):
-        super().__init__(**kwargs)
-        self.api_url = api_url
-        self.page = 1
-
-    def read_records(
-        self,
-        sync_mode: SyncMode,
-        cursor_field: List[str] = None,
-        stream_slice: Mapping[str, Any] = None,
-        stream_state: Mapping[str, Any] = None,
-    ) -> Iterable[StreamData]:
-        self.page = 1
-        yield from super().read_records(sync_mode, cursor_field, stream_slice, stream_state)
-
-    def request_params(
-        self,
-        stream_state: Mapping[str, Any],
-        stream_slice: Mapping[str, Any] = None,
-        next_page_token: Mapping[str, Any] = None,
-    ) -> MutableMapping[str, Any]:
-        params = {"per_page": self.per_page}
-        if next_page_token:
-            params.update(next_page_token)
-        params.update(self.stream_base_params)
-        return params
-
-    @property
-    def url_base(self) -> str:
-        _, scheme, host = parse_url(self.api_url)
-        return f"{scheme}://{host}/api/v4/"
-
-    @property
-    def availability_strategy(self) -> Optional["AvailabilityStrategy"]:
-        return None
-
-    def should_retry(self, response: requests.Response) -> bool:
-        # Gitlab API returns a 403 response in case a feature is disabled in a project (pipelines/jobs for instance).
-        if response.status_code in self.non_retriable_codes:
-            setattr(self, "raise_on_http_errors", False)
-            self.logger.warning(
-                f"Got {response.status_code} error when accessing URL {response.request.url}."
-                f" Very likely the feature is disabled for this project and/or group. Please double check it, or report a bug otherwise."
-            )
-            return False
-        return super().should_retry(response)
-
-    def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
-        if response.status_code in self.non_retriable_codes:
-            return
-        response_data = response.json()
-        if isinstance(response_data, dict):
-            return None
-        if len(response_data) == self.per_page:
-            self.page += 1
-            return {"page": self.page}
-
-    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        if response.status_code in self.non_retriable_codes:
-            return []
-        response_data = response.json()
-        if isinstance(response_data, list):
-            for record in response_data:
-                yield self.transform(record, **kwargs)
-        elif isinstance(response_data, dict):
-            yield self.transform(response_data, **kwargs)
-        else:
-            self.logger.info(f"Unsupported type of response data for stream {self.name}")
-
-    def transform(self, record: Dict[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs):
-        for key in self.flatten_id_keys:
-            self._flatten_id(record, key)
-
-        for key in self.flatten_list_keys:
-            self._flatten_list(record, key)
-
-        return record
-
-    def _flatten_id(self, record: Dict[str, Any], target: str):
-        target_value = record.get(target, None)
-        record[target + "_id"] = target_value.get("id") if target_value else None
-
-    def _flatten_list(self, record: Dict[str, Any], target: str):
-        record[target] = [target_data.get("id") for target_data in record.get(target, [])]
-
-
-class GitlabChildStream(GitlabStream):
-    path_list = ["id"]
-    flatten_parent_id = False
-
-    def __init__(self, parent_stream: GitlabStream, repository_part: bool = False, **kwargs):
-        super().__init__(**kwargs)
-        self.parent_stream = parent_stream
-        self.repo_url = repository_part
-
-    @property
-    def path_template(self) -> str:
-        template = [self.parent_stream.name] + ["{" + path_key + "}" for path_key in self.path_list]
-        if self.repo_url:
-            template.append("repository")
-        return "/".join(template + [self.name])
-
-    def stream_slices(
-        self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
-    ) -> Iterable[Optional[Mapping[str, any]]]:
-        for slice in self.parent_stream.stream_slices(sync_mode=SyncMode.full_refresh):
-            for record in self.parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice):
-                yield {path_key: record[path_key] for path_key in self.path_list}
-
-    def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str:
-        return self.path_template.format(**{path_key: stream_slice[path_key] for path_key in self.path_list})
-
-    def transform(self, record: Dict[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs):
-        super().transform(record, stream_slice, **kwargs)
-        if self.flatten_parent_id:
-            record[f"{self.parent_stream.name[:-1]}_id"] = stream_slice["id"]
-        return record
-
-
-class IncrementalGitlabChildStream(GitlabChildStream):
-    state_checkpoint_interval = 100
-    cursor_field = "updated_at"
-    lower_bound_filter = "updated_after"
-    upper_bound_filter = "updated_before"
-
-    def __init__(self, start_date, **kwargs):
-        super().__init__(**kwargs)
-        self._start_date = start_date
-
-    def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]:
-        """
-        Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object
-        and returning an updated state object.
-        """
-        project_id = latest_record.get("project_id")
-        latest_cursor_value = latest_record.get(self.cursor_field)
-        current_state = current_stream_state.get(str(project_id))
-        if current_state:
-            current_state = current_state.get(self.cursor_field)
-        current_state_value = current_state or latest_cursor_value
-        max_value = max(pendulum.parse(current_state_value), pendulum.parse(latest_cursor_value))
-        current_stream_state[str(project_id)] = {self.cursor_field: max_value.to_iso8601_string()}
-        return current_stream_state
-
-    @staticmethod
-    def _chunk_date_range(start_point: datetime.datetime) -> Iterable[Tuple[str, str]]:
-        end_point = datetime.datetime.now(datetime.timezone.utc)
-        if start_point > end_point:
-            return []
-        current_start, current_end = start_point, start_point
-        while current_end < end_point:
-            current_end = current_start + datetime.timedelta(days=180)
-            current_end = min(current_end, end_point)
-            yield str(current_start), str(current_end)
-            current_start = current_end + datetime.timedelta(seconds=1)
-
-    def stream_slices(
-        self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
-    ) -> Iterable[Optional[Mapping[str, Any]]]:
-        stream_state = stream_state or {}
-        super_slices = super().stream_slices(sync_mode, cursor_field, stream_state)
-        for super_slice in super_slices:
-            state_project_value = stream_state.get(str(super_slice["id"]))
-            if self._start_date or state_project_value:
-                start_point = self._start_date
-                if state_project_value:
-                    state_value = state_project_value.get(self.cursor_field)
-                    if state_value and start_point:
-                        start_point = max(start_point, state_value)
-                    else:
-                        start_point = state_value or start_point
-                for start_dt, end_dt in self._chunk_date_range(pendulum.parse(start_point)):
-                    stream_slice = {key: value for key, value in super_slice.items()}
-                    stream_slice[self.lower_bound_filter] = start_dt
-                    stream_slice[self.upper_bound_filter] = end_dt
-                    yield stream_slice
-            else:
-                stream_slice = {key: value for key, value in super_slice.items()}
-                yield stream_slice
-
-    def request_params(self, stream_state=None, stream_slice: Mapping[str, Any] = None, **kwargs):
-        params = super().request_params(stream_state, stream_slice, **kwargs)
-        lower_bound_filter = stream_slice.get(self.lower_bound_filter)
-        upper_bound_filter = stream_slice.get(self.upper_bound_filter)
-        if lower_bound_filter and upper_bound_filter:
-            params[self.lower_bound_filter] = lower_bound_filter
-            params[self.upper_bound_filter] = upper_bound_filter
-        return params
-
-
-class Groups(GitlabStream):
-    use_cache = True
-
-    def __init__(self, group_ids: List, **kwargs):
-        super().__init__(**kwargs)
-        self.group_ids = group_ids
-
-    def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
-        return f"groups/{stream_slice['id']}"
-
-    def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]:
-        for gid in self.group_ids:
-            yield {"id": gid}
-
-    def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs):
-        record["projects"] = [
-            {"id": project["id"], "path_with_namespace": project["path_with_namespace"]} for project in record.pop("projects", [])
-        ]
-        return record
-
-
-class IncludeDescendantGroups(Groups):
-    def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
-        return stream_slice["path"]
-
-    def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]:
-        for gid in self.group_ids:
-            yield {"path": f"groups/{gid}"}
-            yield {"path": f"groups/{gid}/descendant_groups"}
-
-
-class GroupsList(GitlabStream):
-    def path(self, **kwargs) -> str:
-        return "groups"
-
-
-class Projects(GitlabStream):
-    stream_base_params = {"statistics": 1}
-    use_cache = True
-
-    def __init__(self, project_ids: List = None, **kwargs):
-        super().__init__(**kwargs)
-        self.project_ids = project_ids
-
-    def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
-        return f"projects/{stream_slice['id']}"
-
-    def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]:
-        for pid in self.project_ids:
-            yield {"id": pid.replace("/", "%2F")}
-
-
-class GroupProjects(Projects):
-    name = "projects"
-
-    def __init__(self, parent_stream: GitlabStream = None, **kwargs):
-        super().__init__(**kwargs)
-        self.parent_stream = parent_stream
-
-    def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]:
-        group_project_ids = set()
-        for slice in self.parent_stream.stream_slices(sync_mode=SyncMode.full_refresh):
-            for record in self.parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice):
-                group_project_ids.update({i["path_with_namespace"] for i in record["projects"]})
-        for pid in group_project_ids:
-            if not self.project_ids or self.project_ids and pid in self.project_ids:
-                yield {"id": pid.replace("/", "%2F")}
-
-
-class GroupMilestones(GitlabChildStream):
-    path_template = "groups/{id}/milestones"
-
-
-class ProjectMilestones(GitlabChildStream):
-    path_template = "projects/{id}/milestones"
-
-
-class GroupMembers(GitlabChildStream):
-    path_template = "groups/{id}/members"
-    flatten_parent_id = True
-
-
-class ProjectMembers(GitlabChildStream):
-    path_template = "projects/{id}/members"
-    flatten_parent_id = True
-
-
-class GroupLabels(GitlabChildStream):
-    path_template = "groups/{id}/labels"
-    flatten_parent_id = True
-
-
-class ProjectLabels(GitlabChildStream):
-    path_template = "projects/{id}/labels"
-    flatten_parent_id = True
-
-
-class Branches(GitlabChildStream):
-    primary_key = "name"
-    flatten_id_keys = ["commit"]
-    flatten_parent_id = True
-
-
-class Commits(IncrementalGitlabChildStream):
-    cursor_field = "created_at"
-    lower_bound_filter = "since"
-    upper_bound_filter = "until"
-    flatten_parent_id = True
-    stream_base_params = {"with_stats": True}
-
-
-class Issues(IncrementalGitlabChildStream):
-    stream_base_params = {"scope": "all"}
-    flatten_id_keys = ["author", "assignee", "closed_by", "milestone"]
-    flatten_list_keys = ["assignees"]
-
-
-class MergeRequests(IncrementalGitlabChildStream):
-    stream_base_params = {"scope": "all"}
-    flatten_id_keys = ["author", "assignee", "closed_by", "milestone", "merged_by"]
-    flatten_list_keys = ["assignees"]
-
-
-class MergeRequestCommits(GitlabChildStream):
-    """Docs: https://docs.gitlab.com/ee/api/merge_requests.html#get-single-merge-request-commits"""
-
-    path_list = ["project_id", "iid"]
-    path_template = "projects/{project_id}/merge_requests/{iid}/commits"
-
-    def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs):
-        super().transform(record, stream_slice, **kwargs)
-        record["project_id"] = stream_slice["project_id"]
-        record["merge_request_iid"] = stream_slice["iid"]
-
-        return record
-
-
-class Releases(GitlabChildStream):
-    primary_key = "name"
-    flatten_id_keys = ["author", "commit"]
-    flatten_list_keys = ["milestones"]
-
-    def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs):
-        super().transform(record, stream_slice, **kwargs)
-        record["project_id"] = stream_slice["id"]
-
-        return record
-
-
-class Tags(GitlabChildStream):
-    primary_key = "name"
-    flatten_id_keys = ["commit"]
-
-    def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs):
-        super().transform(record, stream_slice, **kwargs)
-        record["project_id"] = stream_slice["id"]
-
-        return record
-
-
-class Pipelines(IncrementalGitlabChildStream):
-    pass
-
-
-class PipelinesExtended(GitlabChildStream):
-    path_list = ["project_id", "id"]
-    path_template = "projects/{project_id}/pipelines/{id}"
-
-
-class Jobs(GitlabChildStream):
-    flatten_id_keys = ["user", "pipeline", "runner", "commit"]
-    path_list = ["project_id", "id"]
-    path_template = "projects/{project_id}/pipelines/{id}/jobs"
-
-    def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs):
-        super().transform(record, stream_slice, **kwargs)
-        record["project_id"] = stream_slice["project_id"]
-        return record
-
-
-class GroupIssueBoards(GitlabChildStream):
-    path_template = "groups/{id}/boards"
-    flatten_parent_id = True
-
-
-class Users(GitlabChildStream):
-    pass
-
-
-class Epics(GitlabChildStream):
-    primary_key = "iid"
-    flatten_id_keys = ["author"]
-
-
-class EpicIssues(GitlabChildStream):
-    primary_key = "epic_issue_id"
-    path_list = ["group_id", "iid"]
-    flatten_id_keys = ["milestone", "assignee", "author"]
-    flatten_list_keys = ["assignees"]
-    path_template = "groups/{group_id}/epics/{iid}/issues"
-
-
-class Deployments(GitlabChildStream):
-    primary_key = "id"
-    flatten_id_keys = ["user", "environment"]
-    path_template = "projects/{id}/deployments"
-
-    def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs):
-        super().transform(record, stream_slice, **kwargs)
-        record["user_username"] = record["user"]["username"]
-        record["user_full_name"] = record["user"]["name"]
-        record["environment_name"] = record["environment"]["name"]
-        record["project_id"] = stream_slice["id"]
-        return record