qontract-reconcile 0.10.2.dev174__py3-none-any.whl → 0.10.2.dev176__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: qontract-reconcile
3
- Version: 0.10.2.dev174
3
+ Version: 0.10.2.dev176
4
4
  Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
5
5
  Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
6
6
  Project-URL: repository, https://github.com/app-sre/qontract-reconcile
@@ -32,10 +32,10 @@ reconcile/github_validator.py,sha256=-j17tn3csFVjPMSPL3te48iWVkPZCncRXdeKeLdGjjQ
32
32
  reconcile/gitlab_fork_compliance.py,sha256=RbHckzLnE9zkOFHJANzoejEMMbMAivmqJVs3Suvp9lU,4591
33
33
  reconcile/gitlab_housekeeping.py,sha256=c31Jtw5t8bnOzUO9jMWF_0DHitPzol93AA7YWBxM5L0,25416
34
34
  reconcile/gitlab_labeler.py,sha256=BA2dbXsN9hErUwJl22qcxfeH7XiPCuQ9LN3NddWdnpo,4540
35
- reconcile/gitlab_members.py,sha256=MUIgYDLeJx2-_vMypyq2Pa17cpKdXATYhtVACS2ghpQ,8297
35
+ reconcile/gitlab_members.py,sha256=yRZOZqwB9_FJ5DWIFEod6hoG0X38z36atInNshAWddI,8263
36
36
  reconcile/gitlab_mr_sqs_consumer.py,sha256=i_MDVfA3Uk_TJiNkfEJzhO6_rwR7z3I3dH9oEw686U4,2681
37
37
  reconcile/gitlab_owners.py,sha256=nIEsf3QWI3yIw_Bxy5oMaCmszTaNZDwQVaaZZxPgh4g,14447
38
- reconcile/gitlab_permissions.py,sha256=gSGH6gAdJbPy5Z0rQGUqiNQSHty_tXQ_3Y4OQP_8nFs,8067
38
+ reconcile/gitlab_permissions.py,sha256=kZEdWL0rewP7Odz8amRBPToKxkn0IQn81IoroHGdga4,8101
39
39
  reconcile/gitlab_projects.py,sha256=K3tFf_aD1W4Ijp5q-9Qek3kwFGEWPcZ1kd7tzFJ4GyQ,1781
40
40
  reconcile/integrations_manager.py,sha256=CY7cOj5dzt2se4IOg11VQvGQ-eTvLML5Q42Z9SSgeSk,9463
41
41
  reconcile/jenkins_base.py,sha256=0Gocu3fU2YTltaxBlbDQOUvP-7CP2OSQV1ZRwtWeVXw,875
@@ -123,7 +123,7 @@ reconcile/vpc_peerings_validator.py,sha256=aESqrhm1tpkc2iqSL1UV5to_HjNgjRSffD0cr
123
123
  reconcile/aus/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
124
124
  reconcile/aus/advanced_upgrade_service.py,sha256=lt684trHbKvVDLwwuNVz3Wu_MnytFSbS_7MZTIITh9k,23969
125
125
  reconcile/aus/aus_label_source.py,sha256=o0S2f0qwcII_8nzhHZhRQ83gEZ1DrSXyO4xzSwLebuU,4382
126
- reconcile/aus/base.py,sha256=K-CLPUDpaFSfFsf_UgcvaUPCgqTLTGkx_aoUH0VWhv8,50431
126
+ reconcile/aus/base.py,sha256=rx2OuShoFRP7O6Kov9rRjEkhCPpPavfDF81tieB6XFg,50747
127
127
  reconcile/aus/cluster_version_data.py,sha256=VZWbUEIbrDKO-sroMpQtiWCTqDraTMd8tssKV0HyTQ0,7140
128
128
  reconcile/aus/healthchecks.py,sha256=jR9c-syh9impnkV0fd6XW3Bnk7iRN5zv8oCRYM-yIRY,2700
129
129
  reconcile/aus/metrics.py,sha256=nKT4m2zGT-QOMR0c-z-npVNKWsNMubzdffpU_f9n4II,3927
@@ -615,8 +615,8 @@ reconcile/utils/external_resource_spec.py,sha256=qeupz4t4trd2uPjlHjf_AFA9Y-EKrMn
615
615
  reconcile/utils/external_resources.py,sha256=YzTb0xAcNdmKO326mGQy7BmST56CZcdru4lX7ai_7kw,7579
616
616
  reconcile/utils/filtering.py,sha256=S4PbMHuFr3ED0P2Q_ea5CAaB7FimI62B-F5YTaKrphA,402
617
617
  reconcile/utils/git.py,sha256=o4p9m8jlzCJDcutl2HErvGLhL6sZ1NB4Aw3zGcQIzso,2427
618
- reconcile/utils/github_api.py,sha256=y3fxty7FKvfhdzfHgGSaIstL6A_Y2loUcMiyIK5TMDg,2750
619
- reconcile/utils/gitlab_api.py,sha256=SYSKm5WulWinO7P-ZYy_oerKdfNHHob2V6i7Mfr4oCU,26643
618
+ reconcile/utils/github_api.py,sha256=o4J0ZU1ZSr9808uoorKHv19iae-eLo85yrCZX67p2kw,2822
619
+ reconcile/utils/gitlab_api.py,sha256=5zyC4bYKxwKdPVbRRTXUBaqxgEiQBh52jHbi9G_tBHI,27398
620
620
  reconcile/utils/gpg.py,sha256=EKG7_fdMv8BMlV5yUdPiqoTx-KrzmVSEAl2sLkaKwWI,1123
621
621
  reconcile/utils/gql.py,sha256=C0thIm_k9MBldfqwHzyqtYZk9sIvMdm9IbbnXLGwjD8,14158
622
622
  reconcile/utils/grouping.py,sha256=vr9SFHZ7bqmHYrvYcEZt-Er3-yQYfAAdq5sHLZVmXPY,456
@@ -653,7 +653,7 @@ reconcile/utils/promtool.py,sha256=xmPBWEApkk0L2qZBAvTxakNXxfTz-tVLPFxGnpsxXnM,2
653
653
  reconcile/utils/quay_api.py,sha256=uE_jxcdy3ViHtYFAfwDQuFDaO7Pr6AAPoVnmORbyHio,7822
654
654
  reconcile/utils/quay_mirror.py,sha256=dpWCNv5lITwIk6Q9RkmqaQKHNk_JPy27UQEribJ7E-U,1324
655
655
  reconcile/utils/raw_github_api.py,sha256=2WKtE8ABYYB9UGOAh9N_kLkksBWL3320Z2_scteZddI,2805
656
- reconcile/utils/repo_owners.py,sha256=BHrAXxKyvn4qWJwFPWYGTtfgnLmYnWtYFEJGFeD__FE,6573
656
+ reconcile/utils/repo_owners.py,sha256=Xwe1HOcMZe7Pknk47GLZHg5LDpDElmGfmc_x6pAdzsg,6589
657
657
  reconcile/utils/rest_api_base.py,sha256=MT7tp6CQO2S5aKfVOzw_hipWg7wAGoOqkm4qurI1hEU,4342
658
658
  reconcile/utils/ruamel.py,sha256=FzL4_L0FnMOUZmgThrZSMJs5MTdXwiy-E9MZWfk8bh8,397
659
659
  reconcile/utils/secret_reader.py,sha256=MaP56KZaAE35EyYbgAitdm6fUSxdzWeGFSOym9qiZkw,10206
@@ -761,7 +761,7 @@ reconcile/utils/runtime/sharding.py,sha256=r0ieUtNed7NvknSw6qQrCkKpVXE1shuHGnfFc
761
761
  reconcile/utils/saasherder/__init__.py,sha256=3U8plqMAPRE1kjwZ5YnIsYsggTf4_gS7flRUEuXVBAs,343
762
762
  reconcile/utils/saasherder/interfaces.py,sha256=NEYQspYfyWQhBeJyNCqSFbixi1A4wRVGB7FeNM5BDCk,9141
763
763
  reconcile/utils/saasherder/models.py,sha256=JaOz_DEtudJZhiDe90kaBlJkppFufn81V92oK9PHYx0,10208
764
- reconcile/utils/saasherder/saasherder.py,sha256=ZeYwUSrWbJ0XkmQv92dUGPrhxd5zBKnDEM7_uzRroFE,87067
764
+ reconcile/utils/saasherder/saasherder.py,sha256=crF_hQBeL9TBv_R6SafAze6xutXFSNEX77KaT4XqjF8,87135
765
765
  reconcile/utils/terraform/__init__.py,sha256=zNbiyTWo35AT1sFTElL2j_AA0jJ_yWE_bfFn-nD2xik,250
766
766
  reconcile/utils/terraform/config.py,sha256=5UVrd563TMcvi4ooa5JvWVDW1I3bIWg484u79evfV_8,164
767
767
  reconcile/utils/terraform/config_client.py,sha256=gRL1rQ0AqvShei_rcGqC3HDYGskOFKE1nPrJyJE9yno,4676
@@ -807,7 +807,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
807
807
  tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
808
808
  tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
809
809
  tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
810
- qontract_reconcile-0.10.2.dev174.dist-info/METADATA,sha256=BxdoN9UJDbN5C4PogvVgSSyKg_ruSMt9Ee0gMjCnZB8,24627
811
- qontract_reconcile-0.10.2.dev174.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
812
- qontract_reconcile-0.10.2.dev174.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
813
- qontract_reconcile-0.10.2.dev174.dist-info/RECORD,,
810
+ qontract_reconcile-0.10.2.dev176.dist-info/METADATA,sha256=VAwixwvzafGc_zGmUcXBP610KqCPyqenHilqpzHXIW8,24627
811
+ qontract_reconcile-0.10.2.dev176.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
812
+ qontract_reconcile-0.10.2.dev176.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
813
+ qontract_reconcile-0.10.2.dev176.dist-info/RECORD,,
reconcile/aus/base.py CHANGED
@@ -970,56 +970,53 @@ def verify_schedule_should_skip(
970
970
  return next_schedule.strftime("%Y-%m-%dT%H:%M:%SZ")
971
971
 
972
972
 
973
- def verify_lock_should_skip(
974
- desired: ClusterUpgradeSpec, locked: dict[str, str]
975
- ) -> bool:
976
- mutexes = desired.effective_mutexes
977
- if any(lock in locked for lock in mutexes):
978
- locking = {lock: locked[lock] for lock in mutexes if lock in locked}
979
- logging.debug(
980
- f"[{desired.org.org_id}/{desired.org.name}/{desired.cluster.name}] skipping cluster: locked out by {locking}"
981
- )
982
- return True
983
- return False
984
-
985
-
986
973
  def verify_max_upgrades_should_skip(
987
974
  desired: ClusterUpgradeSpec,
988
- sector_upgrades: dict[str, set[str]],
975
+ locked: dict[str, str],
976
+ sector_mutex_upgrades: dict[tuple[str, str], set[str]],
989
977
  sector: Sector | None,
990
978
  ) -> bool:
991
- if sector is None:
992
- return False
979
+ mutexes = desired.effective_mutexes
993
980
 
994
- current_upgrades = sector_upgrades[sector.name]
995
- # Allow at least one upgrade
996
- if len(current_upgrades) == 0:
981
+ # if sector.max_parallel_upgrades is not set, we allow 1 upgrade per mutex, across the whole org
982
+ if sector is None or sector.max_parallel_upgrades is None:
983
+ if any(lock in locked for lock in mutexes):
984
+ locking = {lock: locked[lock] for lock in mutexes if lock in locked}
985
+ logging.debug(
986
+ f"[{desired.org.org_id}/{desired.org.name}/{desired.cluster.name}] skipping cluster: locked out by {locking}"
987
+ )
988
+ return True
997
989
  return False
998
990
 
999
- # if sector.max_parallel_upgrades is not set, we allow all upgrades
1000
- if sector.max_parallel_upgrades is None:
1001
- return False
991
+ current_upgrades_count_per_mutex = {
992
+ mutex: len(sector_mutex_upgrades[sector.name, mutex]) for mutex in mutexes
993
+ }
1002
994
 
1003
- sector_cluster_count = len(sector.specs)
995
+ current_upgrades_total_count = sum(current_upgrades_count_per_mutex.values())
996
+ if current_upgrades_total_count == 0:
997
+ return False
1004
998
 
1005
- if sector.max_parallel_upgrades.endswith("%"):
1006
- max_parallel_upgrades_percent = int(sector.max_parallel_upgrades[:-1])
1007
- max_parallel_upgrades = round(
1008
- sector_cluster_count * max_parallel_upgrades_percent / 100
1009
- )
1010
- else:
1011
- max_parallel_upgrades = int(sector.max_parallel_upgrades)
999
+ for mutex in mutexes:
1000
+ cluster_count = len([s for s in sector.specs if mutex in s.effective_mutexes])
1001
+ if sector.max_parallel_upgrades.endswith("%"):
1002
+ max_parallel_upgrades_percent = int(sector.max_parallel_upgrades[:-1])
1003
+ max_parallel_upgrades = round(
1004
+ cluster_count * max_parallel_upgrades_percent / 100
1005
+ )
1006
+ else:
1007
+ max_parallel_upgrades = int(sector.max_parallel_upgrades)
1012
1008
 
1013
- # we allow at least one upgrade
1014
- if max_parallel_upgrades == 0:
1015
- max_parallel_upgrades = 1
1009
+ # we allow at least one upgrade
1010
+ if max_parallel_upgrades == 0:
1011
+ max_parallel_upgrades = 1
1016
1012
 
1017
- if len(current_upgrades) >= max_parallel_upgrades:
1018
- logging.debug(
1019
- f"[{desired.org.org_id}/{desired.org.name}/{desired.cluster.name}] skipping cluster: "
1020
- f"sector '{sector.name}' has reached max parallel upgrades {sector.max_parallel_upgrades}"
1021
- )
1022
- return True
1013
+ if current_upgrades_count_per_mutex.get(mutex, 0) >= max_parallel_upgrades:
1014
+ logging.debug(
1015
+ f"[{desired.org.org_id}/{desired.org.name}/{desired.cluster.name}] skipping cluster: "
1016
+ f"sector '{sector.name}' has reached max parallel upgrades {sector.max_parallel_upgrades} "
1017
+ f"for mutex '{mutex}'"
1018
+ )
1019
+ return True
1023
1020
 
1024
1021
  return False
1025
1022
 
@@ -1085,15 +1082,15 @@ def calculate_diff(
1085
1082
  """
1086
1083
 
1087
1084
  locked: dict[str, str] = {}
1088
- sector_upgrades: dict[str, set[str]] = defaultdict(set)
1085
+ sector_mutex_upgrades: dict[tuple[str, str], set[str]] = defaultdict(set)
1089
1086
 
1090
1087
  def set_upgrading(
1091
- cluster_id: str, mutexes: set[str] | None, sector_name: str | None
1088
+ cluster_id: str, mutexes: set[str], sector_name: str | None
1092
1089
  ) -> None:
1093
- for mutex in mutexes or set():
1090
+ for mutex in mutexes:
1094
1091
  locked[mutex] = cluster_id
1095
- if sector_name:
1096
- sector_upgrades[sector_name].add(cluster_id)
1092
+ if sector_name:
1093
+ sector_mutex_upgrades[sector_name, mutex].add(cluster_id)
1097
1094
 
1098
1095
  diffs: list[UpgradePolicyHandler] = []
1099
1096
 
@@ -1113,7 +1110,9 @@ def calculate_diff(
1113
1110
  # Upgrading node pools, only required for Hypershift clusters
1114
1111
  # do this in the same loop, to skip cluster on node pool upgrade
1115
1112
  if spec.cluster.is_rosa_hypershift():
1116
- if verify_lock_should_skip(spec, locked):
1113
+ if verify_max_upgrades_should_skip(
1114
+ spec, locked, sector_mutex_upgrades, sector
1115
+ ):
1117
1116
  continue
1118
1117
 
1119
1118
  node_pool_update = _calculate_node_pool_diffs(spec, now)
@@ -1135,10 +1134,7 @@ def calculate_diff(
1135
1134
  if not next_schedule:
1136
1135
  continue
1137
1136
 
1138
- if verify_lock_should_skip(spec, locked):
1139
- continue
1140
-
1141
- if verify_max_upgrades_should_skip(spec, sector_upgrades, sector):
1137
+ if verify_max_upgrades_should_skip(spec, locked, sector_mutex_upgrades, sector):
1142
1138
  continue
1143
1139
 
1144
1140
  version = upgradeable_version(spec, version_data, sector)
@@ -68,9 +68,7 @@ def get_current_state(
68
68
  """Get current gitlab group members for all managed groups."""
69
69
  return {
70
70
  g: CurrentStateSpec(
71
- members={
72
- u.username: u for u in gl.get_group_members(gitlab_groups_map.get(g))
73
- },
71
+ members={u.username: u for u in gl.get_group_members(gitlab_groups_map[g])},
74
72
  )
75
73
  for g in instance.managed_groups
76
74
  }
@@ -1,12 +1,12 @@
1
1
  import itertools
2
2
  import logging
3
3
  from dataclasses import dataclass
4
- from typing import Any
4
+ from typing import Any, cast
5
5
 
6
6
  from gitlab.exceptions import GitlabGetError
7
7
  from gitlab.v4.objects import (
8
- GroupProject,
9
8
  Project,
9
+ SharedProject,
10
10
  )
11
11
  from sretoolbox.utils import threaded
12
12
 
@@ -51,9 +51,9 @@ class GroupPermissionHandler:
51
51
  for project_repo_url in filtered_project_repos
52
52
  }
53
53
  # get all projects shared with group
54
- shared_projects = self.gl.get_items(self.group.shared_projects.list)
54
+ shared_projects = self.group.shared_projects.list(iterator=True)
55
55
  current_state = {
56
- project.web_url: self.extract_group_spec(project)
56
+ project.web_url: self.extract_group_spec(cast(SharedProject, project))
57
57
  for project in shared_projects
58
58
  }
59
59
  self.reconcile(desired_state, current_state)
@@ -61,13 +61,14 @@ class GroupPermissionHandler:
61
61
  def filter_group_owned_projects(self, repos: list[str]) -> set[str]:
62
62
  # get only the projects that are owned by group and its sub groups
63
63
  query = {"with_shared": False, "include_subgroups": True}
64
- group_owned_projects = self.gl.get_items(
65
- self.group.projects.list, query_parameters=query
64
+ group_owned_projects = self.group.projects.list(
65
+ query_parameters=query,
66
+ iterator=True,
66
67
  )
67
68
  group_owned_repo_list = {project.web_url for project in group_owned_projects}
68
69
  return set(repos) - group_owned_repo_list
69
70
 
70
- def extract_group_spec(self, project: GroupProject) -> GroupSpec:
71
+ def extract_group_spec(self, project: SharedProject) -> GroupSpec:
71
72
  return next(
72
73
  GroupSpec(
73
74
  group_name=self.group.name,
@@ -54,9 +54,14 @@ class GithubRepositoryApi:
54
54
  Align with GitLabApi
55
55
  """
56
56
 
57
- def get_repository_tree(self, ref: str = "master") -> list[dict[str, str]]:
57
+ def get_repository_tree(
58
+ self,
59
+ *,
60
+ ref: str = "master",
61
+ recursive: bool = False,
62
+ ) -> list[dict[str, str]]:
58
63
  tree_items = []
59
- for item in self._repo.get_git_tree(sha=ref, recursive=True).tree:
64
+ for item in self._repo.get_git_tree(sha=ref, recursive=recursive).tree:
60
65
  tree_item = {"path": item.path, "name": Path(item.path).name}
61
66
  tree_items.append(tree_item)
62
67
  return tree_items
@@ -2,10 +2,8 @@ import logging
2
2
  import os
3
3
  import re
4
4
  from collections.abc import (
5
- Callable,
6
5
  Iterable,
7
6
  Mapping,
8
- Set,
9
7
  )
10
8
  from functools import cached_property
11
9
  from operator import (
@@ -66,6 +64,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
66
64
  MR_DESCRIPTION_COMMENT_ID = 0
67
65
 
68
66
  DEFAULT_MAIN_BRANCH = "master"
67
+ MAX_PER_PAGE = 100
69
68
 
70
69
 
71
70
  class MRState:
@@ -135,6 +134,8 @@ class GitLabApi:
135
134
  ssl_verify=self.ssl_verify,
136
135
  timeout=timeout,
137
136
  session=self.session,
137
+ per_page=MAX_PER_PAGE,
138
+ pagination="keyset",
138
139
  )
139
140
  self._auth()
140
141
  assert self.gl.user
@@ -267,15 +268,15 @@ class GitLabApi:
267
268
  project = self.project if repo_url is None else self.get_project(repo_url)
268
269
  if project is None:
269
270
  return None
270
- if query:
271
- members = self.get_items(project.members_all.list, query_parameters=query)
272
- else:
273
- members = self.get_items(project.members_all.list)
271
+ members = project.members.list(iterator=True, query_parameters=query or {})
274
272
  return [m.username for m in members if m.access_level >= 40]
275
273
 
276
274
  def get_app_sre_group_users(self) -> list[GroupMember]:
277
275
  app_sre_group = self.gl.groups.get("app-sre")
278
- return self.get_items(app_sre_group.members.list)
276
+ return cast(
277
+ list[GroupMember],
278
+ app_sre_group.members.list(get_all=True),
279
+ )
279
280
 
280
281
  def get_group_if_exists(self, group_name: str) -> Group | None:
281
282
  try:
@@ -309,16 +310,12 @@ class GitLabApi:
309
310
  """
310
311
  return GROUP_BOT_NAME_REGEX.match(username) is not None
311
312
 
312
- def get_group_members(self, group: Group | None) -> list[GroupMember]:
313
- if group is None:
314
- logging.error("no group provided")
315
- return []
316
- else:
317
- return [
318
- m
319
- for m in self.get_items(group.members.list)
320
- if not self._is_bot_username(m.username)
321
- ]
313
+ def get_group_members(self, group: Group) -> list[GroupMember]:
314
+ return [
315
+ cast(GroupMember, m)
316
+ for m in group.members.list(iterator=True)
317
+ if not self._is_bot_username(m.username)
318
+ ]
322
319
 
323
320
  def add_project_member(
324
321
  self, repo_url: str, user: GroupMember, access: str = "maintainer"
@@ -374,9 +371,9 @@ class GitLabApi:
374
371
  case _:
375
372
  raise ValueError(f"Invalid access level: {access}")
376
373
 
377
- def get_group_id_and_projects(self, group_name: str) -> tuple[str, list[str]]:
374
+ def get_group_id_and_projects(self, group_name: str) -> tuple[str, set[str]]:
378
375
  group = self.gl.groups.get(group_name)
379
- return group.id, [p.name for p in self.get_items(group.projects.list)]
376
+ return group.id, {p.name for p in group.projects.list(iterator=True)}
380
377
 
381
378
  def get_group(self, group_name: str) -> Group:
382
379
  return self.gl.groups.get(group_name)
@@ -401,23 +398,34 @@ class GitLabApi:
401
398
  return self.gl.projects.get(project_id)
402
399
 
403
400
  def get_issues(self, state: str) -> list[ProjectIssue]:
404
- return self.get_items(self.project.issues.list, state=state)
401
+ return cast(
402
+ list[ProjectIssue],
403
+ self.project.issues.list(state=state, get_all=True),
404
+ )
405
405
 
406
406
  def get_merge_request(self, mr_id: str | int) -> ProjectMergeRequest:
407
407
  return self.project.mergerequests.get(mr_id)
408
408
 
409
409
  def get_merge_requests(self, state: str) -> list[ProjectMergeRequest]:
410
- return self.get_items(self.project.mergerequests.list, state=state)
410
+ return cast(
411
+ list[ProjectMergeRequest],
412
+ self.project.mergerequests.list(state=state, get_all=True),
413
+ )
411
414
 
415
+ @staticmethod
412
416
  def get_merge_request_label_events(
413
- self, mr: ProjectMergeRequest
417
+ mr: ProjectMergeRequest,
414
418
  ) -> list[ProjectMergeRequestResourceLabelEvent]:
415
- return self.get_items(mr.resourcelabelevents.list)
419
+ return cast(
420
+ list[ProjectMergeRequestResourceLabelEvent],
421
+ mr.resourcelabelevents.list(get_all=True),
422
+ )
416
423
 
417
- def get_merge_request_pipelines(self, mr: ProjectMergeRequest) -> list[dict]:
424
+ @staticmethod
425
+ def get_merge_request_pipelines(mr: ProjectMergeRequest) -> list[dict]:
418
426
  # TODO: use typed object in return
419
427
  # TODO: use server side order_by
420
- items = self.get_items(mr.pipelines.list)
428
+ items = mr.pipelines.list(iterator=True)
421
429
  return sorted(
422
430
  [i.asdict() for i in items],
423
431
  key=itemgetter("created_at"),
@@ -457,7 +465,7 @@ class GitLabApi:
457
465
  "created_at": merge_request.created_at,
458
466
  "id": MR_DESCRIPTION_COMMENT_ID,
459
467
  })
460
- for note in GitLabApi.get_items(merge_request.notes.list):
468
+ for note in merge_request.notes.list(iterator=True):
461
469
  if note.system:
462
470
  continue
463
471
  comments.append({
@@ -485,8 +493,8 @@ class GitLabApi:
485
493
  self.delete_comment(c["note"])
486
494
 
487
495
  @retry()
488
- def get_project_labels(self) -> Set[str]:
489
- return {ln.name for ln in self.get_items(self.project.labels.list)}
496
+ def get_project_labels(self) -> set[str]:
497
+ return {label.name for label in self.project.labels.list(iterator=True)}
490
498
 
491
499
  @staticmethod
492
500
  def add_label_to_merge_request(
@@ -552,20 +560,6 @@ class GitLabApi:
552
560
  ) -> None:
553
561
  merge_request.notes.create({"body": body})
554
562
 
555
- # TODO: deprecated this method as new support of list(get_all=True)
556
- @staticmethod
557
- def get_items(method: Callable, **kwargs: Any) -> list:
558
- all_items = []
559
- page = 1
560
- while True:
561
- items = method(page=page, per_page=100, **kwargs)
562
- all_items.extend(items)
563
- if len(items) < 100:
564
- break
565
- page += 1
566
-
567
- return all_items
568
-
569
563
  def create_label(self, label_text: str, label_color: str) -> None:
570
564
  self.project.labels.create({"name": label_text, "color": label_color})
571
565
 
@@ -665,11 +659,36 @@ class GitLabApi:
665
659
  }
666
660
  p.hooks.create(hook)
667
661
 
668
- def get_repository_tree(self, ref: str = "master") -> list[dict]:
662
+ def get_repository_tree(
663
+ self,
664
+ *,
665
+ ref: str = "master",
666
+ recursive: bool = False,
667
+ project: Project | None = None,
668
+ path: str = "",
669
+ ) -> list[dict]:
669
670
  """
670
- Wrapper around Gitlab.repository_tree() with pagination enabled.
671
+ Get a list of repository files and directories in a project.
672
+
673
+ :param ref: The name of a repository branch or tag or, if not given, the default branch.
674
+ :param recursive: Boolean value used to get a recursive tree. Default is false.
675
+ :param project: The project to get the tree from, if None, use the current project
676
+ :param path: The path inside the repository. Used to get content of subdirectories.
677
+
678
+ :return: list of tree objects
671
679
  """
672
- return self.get_items(self.project.repository_tree, ref=ref, recursive=True)
680
+ target_project = self.project if project is None else project
681
+ return cast(
682
+ list[dict],
683
+ target_project.repository_tree(
684
+ ref=ref,
685
+ path=path,
686
+ recursive=recursive,
687
+ pagination="keyset",
688
+ per_page=MAX_PER_PAGE,
689
+ get_all=True,
690
+ ),
691
+ )
673
692
 
674
693
  def get_file(self, path: str, ref: str = "master") -> ProjectFile | None:
675
694
  """
@@ -762,22 +781,23 @@ class GitLabApi:
762
781
  author, assignee = last_assignment[0], last_assignment[1]
763
782
  return author in team_usernames and mr.assignee["username"] == assignee
764
783
 
765
- def last_assignment(self, mr: ProjectMergeRequest) -> tuple[str, str] | None:
784
+ @staticmethod
785
+ def last_assignment(mr: ProjectMergeRequest) -> tuple[str, str] | None:
786
+ """
787
+ Get the last assignment of a merge request.
788
+ :param mr: merge request
789
+ :return: tuple of author name and assignee name or None
790
+ """
766
791
  body_format = "assigned to @"
767
- notes = self.get_items(mr.notes.list)
768
-
769
- for note in notes:
770
- if not note.system:
771
- continue
772
- body = note.body
773
- if not body.startswith(body_format):
774
- continue
775
- assignee = body.replace(body_format, "")
776
- author = note.author["username"]
777
-
778
- return author, assignee
779
-
780
- return None
792
+ notes = mr.notes.list(sort="desc", order_by="created_at", iterator=True)
793
+ return next(
794
+ (
795
+ (note.author["username"], note.body.removeprefix(body_format))
796
+ for note in notes
797
+ if note.system and note.body.startswith(body_format)
798
+ ),
799
+ None,
800
+ )
781
801
 
782
802
  def last_comment(
783
803
  self, mr: ProjectMergeRequest, exclude_bot: bool = True
@@ -804,4 +824,7 @@ class GitLabApi:
804
824
  return response.get("commits", [])
805
825
 
806
826
  def get_personal_access_tokens(self) -> list[PersonalAccessToken]:
807
- return self.get_items(self.gl.personal_access_tokens.list)
827
+ return cast(
828
+ list[PersonalAccessToken],
829
+ self.gl.personal_access_tokens.list(get_all=True),
830
+ )
@@ -120,7 +120,7 @@ class RepoOwners:
120
120
  aliases = self._get_aliases()
121
121
 
122
122
  if self._recursive:
123
- repo_tree = self._git_cli.get_repository_tree(ref=self._ref)
123
+ repo_tree = self._git_cli.get_repository_tree(ref=self._ref, recursive=True)
124
124
  owner_files = [item for item in repo_tree if item["name"] == "OWNERS"]
125
125
  else:
126
126
  owner_files = [{"path": "OWNERS"}]
@@ -798,8 +798,11 @@ class SaasHerder: # pylint: disable=too-many-public-methods
798
798
  if not self.gitlab:
799
799
  raise Exception("gitlab is not initialized")
800
800
  project = self.gitlab.get_project(url)
801
- for item in self.gitlab.get_items(
802
- project.repository_tree, path=path.lstrip("/"), ref=commit_sha
801
+ for item in self.gitlab.get_repository_tree(
802
+ project=project,
803
+ path=path.lstrip("/"),
804
+ ref=commit_sha,
805
+ recursive=False,
803
806
  ):
804
807
  file_contents = project.files.get(
805
808
  file_path=item["path"], ref=commit_sha