qontract-reconcile 0.10.2.dev183__py3-none-any.whl → 0.10.2.dev184__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {qontract_reconcile-0.10.2.dev183.dist-info → qontract_reconcile-0.10.2.dev184.dist-info}/METADATA +1 -1
- {qontract_reconcile-0.10.2.dev183.dist-info → qontract_reconcile-0.10.2.dev184.dist-info}/RECORD +12 -12
- reconcile/dashdotdb_dora.py +18 -21
- reconcile/github_repo_permissions_validator.py +3 -3
- reconcile/service_dependencies.py +8 -10
- reconcile/slack_usergroups.py +7 -9
- reconcile/utils/gitlab_api.py +1 -1
- reconcile/utils/saasherder/saasherder.py +69 -64
- reconcile/utils/terrascript_aws_client.py +15 -15
- reconcile/utils/vcs.py +67 -28
- {qontract_reconcile-0.10.2.dev183.dist-info → qontract_reconcile-0.10.2.dev184.dist-info}/WHEEL +0 -0
- {qontract_reconcile-0.10.2.dev183.dist-info → qontract_reconcile-0.10.2.dev184.dist-info}/entry_points.txt +0 -0
{qontract_reconcile-0.10.2.dev183.dist-info → qontract_reconcile-0.10.2.dev184.dist-info}/METADATA
RENAMED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: qontract-reconcile
|
3
|
-
Version: 0.10.2.
|
3
|
+
Version: 0.10.2.dev184
|
4
4
|
Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
|
5
5
|
Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
|
6
6
|
Project-URL: repository, https://github.com/app-sre/qontract-reconcile
|
{qontract_reconcile-0.10.2.dev183.dist-info → qontract_reconcile-0.10.2.dev184.dist-info}/RECORD
RENAMED
@@ -15,7 +15,7 @@ reconcile/closedbox_endpoint_monitoring_base.py,sha256=al7m8EgnnYx90rY1REryW3byN
|
|
15
15
|
reconcile/cluster_deployment_mapper.py,sha256=5gumAaRCcFXsabUJ1dnuUy9WrP_FEEM5JnOnE8ch9sE,2326
|
16
16
|
reconcile/dashdotdb_base.py,sha256=83ZWIf5JJk3P_D69y2TmXRcQr6ELJGlv10OM0h7fJVs,4767
|
17
17
|
reconcile/dashdotdb_cso.py,sha256=QRK0YfIqO4rehs8btD3l_GXIO2ZIycTQEKEthBdB0xA,3639
|
18
|
-
reconcile/dashdotdb_dora.py,sha256=
|
18
|
+
reconcile/dashdotdb_dora.py,sha256=juZvr7R45f3TLbIZy290Oa1OtsSnqX-Ez71ppYAUjXw,17780
|
19
19
|
reconcile/dashdotdb_dvo.py,sha256=lCkZ0iby6HrNQb-3kYb6xrt8wCjVUZYxKzz9SiStfHU,8946
|
20
20
|
reconcile/dashdotdb_slo.py,sha256=TvKdMOtUZcZP9QydcUJMKh0zURHgOMN_RTpQpCkD1Z8,3960
|
21
21
|
reconcile/database_access_manager.py,sha256=Z3aAmw2LsmMIIor-bOGzziVZdVNC82Gmw8oHBUAFf-8,25577
|
@@ -26,7 +26,7 @@ reconcile/gcp_image_mirror.py,sha256=1ThuUff_04ZdF6uxcLoDuHhoNA3OIw0V-z0-CwdPE2w
|
|
26
26
|
reconcile/github_org.py,sha256=Wc5cZamatuWsW2ZJT2ib5ps8l3iY3RXHwNUxVJerqz0,14173
|
27
27
|
reconcile/github_owners.py,sha256=viE1KJ-zaTxuZ5yItg2C263J0brn-Q-3hR_DkYDMbhY,3122
|
28
28
|
reconcile/github_repo_invites.py,sha256=U9UCzNVwrZ7MqODtFah8ogH0NNY-XjBin7G9gqHtCUY,2690
|
29
|
-
reconcile/github_repo_permissions_validator.py,sha256=
|
29
|
+
reconcile/github_repo_permissions_validator.py,sha256=PNqL4dqa2OaNBy-NmLVN-t1HZa6eS6HgSYmfOunYqtA,1798
|
30
30
|
reconcile/github_users.py,sha256=QdX164LZrm8sqggMj-0beCzWofpS6OEBfzKNrWPrfj0,3934
|
31
31
|
reconcile/github_validator.py,sha256=-j17tn3csFVjPMSPL3te48iWVkPZCncRXdeKeLdGjjQ,931
|
32
32
|
reconcile/gitlab_fork_compliance.py,sha256=RbHckzLnE9zkOFHJANzoejEMMbMAivmqJVs3Suvp9lU,4591
|
@@ -102,10 +102,10 @@ reconcile/resource_template_tester.py,sha256=DsKvBuNLPxm4Fa-e1YHHySnhThm5i_j-nF3
|
|
102
102
|
reconcile/run_integration.py,sha256=8kc-lMKF9n2bvyrItJ-3nOgQMqK7lh1UAubXQrxQDPY,9711
|
103
103
|
reconcile/saas_file_validator.py,sha256=tyvFYU6lnkfDYIkAIr5pWqSvO5Yc6TagZ-quJYD2dtI,2547
|
104
104
|
reconcile/sendgrid_teammates.py,sha256=oO8QbLb4s1o8A6CGiCagN9CmS05BSS_WLztuY0Ym9D8,4773
|
105
|
-
reconcile/service_dependencies.py,sha256=
|
105
|
+
reconcile/service_dependencies.py,sha256=G2qCuYFc8wQLpRxkdhmibxSAl3nUM3hcan4x50W_mCA,4335
|
106
106
|
reconcile/signalfx_endpoint_monitoring.py,sha256=Nqgsg1cflSd2nNnm89y_e8c--7xLUqTrKOHkDs-qADE,2868
|
107
107
|
reconcile/slack_base.py,sha256=I-msunWxfgu5bSwXYulGbtLjxUB_tRmTCAUCU-3nabI,3484
|
108
|
-
reconcile/slack_usergroups.py,sha256=
|
108
|
+
reconcile/slack_usergroups.py,sha256=xFkVe67RXSUj8JvpfSFEiRdQzB0TnJJEHW_b5PEwLng,30213
|
109
109
|
reconcile/sql_query.py,sha256=OEzEZaqgv-kzG3GR2x9w3uMIfSFXP6EdhlW4u5mc1Dg,25895
|
110
110
|
reconcile/status.py,sha256=cY4IJFXemhxptRJqR4qaaOWqei9e4jgLXuVSGajMsjg,544
|
111
111
|
reconcile/status_board.py,sha256=kJ0bus_wdyX3zsFJuUPrH4n9BNG_jhDbiQ3waOLVRBE,8538
|
@@ -616,7 +616,7 @@ reconcile/utils/external_resources.py,sha256=YzTb0xAcNdmKO326mGQy7BmST56CZcdru4l
|
|
616
616
|
reconcile/utils/filtering.py,sha256=S4PbMHuFr3ED0P2Q_ea5CAaB7FimI62B-F5YTaKrphA,402
|
617
617
|
reconcile/utils/git.py,sha256=o4p9m8jlzCJDcutl2HErvGLhL6sZ1NB4Aw3zGcQIzso,2427
|
618
618
|
reconcile/utils/github_api.py,sha256=o4J0ZU1ZSr9808uoorKHv19iae-eLo85yrCZX67p2kw,2822
|
619
|
-
reconcile/utils/gitlab_api.py,sha256=
|
619
|
+
reconcile/utils/gitlab_api.py,sha256=U7l26fM1lQ4g-O06D0ZawvOXF_yzzH5NdR5iDx2ZJFA,28616
|
620
620
|
reconcile/utils/gpg.py,sha256=EKG7_fdMv8BMlV5yUdPiqoTx-KrzmVSEAl2sLkaKwWI,1123
|
621
621
|
reconcile/utils/gql.py,sha256=C0thIm_k9MBldfqwHzyqtYZk9sIvMdm9IbbnXLGwjD8,14158
|
622
622
|
reconcile/utils/grouping.py,sha256=vr9SFHZ7bqmHYrvYcEZt-Er3-yQYfAAdq5sHLZVmXPY,456
|
@@ -667,12 +667,12 @@ reconcile/utils/state.py,sha256=az4tBmZ0EdbFcAGiBVUxs3cr2-BVWsuDQiNTvjjQq8s,1637
|
|
667
667
|
reconcile/utils/structs.py,sha256=LcbLEg8WxfRqM6nW7NhcWN0YeqF7SQzxOgntmLs1SgY,352
|
668
668
|
reconcile/utils/template.py,sha256=wTvRU4AnAV_o042tD4Mwls2dwWMuk7MKnde3MaCjaYg,331
|
669
669
|
reconcile/utils/terraform_client.py,sha256=IDlrNvGEc2i6ElZIL_fzaJEad1nRC3DkP9_VXhJXmU0,37329
|
670
|
-
reconcile/utils/terrascript_aws_client.py,sha256
|
670
|
+
reconcile/utils/terrascript_aws_client.py,sha256=RkiYjRietHFNXtfA1-WEZ1lZbJFBA_XCtTOsZUij5VM,292360
|
671
671
|
reconcile/utils/three_way_diff_strategy.py,sha256=oQcHXd9LVhirJfoaOBoHUYuZVGfyL2voKr6KVI34zZE,4833
|
672
672
|
reconcile/utils/throughput.py,sha256=iP4UWAe2LVhDo69mPPmgo9nQ7RxHD6_GS8MZe-aSiuM,344
|
673
673
|
reconcile/utils/vault.py,sha256=aSA8l9cJlPUHpChFGl27nSY-Mpq9FMjBo7Dcgb1BVfM,15036
|
674
674
|
reconcile/utils/vaultsecretref.py,sha256=0KUSzuvTRxPyKY919TO3-B_eYg4_76fzKvMF8j5s1G0,911
|
675
|
-
reconcile/utils/vcs.py,sha256=
|
675
|
+
reconcile/utils/vcs.py,sha256=AK35vIjx9bXYclKmvNekpaG_OETt-ZybibwV-m123xc,10186
|
676
676
|
reconcile/utils/acs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
677
677
|
reconcile/utils/acs/base.py,sha256=4UsDrCpAOuddL3PKNuIQYoJP1BtZQNNB8_KEX0lXneg,2532
|
678
678
|
reconcile/utils/acs/notifiers.py,sha256=DlzTDM9arWQlBSiDy70y5Mf38OKVs9V0FzFe2LfOKXA,5046
|
@@ -761,7 +761,7 @@ reconcile/utils/runtime/sharding.py,sha256=r0ieUtNed7NvknSw6qQrCkKpVXE1shuHGnfFc
|
|
761
761
|
reconcile/utils/saasherder/__init__.py,sha256=3U8plqMAPRE1kjwZ5YnIsYsggTf4_gS7flRUEuXVBAs,343
|
762
762
|
reconcile/utils/saasherder/interfaces.py,sha256=NEYQspYfyWQhBeJyNCqSFbixi1A4wRVGB7FeNM5BDCk,9141
|
763
763
|
reconcile/utils/saasherder/models.py,sha256=JaOz_DEtudJZhiDe90kaBlJkppFufn81V92oK9PHYx0,10208
|
764
|
-
reconcile/utils/saasherder/saasherder.py,sha256=
|
764
|
+
reconcile/utils/saasherder/saasherder.py,sha256=V8DOaMvY1POYd_EquIlU2sI62l64C-PjtaTCuzwVbn4,87306
|
765
765
|
reconcile/utils/terraform/__init__.py,sha256=zNbiyTWo35AT1sFTElL2j_AA0jJ_yWE_bfFn-nD2xik,250
|
766
766
|
reconcile/utils/terraform/config.py,sha256=5UVrd563TMcvi4ooa5JvWVDW1I3bIWg484u79evfV_8,164
|
767
767
|
reconcile/utils/terraform/config_client.py,sha256=gRL1rQ0AqvShei_rcGqC3HDYGskOFKE1nPrJyJE9yno,4676
|
@@ -807,7 +807,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
|
|
807
807
|
tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
|
808
808
|
tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
|
809
809
|
tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
|
810
|
-
qontract_reconcile-0.10.2.
|
811
|
-
qontract_reconcile-0.10.2.
|
812
|
-
qontract_reconcile-0.10.2.
|
813
|
-
qontract_reconcile-0.10.2.
|
810
|
+
qontract_reconcile-0.10.2.dev184.dist-info/METADATA,sha256=4cOL2ogbG9Fe9GL834sDkyleO4EK2cgWR-AoP0ZIuls,24627
|
811
|
+
qontract_reconcile-0.10.2.dev184.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
812
|
+
qontract_reconcile-0.10.2.dev184.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
|
813
|
+
qontract_reconcile-0.10.2.dev184.dist-info/RECORD,,
|
reconcile/dashdotdb_dora.py
CHANGED
@@ -34,7 +34,7 @@ from reconcile.typed_queries.saas_files import get_saas_files
|
|
34
34
|
from reconcile.utils.github_api import GithubRepositoryApi
|
35
35
|
from reconcile.utils.gitlab_api import GitLabApi
|
36
36
|
from reconcile.utils.secret_reader import create_secret_reader
|
37
|
-
from reconcile.utils.vcs import
|
37
|
+
from reconcile.utils.vcs import VCS
|
38
38
|
|
39
39
|
QONTRACT_INTEGRATION = "dashdotdb-dora"
|
40
40
|
|
@@ -422,19 +422,21 @@ class DashdotdbDORA(DashdotdbBase):
|
|
422
422
|
return rc, []
|
423
423
|
|
424
424
|
LOG.info("Fetching commits %s", rc)
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
|
425
|
+
repo_info = VCS.parse_repo_url(rc.repo_url)
|
426
|
+
match repo_info.platform:
|
427
|
+
case "github":
|
428
|
+
try:
|
429
|
+
commits = self._github_compare_commits(rc, repo_info.name)
|
430
|
+
except GithubException as e:
|
431
|
+
if e.status == 404:
|
432
|
+
LOG.info(
|
433
|
+
f"Ignoring RepoChanges for {rc} because could not calculate them: {e.data['message']}"
|
434
|
+
)
|
435
|
+
return rc, []
|
436
|
+
case "gitlab":
|
437
|
+
commits = self._gitlab_compare_commits(rc, repo_info.name)
|
438
|
+
case _:
|
439
|
+
raise Exception(f"Unknown git hosting {rc.repo_url}")
|
438
440
|
|
439
441
|
return rc, commits
|
440
442
|
|
@@ -455,13 +457,10 @@ class DashdotdbDORA(DashdotdbBase):
|
|
455
457
|
|
456
458
|
return app_env, datetime.fromisoformat(response.json()["finish_timestamp"])
|
457
459
|
|
458
|
-
def _gitlab_compare_commits(self, rc: RepoChanges) -> list[Commit]:
|
460
|
+
def _gitlab_compare_commits(self, rc: RepoChanges, repo: str) -> list[Commit]:
|
459
461
|
if not rc.repo_url or not rc.ref_from or not rc.ref_to:
|
460
462
|
return []
|
461
463
|
|
462
|
-
prefix = "https://gitlab.cee.redhat.com/"
|
463
|
-
repo = rc.repo_url[rc.repo_url.startswith(prefix) and len(prefix) :]
|
464
|
-
|
465
464
|
commits = self.gl.repository_compare(repo, rc.ref_from, rc.ref_to)
|
466
465
|
|
467
466
|
return [
|
@@ -473,12 +472,10 @@ class DashdotdbDORA(DashdotdbBase):
|
|
473
472
|
for commit in commits
|
474
473
|
]
|
475
474
|
|
476
|
-
def _github_compare_commits(self, rc: RepoChanges) -> list[Commit]:
|
475
|
+
def _github_compare_commits(self, rc: RepoChanges, repo: str) -> list[Commit]:
|
477
476
|
if not rc.repo_url:
|
478
477
|
return []
|
479
478
|
|
480
|
-
repo = rc.repo_url.removeprefix(GITHUB_BASE_URL).rstrip("/")
|
481
|
-
|
482
479
|
return [
|
483
480
|
Commit(rc.repo_url, commit.sha, commit.commit.committer.date)
|
484
481
|
for commit in self.gh_api(repo).compare(rc.ref_from, rc.ref_to)
|
@@ -11,7 +11,7 @@ from reconcile.jenkins_job_builder import init_jjb
|
|
11
11
|
from reconcile.utils.jjb_client import JJB
|
12
12
|
from reconcile.utils.secret_reader import SecretReader
|
13
13
|
from reconcile.utils.semver_helper import make_semver
|
14
|
-
from reconcile.utils.vcs import
|
14
|
+
from reconcile.utils.vcs import VCS
|
15
15
|
|
16
16
|
QONTRACT_INTEGRATION = "github-repo-permissions-validator"
|
17
17
|
QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
|
@@ -46,8 +46,8 @@ def run(dry_run: bool, instance_name: str) -> None:
|
|
46
46
|
error = False
|
47
47
|
for job in pr_check_jobs:
|
48
48
|
repo_url = jjb.get_repo_url(job)
|
49
|
-
|
50
|
-
repo = gh.get_repo(
|
49
|
+
repo_info = VCS.parse_repo_url(repo_url)
|
50
|
+
repo = gh.get_repo(repo_info.name)
|
51
51
|
permissions = repo.permissions
|
52
52
|
if not permissions.push and repo_url not in invitations:
|
53
53
|
logging.error(f"missing write permissions for bot in repo {repo_url}")
|
@@ -16,7 +16,7 @@ from reconcile.gql_definitions.service_dependencies.service_dependencies import
|
|
16
16
|
SaasResourceTemplateV2,
|
17
17
|
)
|
18
18
|
from reconcile.utils import gql
|
19
|
-
from reconcile.utils.vcs import
|
19
|
+
from reconcile.utils.vcs import VCS
|
20
20
|
|
21
21
|
QONTRACT_INTEGRATION = "service-dependencies"
|
22
22
|
|
@@ -32,15 +32,13 @@ def get_desired_dependency_names(
|
|
32
32
|
required_dep_names = set()
|
33
33
|
|
34
34
|
code_components: list[AppCodeComponentsV1] = app.code_components or []
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
if github_urls:
|
43
|
-
required_dep_names.update(get_dependency_names(dependency_map, "github"))
|
35
|
+
code_component_platforms = {
|
36
|
+
platform
|
37
|
+
for cc in code_components
|
38
|
+
if (platform := VCS.parse_repo_url(cc.url).platform)
|
39
|
+
}
|
40
|
+
for platform in code_component_platforms:
|
41
|
+
required_dep_names.update(get_dependency_names(dependency_map, platform))
|
44
42
|
|
45
43
|
jenkins_configs: list[JenkinsConfigV1] = app.jenkins_configs or []
|
46
44
|
if jenkins_configs:
|
reconcile/slack_usergroups.py
CHANGED
@@ -10,7 +10,6 @@ from typing import (
|
|
10
10
|
Any,
|
11
11
|
TypedDict,
|
12
12
|
)
|
13
|
-
from urllib.parse import urlparse
|
14
13
|
|
15
14
|
from github.GithubException import UnknownObjectException
|
16
15
|
from pydantic import BaseModel
|
@@ -67,6 +66,7 @@ from reconcile.utils.slack_api import (
|
|
67
66
|
SlackApiError,
|
68
67
|
UsergroupNotFoundException,
|
69
68
|
)
|
69
|
+
from reconcile.utils.vcs import VCS
|
70
70
|
|
71
71
|
DATE_FORMAT = "%Y-%m-%d %H:%M"
|
72
72
|
QONTRACT_INTEGRATION = "slack-usergroups"
|
@@ -77,25 +77,23 @@ error_occurred = False
|
|
77
77
|
|
78
78
|
def get_git_api(url: str) -> GithubRepositoryApi | GitLabApi:
|
79
79
|
"""Return GitHub/GitLab API based on url."""
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
if "github" in parsed_url.hostname:
|
80
|
+
repo_info = VCS.parse_repo_url(url)
|
81
|
+
match repo_info.platform:
|
82
|
+
case "github":
|
84
83
|
vault_settings = get_app_interface_vault_settings()
|
85
84
|
secret_reader = create_secret_reader(use_vault=vault_settings.vault)
|
86
85
|
instance = queries.get_github_instance()
|
87
86
|
token = secret_reader.read(instance["token"])
|
88
|
-
|
89
87
|
return GithubRepositoryApi(
|
90
88
|
repo_url=url,
|
91
89
|
token=token,
|
92
90
|
)
|
93
|
-
|
91
|
+
case "gitlab":
|
94
92
|
settings = queries.get_app_interface_settings()
|
95
93
|
instance = queries.get_gitlab_instance()
|
96
94
|
return GitLabApi(instance=instance, project_url=url, settings=settings)
|
97
|
-
|
98
|
-
|
95
|
+
case _:
|
96
|
+
raise ValueError(f"Unable to handle URL: {url}")
|
99
97
|
|
100
98
|
|
101
99
|
class SlackObject(BaseModel):
|
reconcile/utils/gitlab_api.py
CHANGED
@@ -388,7 +388,7 @@ class GitLabApi:
|
|
388
388
|
|
389
389
|
@retry()
|
390
390
|
def get_project(self, repo_url: str) -> Project | None:
|
391
|
-
repo = repo_url.
|
391
|
+
repo = repo_url.removeprefix(self.server).strip("/")
|
392
392
|
try:
|
393
393
|
project = self.gl.projects.get(repo)
|
394
394
|
except GitlabGetError:
|
@@ -20,7 +20,6 @@ from contextlib import suppress
|
|
20
20
|
from datetime import UTC, datetime, timedelta
|
21
21
|
from types import TracebackType
|
22
22
|
from typing import Any
|
23
|
-
from urllib.parse import urlparse
|
24
23
|
|
25
24
|
import yaml
|
26
25
|
from github import (
|
@@ -83,7 +82,7 @@ from reconcile.utils.saasherder.models import (
|
|
83
82
|
)
|
84
83
|
from reconcile.utils.secret_reader import SecretReaderBase
|
85
84
|
from reconcile.utils.state import State
|
86
|
-
from reconcile.utils.vcs import
|
85
|
+
from reconcile.utils.vcs import VCS
|
87
86
|
|
88
87
|
TARGET_CONFIG_HASH = "target_config_hash"
|
89
88
|
|
@@ -741,17 +740,21 @@ class SaasHerder: # pylint: disable=too-many-public-methods
|
|
741
740
|
trigger_reason: str,
|
742
741
|
) -> tuple[str, str]:
|
743
742
|
[url, sha] = trigger_reason.split(" ")[0].split("/commit/")
|
744
|
-
|
743
|
+
repo_info = VCS.parse_repo_url(url)
|
744
|
+
repo_name = repo_info.name
|
745
745
|
file_name = f"{repo_name.replace('/', '-')}-{sha}.tar.gz"
|
746
|
-
|
747
|
-
|
748
|
-
|
749
|
-
|
750
|
-
|
751
|
-
|
752
|
-
|
753
|
-
|
754
|
-
|
746
|
+
match repo_info.platform:
|
747
|
+
case "github":
|
748
|
+
github = self._initiate_github(
|
749
|
+
saas_file, base_url="https://api.github.com"
|
750
|
+
)
|
751
|
+
repo = github.get_repo(repo_name)
|
752
|
+
# get_archive_link get redirect url form header, it does not work with github-mirror
|
753
|
+
archive_url = repo.get_archive_link("tarball", ref=sha)
|
754
|
+
case "gitlab":
|
755
|
+
archive_url = f"{url}/-/archive/{sha}/{file_name}"
|
756
|
+
case _:
|
757
|
+
raise Exception(f"Only GitHub and GitLab are supported: {url}")
|
755
758
|
|
756
759
|
return file_name, archive_url
|
757
760
|
|
@@ -761,18 +764,19 @@ class SaasHerder: # pylint: disable=too-many-public-methods
|
|
761
764
|
) -> tuple[Any, str]:
|
762
765
|
commit_sha = self._get_commit_sha(url, ref, github)
|
763
766
|
|
764
|
-
|
765
|
-
|
766
|
-
|
767
|
-
|
768
|
-
|
769
|
-
|
770
|
-
|
771
|
-
|
772
|
-
|
773
|
-
|
774
|
-
|
775
|
-
|
767
|
+
repo_info = VCS.parse_repo_url(url)
|
768
|
+
match repo_info.platform:
|
769
|
+
case "github":
|
770
|
+
repo = github.get_repo(repo_info.name)
|
771
|
+
content = self._get_file_contents_github(repo, path, commit_sha)
|
772
|
+
case "gitlab":
|
773
|
+
if not self.gitlab:
|
774
|
+
raise Exception("gitlab is not initialized")
|
775
|
+
project = self.gitlab.get_project(url)
|
776
|
+
f = project.files.get(file_path=path.lstrip("/"), ref=commit_sha)
|
777
|
+
content = f.decode()
|
778
|
+
case _:
|
779
|
+
raise Exception(f"Only GitHub and GitLab are supported: {url}")
|
776
780
|
|
777
781
|
return yaml.safe_load(content), commit_sha
|
778
782
|
|
@@ -782,52 +786,53 @@ class SaasHerder: # pylint: disable=too-many-public-methods
|
|
782
786
|
) -> tuple[list[Any], str]:
|
783
787
|
commit_sha = self._get_commit_sha(url, ref, github)
|
784
788
|
resources: list[Any] = []
|
785
|
-
|
786
|
-
|
787
|
-
|
788
|
-
|
789
|
-
|
790
|
-
|
791
|
-
|
792
|
-
|
793
|
-
|
794
|
-
|
789
|
+
repo_info = VCS.parse_repo_url(url)
|
790
|
+
match repo_info.platform:
|
791
|
+
case "github":
|
792
|
+
repo = github.get_repo(repo_info.name)
|
793
|
+
directory = repo.get_contents(path, commit_sha)
|
794
|
+
if isinstance(directory, ContentFile):
|
795
|
+
raise Exception(f"Path {path} and sha {commit_sha} is a file!")
|
796
|
+
for f in directory:
|
797
|
+
file_path = os.path.join(path, f.name)
|
798
|
+
file_contents_decoded = self._get_file_contents_github(
|
799
|
+
repo, file_path, commit_sha
|
800
|
+
)
|
801
|
+
result_resources = yaml.safe_load_all(file_contents_decoded)
|
802
|
+
resources.extend(result_resources)
|
803
|
+
case "gitlab":
|
804
|
+
if not self.gitlab:
|
805
|
+
raise Exception("gitlab is not initialized")
|
806
|
+
project = self.gitlab.get_project(url)
|
807
|
+
dir_contents = self.gitlab.get_directory_contents(
|
808
|
+
project,
|
809
|
+
ref=commit_sha,
|
810
|
+
path=path,
|
795
811
|
)
|
796
|
-
|
797
|
-
|
798
|
-
|
799
|
-
|
800
|
-
raise Exception("
|
801
|
-
project = self.gitlab.get_project(url)
|
802
|
-
dir_contents = self.gitlab.get_directory_contents(
|
803
|
-
project,
|
804
|
-
ref=commit_sha,
|
805
|
-
path=path,
|
806
|
-
)
|
807
|
-
for content in dir_contents.values():
|
808
|
-
result_resources = yaml.safe_load_all(content)
|
809
|
-
resources.extend(result_resources)
|
810
|
-
else:
|
811
|
-
raise Exception(f"Only GitHub and GitLab are supported: {url}")
|
812
|
+
for content in dir_contents.values():
|
813
|
+
result_resources = yaml.safe_load_all(content)
|
814
|
+
resources.extend(result_resources)
|
815
|
+
case _:
|
816
|
+
raise Exception(f"Only GitHub and GitLab are supported: {url}")
|
812
817
|
|
813
818
|
return resources, commit_sha
|
814
819
|
|
815
820
|
@retry()
|
816
821
|
def _get_commit_sha(self, url: str, ref: str, github: Github) -> str:
|
817
|
-
|
818
|
-
|
819
|
-
|
820
|
-
|
821
|
-
|
822
|
-
|
823
|
-
|
824
|
-
|
825
|
-
|
826
|
-
|
827
|
-
|
828
|
-
|
829
|
-
|
830
|
-
|
822
|
+
repo_info = VCS.parse_repo_url(url)
|
823
|
+
match repo_info.platform:
|
824
|
+
case "github":
|
825
|
+
repo = github.get_repo(repo_info.name)
|
826
|
+
commit = repo.get_commit(sha=ref)
|
827
|
+
return commit.sha
|
828
|
+
case "gitlab":
|
829
|
+
if not self.gitlab:
|
830
|
+
raise Exception("gitlab is not initialized")
|
831
|
+
project = self.gitlab.get_project(url)
|
832
|
+
commits = project.commits.list(ref_name=ref, per_page=1, page=1)
|
833
|
+
return commits[0].id
|
834
|
+
case _:
|
835
|
+
return ""
|
831
836
|
|
832
837
|
@staticmethod
|
833
838
|
def _additional_resource_process(resources: Resources, html_url: str) -> None:
|
@@ -186,7 +186,7 @@ from reconcile.utils.password_validator import (
|
|
186
186
|
)
|
187
187
|
from reconcile.utils.secret_reader import SecretReader, SecretReaderBase
|
188
188
|
from reconcile.utils.terraform import safe_resource_id
|
189
|
-
from reconcile.utils.vcs import
|
189
|
+
from reconcile.utils.vcs import VCS
|
190
190
|
|
191
191
|
GH_BASE_URL = os.environ.get("GITHUB_API", "https://api.github.com")
|
192
192
|
LOGTOES_RELEASE = "repos/app-sre/logs-to-elasticsearch-lambda/releases/latest"
|
@@ -5686,20 +5686,20 @@ class TerrascriptClient: # pylint: disable=too-many-public-methods
|
|
5686
5686
|
return ref
|
5687
5687
|
|
5688
5688
|
# get commit_sha from branch
|
5689
|
-
|
5690
|
-
|
5691
|
-
|
5692
|
-
|
5693
|
-
|
5694
|
-
|
5695
|
-
|
5696
|
-
|
5697
|
-
|
5698
|
-
|
5699
|
-
|
5700
|
-
|
5701
|
-
|
5702
|
-
|
5689
|
+
repo_url_info = VCS.parse_repo_url(url)
|
5690
|
+
match repo_url_info.platform:
|
5691
|
+
case "github":
|
5692
|
+
github = self.init_github()
|
5693
|
+
repo = github.get_repo(repo_url_info.name)
|
5694
|
+
commit = repo.get_commit(sha=ref)
|
5695
|
+
return commit.sha
|
5696
|
+
case "gitlab":
|
5697
|
+
gitlab = self.init_gitlab()
|
5698
|
+
project = gitlab.get_project(url)
|
5699
|
+
commits = project.commits.list(ref_name=ref, per_page=1, page=1)
|
5700
|
+
return commits[0].id
|
5701
|
+
case _:
|
5702
|
+
return ""
|
5703
5703
|
|
5704
5704
|
def get_asg_image_id(
|
5705
5705
|
self, filters: Iterable[Mapping[str, Any]], account: str, region: str
|
reconcile/utils/vcs.py
CHANGED
@@ -6,6 +6,8 @@ from collections.abc import Iterable
|
|
6
6
|
from dataclasses import dataclass
|
7
7
|
from datetime import datetime
|
8
8
|
from enum import Enum
|
9
|
+
from typing import Literal
|
10
|
+
from urllib.parse import urlparse
|
9
11
|
|
10
12
|
from gitlab.v4.objects import ProjectMergeRequest
|
11
13
|
|
@@ -24,6 +26,16 @@ from reconcile.utils.secret_reader import (
|
|
24
26
|
|
25
27
|
GITHUB_BASE_URL = "https://github.com/"
|
26
28
|
|
29
|
+
Platform = Literal["github", "gitlab"]
|
30
|
+
|
31
|
+
SUPPORTED_PLATFORMS: list[Platform] = ["github", "gitlab"]
|
32
|
+
|
33
|
+
|
34
|
+
@dataclass(frozen=True)
|
35
|
+
class RepoInfo:
|
36
|
+
platform: Platform | None
|
37
|
+
name: str
|
38
|
+
|
27
39
|
|
28
40
|
class MRCheckStatus(Enum):
|
29
41
|
NONE = 0
|
@@ -155,16 +167,40 @@ class VCS:
|
|
155
167
|
# Lets assume all other states as non-present
|
156
168
|
return MRCheckStatus.NONE
|
157
169
|
|
170
|
+
@staticmethod
|
171
|
+
def parse_repo_url(url: str) -> RepoInfo:
|
172
|
+
"""
|
173
|
+
Parse a repository URL and return a RepoInfo object.
|
174
|
+
`platform` can be 'github', 'gitlab' or None if not recognized,
|
175
|
+
it's inferred from the URL host.
|
176
|
+
`name` is the path part of the URL, stripped of leading and trailing slashes.
|
177
|
+
"""
|
178
|
+
parsed_url = urlparse(url)
|
179
|
+
platform = next(
|
180
|
+
(
|
181
|
+
p
|
182
|
+
for p in SUPPORTED_PLATFORMS
|
183
|
+
if (hostname := parsed_url.hostname) and p in hostname
|
184
|
+
),
|
185
|
+
None,
|
186
|
+
)
|
187
|
+
name = parsed_url.path.strip("/").removesuffix(".git")
|
188
|
+
return RepoInfo(platform=platform, name=name)
|
189
|
+
|
158
190
|
def get_commit_sha(
|
159
191
|
self, repo_url: str, ref: str, auth_code: HasSecret | None
|
160
192
|
) -> str:
|
161
193
|
if bool(self._is_commit_sha_regex.search(ref)):
|
162
194
|
return ref
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
195
|
+
repo_info = self.parse_repo_url(repo_url)
|
196
|
+
match repo_info.platform:
|
197
|
+
case "github":
|
198
|
+
github = self._init_github(repo_url=repo_url, auth_code=auth_code)
|
199
|
+
return github.get_commit_sha(ref=ref)
|
200
|
+
case "gitlab":
|
201
|
+
return self._gitlab_instance.get_commit_sha(ref=ref, repo_url=repo_url)
|
202
|
+
case _:
|
203
|
+
raise ValueError(f"Unsupported repository URL: {repo_url}")
|
168
204
|
|
169
205
|
def get_commits_between(
|
170
206
|
self,
|
@@ -177,30 +213,33 @@ class VCS:
|
|
177
213
|
Return a list of commits between two commits.
|
178
214
|
Note, that the commit_to is included in the result list, whereas commit_from is not included.
|
179
215
|
"""
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
216
|
+
repo_info = self.parse_repo_url(repo_url)
|
217
|
+
match repo_info.platform:
|
218
|
+
case "github":
|
219
|
+
github = self._init_github(repo_url=repo_url, auth_code=auth_code)
|
220
|
+
data = github.compare(commit_from=commit_from, commit_to=commit_to)
|
221
|
+
return [
|
222
|
+
Commit(
|
223
|
+
repo=repo_url,
|
224
|
+
sha=gh_commit.sha,
|
225
|
+
date=gh_commit.commit.committer.date,
|
226
|
+
)
|
227
|
+
for gh_commit in data
|
228
|
+
]
|
229
|
+
case "gitlab":
|
230
|
+
data = self._gitlab_instance.repository_compare(
|
231
|
+
repo_url=repo_url, ref_from=commit_from, ref_to=commit_to
|
188
232
|
)
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
sha=gl_commit["id"],
|
200
|
-
date=datetime.fromisoformat(gl_commit["committed_date"]),
|
201
|
-
)
|
202
|
-
for gl_commit in data
|
203
|
-
]
|
233
|
+
return [
|
234
|
+
Commit(
|
235
|
+
repo=repo_url,
|
236
|
+
sha=gl_commit["id"],
|
237
|
+
date=datetime.fromisoformat(gl_commit["committed_date"]),
|
238
|
+
)
|
239
|
+
for gl_commit in data
|
240
|
+
]
|
241
|
+
case _:
|
242
|
+
raise ValueError(f"Unsupported repository URL: {repo_url}")
|
204
243
|
|
205
244
|
def close_app_interface_mr(self, mr: ProjectMergeRequest, comment: str) -> None:
|
206
245
|
if not self._allow_deleting_mrs:
|
{qontract_reconcile-0.10.2.dev183.dist-info → qontract_reconcile-0.10.2.dev184.dist-info}/WHEEL
RENAMED
File without changes
|
File without changes
|