recce-nightly 1.10.0.20250629__py3-none-any.whl → 1.25.0.20251112a20664__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. recce/VERSION +1 -1
  2. recce/__init__.py +5 -0
  3. recce/adapter/dbt_adapter/__init__.py +116 -74
  4. recce/artifact.py +76 -3
  5. recce/cli.py +665 -69
  6. recce/config.py +2 -2
  7. recce/connect_to_cloud.py +1 -1
  8. recce/core.py +3 -3
  9. recce/data/404.html +1 -22
  10. recce/data/__next.__PAGE__.txt +10 -0
  11. recce/data/__next._full.txt +23 -0
  12. recce/data/__next._index.txt +8 -0
  13. recce/data/__next._tree.txt +12 -0
  14. recce/data/_next/static/JwV_pqetN5WamZZ7aGdfH/_buildManifest.js +11 -0
  15. recce/data/_next/static/JwV_pqetN5WamZZ7aGdfH/_clientMiddlewareManifest.json +1 -0
  16. recce/data/_next/static/chunks/0a2b2dd4b57049c2.js +1 -0
  17. recce/data/_next/static/chunks/19c10d219a6a21ff.js +1 -0
  18. recce/data/_next/static/chunks/24fd885c7180a612.js +1 -0
  19. recce/data/_next/static/chunks/27e66b2eab4adc32.js +19 -0
  20. recce/data/_next/static/chunks/67b1c6a62f19d429.js +110 -0
  21. recce/data/_next/static/chunks/71f88fcc615bf282.js +1 -0
  22. recce/data/_next/static/chunks/917619ab62a32388.js +1 -0
  23. recce/data/_next/static/chunks/93ba5a62932b704f.js +4 -0
  24. recce/data/_next/static/chunks/a43a2a5e06d5a92b.js +1 -0
  25. recce/data/_next/static/chunks/a6c78b24bd8b84fc.js +1 -0
  26. recce/data/_next/static/chunks/ba2d87265a68599d.css +2 -0
  27. recce/data/_next/static/chunks/c117fd1c1382dd83.js +11 -0
  28. recce/data/_next/static/chunks/c9425ca46eebdde9.js +1 -0
  29. recce/data/_next/static/chunks/cc8a9eadba012be0.css +6 -0
  30. recce/data/_next/static/chunks/e124bccf574a3361.css +1 -0
  31. recce/data/_next/static/chunks/e392ad92847c3e17.js +1 -0
  32. recce/data/_next/static/chunks/e4ce95efe88dae79.js +11 -0
  33. recce/data/_next/static/chunks/e69c777814fea6ed.js +2 -0
  34. recce/data/_next/static/chunks/turbopack-21cfd73037ff57ab.js +3 -0
  35. recce/data/_next/static/media/favicon.a8d38d84.ico +0 -0
  36. recce/data/_next/static/media/montserrat-cyrillic-800-normal.d80d830d.woff2 +0 -0
  37. recce/data/_next/static/media/{montserrat-cyrillic-800-normal.bd5c9f50.woff → montserrat-cyrillic-800-normal.f9d58125.woff} +0 -0
  38. recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.076c2a93.woff2 +0 -0
  39. recce/data/_next/static/media/montserrat-latin-800-normal.cde454cc.woff2 +0 -0
  40. recce/data/_next/static/media/{montserrat-latin-800-normal.fc315020.woff → montserrat-latin-800-normal.d5761935.woff} +0 -0
  41. recce/data/_next/static/media/montserrat-latin-ext-800-normal.40ec0659.woff2 +0 -0
  42. recce/data/_next/static/media/{montserrat-latin-ext-800-normal.2e5381b2.woff → montserrat-latin-ext-800-normal.b671449b.woff} +0 -0
  43. recce/data/_next/static/media/{montserrat-vietnamese-800-normal.20c545e6.woff → montserrat-vietnamese-800-normal.9f7b8541.woff} +0 -0
  44. recce/data/_next/static/media/montserrat-vietnamese-800-normal.f9eb854e.woff2 +0 -0
  45. recce/data/_not-found/__next._full.txt +17 -0
  46. recce/data/_not-found/__next._index.txt +8 -0
  47. recce/data/_not-found/__next._not-found.__PAGE__.txt +5 -0
  48. recce/data/_not-found/__next._not-found.txt +4 -0
  49. recce/data/_not-found/__next._tree.txt +10 -0
  50. recce/data/_not-found.html +1 -0
  51. recce/data/_not-found.txt +17 -0
  52. recce/data/auth_callback.html +1 -1
  53. recce/data/index.html +1 -27
  54. recce/data/index.txt +23 -8
  55. recce/event/__init__.py +9 -8
  56. recce/event/collector.py +6 -2
  57. recce/event/track.py +10 -0
  58. recce/github.py +1 -1
  59. recce/mcp_server.py +632 -0
  60. recce/models/types.py +23 -2
  61. recce/pull_request.py +1 -1
  62. recce/run.py +23 -16
  63. recce/server.py +165 -11
  64. recce/state/__init__.py +31 -0
  65. recce/state/cloud.py +632 -0
  66. recce/state/const.py +26 -0
  67. recce/state/local.py +56 -0
  68. recce/state/state.py +119 -0
  69. recce/state/state_loader.py +174 -0
  70. recce/summary.py +2 -1
  71. recce/tasks/dataframe.py +59 -2
  72. recce/tasks/rowcount.py +4 -1
  73. recce/tasks/schema.py +4 -1
  74. recce/tasks/valuediff.py +1 -1
  75. recce/util/api_token.py +11 -2
  76. recce/util/breaking.py +9 -0
  77. recce/util/cll.py +1 -2
  78. recce/util/io.py +2 -2
  79. recce/util/lineage.py +14 -18
  80. recce/util/perf_tracking.py +85 -0
  81. recce/util/recce_cloud.py +229 -5
  82. recce/yaml/__init__.py +2 -2
  83. recce_cloud/__init__.py +15 -0
  84. recce_cloud/api/__init__.py +17 -0
  85. recce_cloud/api/base.py +104 -0
  86. recce_cloud/api/client.py +150 -0
  87. recce_cloud/api/exceptions.py +26 -0
  88. recce_cloud/api/factory.py +63 -0
  89. recce_cloud/api/github.py +72 -0
  90. recce_cloud/api/gitlab.py +78 -0
  91. recce_cloud/artifact.py +57 -0
  92. recce_cloud/ci_providers/__init__.py +9 -0
  93. recce_cloud/ci_providers/base.py +82 -0
  94. recce_cloud/ci_providers/detector.py +147 -0
  95. recce_cloud/ci_providers/github_actions.py +136 -0
  96. recce_cloud/ci_providers/gitlab_ci.py +130 -0
  97. recce_cloud/cli.py +303 -0
  98. recce_cloud/upload.py +213 -0
  99. {recce_nightly-1.10.0.20250629.dist-info → recce_nightly-1.25.0.20251112a20664.dist-info}/METADATA +31 -27
  100. recce_nightly-1.25.0.20251112a20664.dist-info/RECORD +178 -0
  101. {recce_nightly-1.10.0.20250629.dist-info → recce_nightly-1.25.0.20251112a20664.dist-info}/top_level.txt +1 -0
  102. tests/adapter/dbt_adapter/test_dbt_cll.py +68 -17
  103. tests/recce_cloud/__init__.py +0 -0
  104. tests/recce_cloud/test_ci_providers.py +351 -0
  105. tests/recce_cloud/test_cli.py +372 -0
  106. tests/recce_cloud/test_client.py +273 -0
  107. tests/recce_cloud/test_platform_clients.py +279 -0
  108. tests/test_cli.py +106 -3
  109. tests/test_cli_mcp_optional.py +45 -0
  110. tests/test_cloud_listing_cli.py +324 -0
  111. tests/test_core.py +147 -0
  112. tests/test_mcp_server.py +332 -0
  113. tests/test_server.py +6 -6
  114. tests/test_summary.py +14 -6
  115. recce/data/_next/static/Mrb9CZ3toH6Q8xrzNzCrg/_buildManifest.js +0 -1
  116. recce/data/_next/static/chunks/181-acc61ddada3bc0ca.js +0 -43
  117. recce/data/_next/static/chunks/1bff33f1-1ef85cf5e658a751.js +0 -1
  118. recce/data/_next/static/chunks/217-879a84d70f7a907c.js +0 -2
  119. recce/data/_next/static/chunks/29e3cc0d-60045b2e47aa3916.js +0 -1
  120. recce/data/_next/static/chunks/36e1c10d-8e7be4a6c1f6ab2d.js +0 -1
  121. recce/data/_next/static/chunks/3998a672-03adacad07b346ac.js +0 -1
  122. recce/data/_next/static/chunks/3a92ee20-1081c360214f9602.js +0 -1
  123. recce/data/_next/static/chunks/41-f30276c289169376.js +0 -9
  124. recce/data/_next/static/chunks/450c323b-fd94e7ffaa4a5efa.js +0 -1
  125. recce/data/_next/static/chunks/47d8844f-929aed9b1c73a905.js +0 -1
  126. recce/data/_next/static/chunks/608-3b079b544e5d5f5e.js +0 -15
  127. recce/data/_next/static/chunks/6dc81886-adbfa45836061d79.js +0 -1
  128. recce/data/_next/static/chunks/7a8a3e83-edf6dc64b5d5f0a5.js +0 -1
  129. recce/data/_next/static/chunks/7f27ae6c-d5f0438edd5c2a5b.js +0 -1
  130. recce/data/_next/static/chunks/86730205-cfb14e3f051bab35.js +0 -1
  131. recce/data/_next/static/chunks/8d700b6a.8bb140898499c512.js +0 -1
  132. recce/data/_next/static/chunks/92-68460b15fe448f33.js +0 -1
  133. recce/data/_next/static/chunks/9746af58-a42b7d169cacadf0.js +0 -1
  134. recce/data/_next/static/chunks/a30376cd-de84559016d7e133.js +0 -1
  135. recce/data/_next/static/chunks/app/_not-found/page-01ed58b7f971d311.js +0 -1
  136. recce/data/_next/static/chunks/app/layout-292f035bb0d2a98e.js +0 -1
  137. recce/data/_next/static/chunks/app/page-598f8acc82179d01.js +0 -1
  138. recce/data/_next/static/chunks/b63b1b3f-4282bdcf459e075c.js +0 -1
  139. recce/data/_next/static/chunks/bbda5537-9ec25eb1dd62348a.js +0 -1
  140. recce/data/_next/static/chunks/c132bf7d-08cb668a789d6afd.js +0 -1
  141. recce/data/_next/static/chunks/ce84277d-2e5d1d46910cf052.js +0 -1
  142. recce/data/_next/static/chunks/febdd86e-c6b525341634b860.js +0 -54
  143. recce/data/_next/static/chunks/fee69bc6-2dbccaf9b90474e6.js +0 -1
  144. recce/data/_next/static/chunks/framework-ded83d71b51ce901.js +0 -1
  145. recce/data/_next/static/chunks/main-app-39061b0166c47f55.js +0 -1
  146. recce/data/_next/static/chunks/main-b5b3ae20a1405261.js +0 -1
  147. recce/data/_next/static/chunks/pages/_app-437c455677d62394.js +0 -1
  148. recce/data/_next/static/chunks/pages/_error-e7650df18ca04bde.js +0 -1
  149. recce/data/_next/static/chunks/webpack-7b49d5ba7e3a434d.js +0 -1
  150. recce/data/_next/static/css/17a96168e3a9db13.css +0 -1
  151. recce/data/_next/static/css/35c6679a098e1e34.css +0 -1
  152. recce/data/_next/static/css/951e2e0eea2d4a5b.css +0 -14
  153. recce/data/_next/static/css/a2b12b4ba4227f0a.css +0 -3
  154. recce/data/_next/static/media/montserrat-cyrillic-800-normal.22628180.woff2 +0 -0
  155. recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.94a63aea.woff2 +0 -0
  156. recce/data/_next/static/media/montserrat-latin-800-normal.6f8fa298.woff2 +0 -0
  157. recce/data/_next/static/media/montserrat-latin-ext-800-normal.013b84f9.woff2 +0 -0
  158. recce/data/_next/static/media/montserrat-vietnamese-800-normal.c0035377.woff2 +0 -0
  159. recce/state.py +0 -786
  160. recce_nightly-1.10.0.20250629.dist-info/RECORD +0 -154
  161. tests/test_state.py +0 -134
  162. /recce/data/_next/static/{Mrb9CZ3toH6Q8xrzNzCrg → JwV_pqetN5WamZZ7aGdfH}/_ssgManifest.js +0 -0
  163. /recce/data/_next/static/chunks/{polyfills-42372ed130431b0a.js → a6dad97d9634a72d.js} +0 -0
  164. /recce/data/_next/static/media/{montserrat-cyrillic-ext-800-normal.e6e0d8d0.woff → montserrat-cyrillic-ext-800-normal.a4fa76b5.woff} +0 -0
  165. /recce/data/_next/static/media/{reload-image.79aabb7d.svg → reload-image.7aa931c7.svg} +0 -0
  166. {recce_nightly-1.10.0.20250629.dist-info → recce_nightly-1.25.0.20251112a20664.dist-info}/WHEEL +0 -0
  167. {recce_nightly-1.10.0.20250629.dist-info → recce_nightly-1.25.0.20251112a20664.dist-info}/entry_points.txt +0 -0
  168. {recce_nightly-1.10.0.20250629.dist-info → recce_nightly-1.25.0.20251112a20664.dist-info}/licenses/LICENSE +0 -0
recce/state.py DELETED
@@ -1,786 +0,0 @@
1
- """Define the type to serialize/de-serialize the state of the recce instance."""
2
-
3
- import json
4
- import logging
5
- import os
6
- import threading
7
- import time
8
- from base64 import b64encode
9
- from dataclasses import dataclass
10
- from datetime import datetime
11
- from hashlib import md5, sha256
12
- from typing import Dict, List, Optional, Tuple, Union
13
- from urllib.parse import urlencode
14
-
15
- import botocore.exceptions
16
- from pydantic import BaseModel, Field
17
-
18
- from recce import get_version
19
- from recce.event import get_recce_api_token
20
- from recce.git import current_branch
21
- from recce.models import CheckDAO
22
- from recce.models.types import Check, Run
23
- from recce.pull_request import PullRequestInfo, fetch_pr_metadata
24
- from recce.util.io import SupportedFileTypes, file_io_factory
25
- from recce.util.pydantic_model import pydantic_model_dump, pydantic_model_json_dump
26
- from recce.util.recce_cloud import PresignedUrlMethod, RecceCloud, RecceCloudException
27
-
28
- logger = logging.getLogger("uvicorn")
29
-
30
- RECCE_STATE_FILE = "recce-state.json"
31
- RECCE_STATE_COMPRESSED_FILE = f"{RECCE_STATE_FILE}.gz"
32
-
33
-
34
- @dataclass
35
- class ErrorMessage:
36
- error_message: str
37
- hint_message: str
38
-
39
-
40
- RECCE_CLOUD_TOKEN_MISSING = ErrorMessage(
41
- error_message="No GitHub token is provided to access the pull request information",
42
- hint_message="Please provide a GitHub token in the command argument",
43
- )
44
-
45
- RECCE_CLOUD_PASSWORD_MISSING = ErrorMessage(
46
- error_message="No password provided to access the state file in Recce Cloud",
47
- hint_message='Please provide a password with the option "--password <compress-password>"',
48
- )
49
-
50
- RECCE_API_TOKEN_MISSING = ErrorMessage(
51
- error_message="No Recc API token is provided",
52
- hint_message="Please login to Recce Cloud and copy the API token from the settings page",
53
- )
54
-
55
-
56
- def s3_sse_c_headers(password: str) -> Dict[str, str]:
57
- hashed_password = sha256()
58
- md5_hash = md5()
59
- hashed_password.update(password.encode())
60
- md5_hash.update(hashed_password.digest())
61
- encoded_passwd = b64encode(hashed_password.digest()).decode("utf-8")
62
- encoded_md5 = b64encode(md5_hash.digest()).decode("utf-8")
63
- return {
64
- "x-amz-server-side-encryption-customer-algorithm": "AES256",
65
- "x-amz-server-side-encryption-customer-key": encoded_passwd,
66
- "x-amz-server-side-encryption-customer-key-MD5": encoded_md5,
67
- }
68
-
69
-
70
- def check_s3_bucket(bucket_name: str):
71
- import boto3
72
-
73
- s3_client = boto3.client("s3")
74
- try:
75
- s3_client.head_bucket(Bucket=bucket_name)
76
- except botocore.exceptions.ClientError as e:
77
- error_code = e.response["Error"]["Code"]
78
- if error_code == "404":
79
- return False, f"Bucket '{bucket_name}' does not exist."
80
- elif error_code == "403":
81
- return False, f"Bucket '{bucket_name}' exists but you do not have permission to access it."
82
- else:
83
- return False, f"Failed to access the S3 bucket: '{bucket_name}'"
84
- return True, None
85
-
86
-
87
- class GitRepoInfo(BaseModel):
88
- branch: Optional[str] = None
89
-
90
- @staticmethod
91
- def from_current_repositroy():
92
- branch = current_branch()
93
- if branch is None:
94
- return None
95
-
96
- return GitRepoInfo(branch=branch)
97
-
98
- def to_dict(self):
99
- return pydantic_model_dump(self)
100
-
101
-
102
- class RecceStateMetadata(BaseModel):
103
- schema_version: str = "v0"
104
- recce_version: str = Field(default_factory=lambda: get_version())
105
- generated_at: str = Field(default_factory=lambda: datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"))
106
-
107
-
108
- class ArtifactsRoot(BaseModel):
109
- """
110
- Root of the artifacts.
111
-
112
- base: artifacts of the base env. key is file name, value is dict
113
- current: artifacts of the current env. key is file name, value is dict
114
- """
115
-
116
- base: Dict[str, Optional[dict]] = {}
117
- current: Dict[str, Optional[dict]] = {}
118
-
119
-
120
- class RecceState(BaseModel):
121
- metadata: Optional[RecceStateMetadata] = None
122
- runs: Optional[List[Run]] = Field(default_factory=list)
123
- checks: Optional[List[Check]] = Field(default_factory=list)
124
- artifacts: ArtifactsRoot = ArtifactsRoot(base={}, current={})
125
- git: Optional[GitRepoInfo] = None
126
- pull_request: Optional[PullRequestInfo] = None
127
-
128
- @staticmethod
129
- def from_json(json_content: str):
130
- dict_data = json.loads(json_content)
131
- state = RecceState(**dict_data)
132
- metadata = state.metadata
133
- if metadata:
134
- if metadata.schema_version is None:
135
- pass
136
- if metadata.schema_version == "v0":
137
- pass
138
- else:
139
- raise Exception(f"Unsupported state file version: {metadata.schema_version}")
140
- return state
141
-
142
- @staticmethod
143
- def from_file(file_path: str, file_type: SupportedFileTypes = SupportedFileTypes.FILE):
144
- """
145
- Load the state from a recce state file.
146
- """
147
- from pathlib import Path
148
-
149
- logger.debug(f"Load state file from: '{file_path}'")
150
- if not Path(file_path).is_file():
151
- return None
152
-
153
- io = file_io_factory(file_type)
154
- json_content = io.read(file_path)
155
- return RecceState.from_json(json_content)
156
-
157
- def to_json(self):
158
- return pydantic_model_json_dump(self)
159
-
160
- def to_file(self, file_path: str, file_type: SupportedFileTypes = SupportedFileTypes.FILE):
161
-
162
- json_data = self.to_json()
163
- io = file_io_factory(file_type)
164
-
165
- io.write(file_path, json_data)
166
- return f"The state file is stored at '{file_path}'"
167
-
168
- def _merge_run(self, run: Run):
169
- for r in self.runs:
170
- if r.run_id == run.run_id:
171
- break
172
- else:
173
- self.runs.append(run)
174
-
175
- def _merge_check(self, check: Check):
176
- for c in self.checks:
177
- if c.check_id == check.check_id:
178
- c.merge(check)
179
- break
180
- else:
181
- self.checks.append(check)
182
-
183
- def _merge_artifacts(self, artifacts: ArtifactsRoot):
184
- self.artifacts.merge(artifacts)
185
-
186
-
187
- class RecceStateLoader:
188
- def __init__(
189
- self,
190
- review_mode: bool = False,
191
- cloud_mode: bool = False,
192
- state_file: Optional[str] = None,
193
- cloud_options: Optional[Dict[str, str]] = None,
194
- initial_state: Optional[RecceState] = None,
195
- ):
196
- self.review_mode = review_mode
197
- self.cloud_mode = cloud_mode
198
- self.state_file = state_file
199
- self.cloud_options = cloud_options or {}
200
- self.error_message = None
201
- self.hint_message = None
202
- self.state: RecceState | None = initial_state
203
- self.state_lock = threading.Lock()
204
- self.state_etag = None
205
- self.pr_info = None
206
-
207
- if self.cloud_mode:
208
- if not self.cloud_options.get("token"):
209
- raise Exception(RECCE_CLOUD_TOKEN_MISSING.error_message)
210
- self.pr_info = fetch_pr_metadata(cloud=self.cloud_mode, github_token=self.cloud_options.get("token"))
211
- if self.pr_info.id is None:
212
- raise Exception("Cannot get the pull request information from GitHub.")
213
-
214
- # Load the state
215
- self.load()
216
-
217
- def verify(self) -> bool:
218
- if self.cloud_mode:
219
- if self.cloud_options.get("token") is None:
220
- self.error_message = RECCE_CLOUD_TOKEN_MISSING.error_message
221
- self.hint_message = RECCE_CLOUD_TOKEN_MISSING.hint_message
222
- return False
223
- if not self.cloud_options.get("host"):
224
- if self.cloud_options.get("password") is None:
225
- self.error_message = RECCE_CLOUD_PASSWORD_MISSING.error_message
226
- self.hint_message = RECCE_CLOUD_PASSWORD_MISSING.hint_message
227
- return False
228
- else:
229
- if self.review_mode is True and self.state_file is None:
230
- self.error_message = "Recce can not launch without a state file."
231
- self.hint_message = "Please provide a state file in the command argument."
232
- return False
233
- pass
234
- return True
235
-
236
- @property
237
- def error_and_hint(self) -> (Union[str, None], Union[str, None]):
238
- return self.error_message, self.hint_message
239
-
240
- def update(self, state: RecceState):
241
- self.state = state
242
-
243
- def load(self, refresh=False) -> RecceState:
244
- if self.state is not None and refresh is False:
245
- return self.state
246
- self.state_lock.acquire()
247
- try:
248
- if self.cloud_mode:
249
- self.state, self.state_etag = self._load_state_from_cloud()
250
- elif self.state_file:
251
- self.state = self._load_state_from_file()
252
- finally:
253
- self.state_lock.release()
254
- return self.state
255
-
256
- def save_as(self, state_file: str, state: RecceState = None):
257
- if self.cloud_mode:
258
- raise Exception("Cannot save the state to Recce Cloud.")
259
-
260
- self.state_file = state_file
261
- self.export(state)
262
-
263
- def export(self, state: RecceState = None) -> Union[str, None]:
264
- if state is not None:
265
- self.update(state)
266
- # TODO: Export the current Recce state to file or cloud storage
267
- start_time = time.time()
268
- self.state_lock.acquire()
269
- try:
270
- if self.cloud_mode:
271
- message, state_etag = self._export_state_to_cloud()
272
- self.state_etag = state_etag
273
- else:
274
- if self.state_file is None:
275
- return "No state file is provided. Skip storing the state."
276
- logger.info(f"Store recce state to '{self.state_file}'")
277
- message = self._export_state_to_file()
278
- end_time = time.time()
279
- elapsed_time = end_time - start_time
280
- finally:
281
- self.state_lock.release()
282
- logger.info(f"Store state completed in {elapsed_time:.2f} seconds")
283
- return message
284
-
285
- def refresh(self):
286
- new_state = self.load(refresh=True)
287
- return new_state
288
-
289
- def check_conflict(self) -> bool:
290
- if not self.cloud_mode:
291
- return False
292
-
293
- if self.cloud_options.get("host", "").startswith("s3://"):
294
- return False
295
-
296
- metadata = self._get_metadata_from_recce_cloud()
297
- if not metadata:
298
- return False
299
-
300
- state_etag = metadata.get("etag")
301
- return state_etag != self.state_etag
302
-
303
- def info(self):
304
- if self.state is None:
305
- self.error_message = "No state is loaded."
306
- return None
307
-
308
- state_info = {
309
- "mode": "cloud" if self.cloud_mode else "local",
310
- "source": None,
311
- }
312
- if self.cloud_mode:
313
- if self.cloud_options.get("host", "").startswith("s3://"):
314
- state_info["source"] = self.cloud_options.get("host")
315
- else:
316
- state_info["source"] = "Recce Cloud"
317
- state_info["pull_request"] = self.pr_info
318
- else:
319
- state_info["source"] = self.state_file
320
- return state_info
321
-
322
- def purge(self) -> bool:
323
- if self.cloud_mode is True:
324
- rc, err_msg = RecceCloudStateManager(self.cloud_options).purge_cloud_state()
325
- if err_msg:
326
- self.error_message = err_msg
327
- return rc
328
- else:
329
- if self.state_file is not None:
330
- try:
331
- os.remove(self.state_file)
332
- except Exception as e:
333
- self.error_message = f"Failed to remove the state file: {e}"
334
- return False
335
- else:
336
- self.error_message = "No state file is provided. Skip removing the state file."
337
- return False
338
-
339
- def _load_state_from_file(self, file_path: Optional[str] = None) -> RecceState:
340
- file_path = file_path or self.state_file
341
- return RecceState.from_file(file_path) if file_path else None
342
-
343
- def _load_state_from_cloud(self) -> Tuple[RecceState, str]:
344
- """
345
- Load the state from Recce Cloud.
346
-
347
- Returns:
348
- RecceState: The state object.
349
- str: The etag of the state file.
350
- """
351
- if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
352
- raise Exception("Cannot get the pull request information from GitHub.")
353
-
354
- if self.cloud_options.get("host", "").startswith("s3://"):
355
- logger.debug("Fetching state from AWS S3 bucket...")
356
- return self._load_state_from_s3_bucket(), None
357
- else:
358
- logger.debug("Fetching state from Recce Cloud...")
359
- metadata = self._get_metadata_from_recce_cloud()
360
- if metadata is None:
361
- return None, None
362
- state_etag = metadata.get("etag")
363
- if self.state_etag and state_etag == self.state_etag:
364
- return self.state, self.state_etag
365
-
366
- return self._load_state_from_recce_cloud(), state_etag
367
-
368
- def _get_metadata_from_recce_cloud(self) -> Union[dict, None]:
369
- recce_cloud = RecceCloud(token=self.cloud_options.get("token"))
370
- return recce_cloud.get_artifact_metadata(pr_info=self.pr_info)
371
-
372
- def _load_state_from_recce_cloud(self) -> Union[RecceState, None]:
373
- import tempfile
374
-
375
- import requests
376
-
377
- recce_cloud = RecceCloud(token=self.cloud_options.get("token"))
378
- presigned_url = recce_cloud.get_presigned_url(
379
- method=PresignedUrlMethod.DOWNLOAD,
380
- pr_id=self.pr_info.id,
381
- repository=self.pr_info.repository,
382
- artifact_name=RECCE_STATE_COMPRESSED_FILE,
383
- )
384
-
385
- password = self.cloud_options.get("password")
386
- if password is None:
387
- raise Exception(RECCE_CLOUD_PASSWORD_MISSING.error_message)
388
-
389
- with tempfile.NamedTemporaryFile() as tmp:
390
- headers = s3_sse_c_headers(password)
391
- response = requests.get(presigned_url, headers=headers)
392
- if response.status_code == 404:
393
- self.error_message = "The state file is not found in Recce Cloud."
394
- return None
395
- elif response.status_code != 200:
396
- self.error_message = response.text
397
- raise Exception(
398
- f"{response.status_code} Failed to download the state file from Recce Cloud. The password could be wrong."
399
- )
400
- with open(tmp.name, "wb") as f:
401
- f.write(response.content)
402
- return RecceState.from_file(tmp.name, file_type=SupportedFileTypes.GZIP)
403
-
404
- def _load_state_from_s3_bucket(self) -> Union[RecceState, None]:
405
- import tempfile
406
-
407
- import boto3
408
-
409
- s3_client = boto3.client("s3")
410
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
411
- s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
412
-
413
- rc, error_message = check_s3_bucket(s3_bucket_name)
414
- if rc is False:
415
- raise Exception(error_message)
416
-
417
- with tempfile.NamedTemporaryFile() as tmp:
418
- try:
419
- s3_client.download_file(s3_bucket_name, s3_bucket_key, tmp.name)
420
- except botocore.exceptions.ClientError as e:
421
- error_code = e.response.get("Error", {}).get("Code")
422
- if error_code == "404":
423
- self.error_message = "The state file is not found in the S3 bucket."
424
- return None
425
- else:
426
- raise e
427
- return RecceState.from_file(tmp.name, file_type=SupportedFileTypes.GZIP)
428
-
429
- def _export_state_to_cloud(self) -> Tuple[Union[str, None], str]:
430
- if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
431
- raise Exception("Cannot get the pull request information from GitHub.")
432
-
433
- check_status = CheckDAO().status()
434
- metadata = {
435
- "total_checks": check_status.get("total", 0),
436
- "approved_checks": check_status.get("approved", 0),
437
- }
438
-
439
- if self.cloud_options.get("host", "").startswith("s3://"):
440
- logger.info("Store recce state to AWS S3 bucket")
441
- return self._export_state_to_s3_bucket(metadata=metadata), None
442
- else:
443
- logger.info("Store recce state to Recce Cloud")
444
- message = self._export_state_to_recce_cloud(metadata=metadata)
445
- metadata = self._get_metadata_from_recce_cloud()
446
- if metadata is None:
447
- return None
448
- state_etag = metadata.get("etag")
449
- return message, state_etag
450
-
451
- def _export_state_to_recce_cloud(self, metadata: dict = None) -> Union[str, None]:
452
- import tempfile
453
-
454
- import requests
455
-
456
- presigned_url = RecceCloud(token=self.cloud_options.get("token")).get_presigned_url(
457
- method=PresignedUrlMethod.UPLOAD,
458
- repository=self.pr_info.repository,
459
- artifact_name=RECCE_STATE_COMPRESSED_FILE,
460
- pr_id=self.pr_info.id,
461
- metadata=metadata,
462
- )
463
- compress_passwd = self.cloud_options.get("password")
464
- headers = s3_sse_c_headers(compress_passwd)
465
- if metadata:
466
- headers["x-amz-tagging"] = urlencode(metadata)
467
- with tempfile.NamedTemporaryFile() as tmp:
468
- self._export_state_to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
469
- response = requests.put(presigned_url, data=open(tmp.name, "rb").read(), headers=headers)
470
- if response.status_code != 200:
471
- self.error_message = response.text
472
- return "Failed to upload the state file to Recce Cloud. Reason: " + response.text
473
- return "The state file is uploaded to Recce Cloud."
474
-
475
- def _export_state_to_s3_bucket(self, metadata: dict = None) -> Union[str, None]:
476
- import tempfile
477
-
478
- import boto3
479
-
480
- s3_client = boto3.client("s3")
481
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
482
- s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
483
-
484
- rc, error_message = check_s3_bucket(s3_bucket_name)
485
- if rc is False:
486
- raise Exception(error_message)
487
-
488
- with tempfile.NamedTemporaryFile() as tmp:
489
- self._export_state_to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
490
-
491
- s3_client.upload_file(
492
- tmp.name,
493
- s3_bucket_name,
494
- s3_bucket_key,
495
- # Casting all the values under metadata to string
496
- ExtraArgs={"Metadata": {k: str(v) for k, v in metadata.items()}},
497
- )
498
- RecceCloud(token=self.cloud_options.get("token")).update_github_pull_request_check(self.pr_info, metadata)
499
- return f"The state file is uploaded to ' s3://{s3_bucket_name}/{s3_bucket_key}'"
500
-
501
- def _get_artifact_metadata_from_s3_bucket(self, artifact_name: str) -> Union[dict, None]:
502
- import boto3
503
-
504
- s3_client = boto3.client("s3")
505
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
506
- s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{artifact_name}"
507
- try:
508
- response = s3_client.head_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
509
- metadata = response["Metadata"]
510
- return metadata
511
- except botocore.exceptions.ClientError as e:
512
- self.error_message = e.response.get("Error", {}).get("Message")
513
- raise Exception("Failed to get artifact metadata from Recce Cloud.")
514
-
515
- def _export_state_to_file(
516
- self, file_path: Optional[str] = None, file_type: SupportedFileTypes = SupportedFileTypes.FILE
517
- ) -> str:
518
- """
519
- Store the state to a file. Store happens when terminating the server or run instance.
520
- """
521
-
522
- file_path = file_path or self.state_file
523
- json_data = self.state.to_json()
524
- io = file_io_factory(file_type)
525
-
526
- io.write(file_path, json_data)
527
- return f"The state file is stored at '{file_path}'"
528
-
529
-
530
- class RecceCloudStateManager:
531
- error_message: str
532
- hint_message: str
533
-
534
- # It is a class to upload, download and purge the state file on Recce Cloud.
535
-
536
- def __init__(self, cloud_options: Optional[Dict[str, str]] = None):
537
- self.cloud_options = cloud_options or {}
538
- self.pr_info = None
539
- self.error_message = None
540
- self.hint_message = None
541
-
542
- if not self.cloud_options.get("token"):
543
- raise Exception(RECCE_CLOUD_TOKEN_MISSING.error_message)
544
- self.pr_info = fetch_pr_metadata(cloud=True, github_token=self.cloud_options.get("token"))
545
- if self.pr_info.id is None:
546
- raise Exception("Cannot get the pull request information from GitHub.")
547
-
548
- def verify(self) -> bool:
549
- if self.cloud_options.get("token") is None:
550
- self.error_message = RECCE_CLOUD_TOKEN_MISSING.error_message
551
- self.hint_message = RECCE_CLOUD_TOKEN_MISSING.hint_message
552
- return False
553
- if self.cloud_options.get("password") is None:
554
- self.error_message = RECCE_CLOUD_PASSWORD_MISSING.error_message
555
- self.hint_message = RECCE_CLOUD_PASSWORD_MISSING.hint_message
556
- return False
557
- return True
558
-
559
- @property
560
- def error_and_hint(self) -> (Union[str, None], Union[str, None]):
561
- return self.error_message, self.hint_message
562
-
563
- def _check_state_in_recce_cloud(self) -> bool:
564
- return RecceCloud(token=self.cloud_options.get("token")).check_artifacts_exists(self.pr_info)
565
-
566
- def _check_state_in_s3_bucket(self) -> bool:
567
- import boto3
568
-
569
- s3_client = boto3.client("s3")
570
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
571
- s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
572
- try:
573
- s3_client.head_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
574
- except botocore.exceptions.ClientError as e:
575
- error_code = e.response.get("Error", {}).get("Code")
576
- if error_code == "404":
577
- return False
578
- return True
579
-
580
- def check_cloud_state_exists(self) -> bool:
581
- if self.cloud_options.get("host", "").startswith("s3://"):
582
- return self._check_state_in_s3_bucket()
583
- else:
584
- return self._check_state_in_recce_cloud()
585
-
586
- def _upload_state_to_recce_cloud(self, state: RecceState, metadata: dict = None) -> Union[str, None]:
587
- import tempfile
588
-
589
- import requests
590
-
591
- presigned_url = RecceCloud(token=self.cloud_options.get("token")).get_presigned_url(
592
- method=PresignedUrlMethod.UPLOAD,
593
- repository=self.pr_info.repository,
594
- artifact_name=RECCE_STATE_COMPRESSED_FILE,
595
- pr_id=self.pr_info.id,
596
- metadata=metadata,
597
- )
598
-
599
- compress_passwd = self.cloud_options.get("password")
600
- headers = s3_sse_c_headers(compress_passwd)
601
- with tempfile.NamedTemporaryFile() as tmp:
602
- state.to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
603
- response = requests.put(presigned_url, data=open(tmp.name, "rb").read(), headers=headers)
604
- if response.status_code != 200:
605
- return f"Failed to upload the state file to Recce Cloud. Reason: {response.text}"
606
- return "The state file is uploaded to Recce Cloud."
607
-
608
- def _upload_state_to_s3_bucket(self, state: RecceState, metadata: dict = None) -> Union[str, None]:
609
- import tempfile
610
-
611
- import boto3
612
-
613
- s3_client = boto3.client("s3")
614
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
615
- s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
616
-
617
- rc, error_message = check_s3_bucket(s3_bucket_name)
618
- if rc is False:
619
- raise Exception(error_message)
620
-
621
- with tempfile.NamedTemporaryFile() as tmp:
622
- state.to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
623
-
624
- s3_client.upload_file(
625
- tmp.name,
626
- s3_bucket_name,
627
- s3_bucket_key,
628
- # Casting all the values under metadata to string
629
- ExtraArgs={"Metadata": {k: str(v) for k, v in metadata.items()}},
630
- )
631
- RecceCloud(token=self.cloud_options.get("token")).update_github_pull_request_check(self.pr_info, metadata)
632
- return f"The state file is uploaded to ' s3://{s3_bucket_name}/{s3_bucket_key}'"
633
-
634
- def upload_state_to_cloud(self, state: RecceState) -> Union[str, None]:
635
- if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
636
- raise Exception("Cannot get the pull request information from GitHub.")
637
-
638
- checks = state.checks
639
-
640
- metadata = {
641
- "total_checks": len(checks),
642
- "approved_checks": len([c for c in checks if c.is_checked]),
643
- }
644
-
645
- if self.cloud_options.get("host", "").startswith("s3://"):
646
- return self._upload_state_to_s3_bucket(state, metadata)
647
- else:
648
- return self._upload_state_to_recce_cloud(state, metadata)
649
-
650
- def _download_state_from_s3_bucket(self, filepath):
651
- import io
652
-
653
- import boto3
654
-
655
- s3_client = boto3.client("s3")
656
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
657
- s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
658
-
659
- rc, error_message = check_s3_bucket(s3_bucket_name)
660
- if rc is False:
661
- raise Exception(error_message)
662
-
663
- response = s3_client.get_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
664
- byte_stream = io.BytesIO(response["Body"].read())
665
- gzip_io = file_io_factory(SupportedFileTypes.GZIP)
666
- decompressed_content = gzip_io.read_fileobj(byte_stream)
667
-
668
- dirs = os.path.dirname(filepath)
669
- if dirs:
670
- os.makedirs(dirs, exist_ok=True)
671
- with open(filepath, "wb") as f:
672
- f.write(decompressed_content)
673
-
674
- def _download_state_from_recce_cloud(self, filepath):
675
- import io
676
-
677
- import requests
678
-
679
- presigned_url = RecceCloud(token=self.cloud_options.get("token")).get_presigned_url(
680
- method=PresignedUrlMethod.DOWNLOAD,
681
- repository=self.pr_info.repository,
682
- artifact_name=RECCE_STATE_COMPRESSED_FILE,
683
- pr_id=self.pr_info.id,
684
- )
685
-
686
- password = self.cloud_options.get("password")
687
- if password is None:
688
- raise Exception(RECCE_CLOUD_PASSWORD_MISSING.error_message)
689
-
690
- headers = s3_sse_c_headers(password)
691
- response = requests.get(presigned_url, headers=headers)
692
-
693
- if response.status_code != 200:
694
- raise Exception(
695
- f"{response.status_code} Failed to download the state file from Recce Cloud. The password could be wrong."
696
- )
697
-
698
- byte_stream = io.BytesIO(response.content)
699
- gzip_io = file_io_factory(SupportedFileTypes.GZIP)
700
- decompressed_content = gzip_io.read_fileobj(byte_stream)
701
-
702
- dirs = os.path.dirname(filepath)
703
- if dirs:
704
- os.makedirs(dirs, exist_ok=True)
705
- with open(filepath, "wb") as f:
706
- f.write(decompressed_content)
707
-
708
- def download_state_from_cloud(self, filepath: str) -> Union[str, None]:
709
- if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
710
- raise Exception("Cannot get the pull request information from GitHub.")
711
-
712
- if self.cloud_options.get("host", "").startswith("s3://"):
713
- logger.debug("Download state file from AWS S3 bucket...")
714
- return self._download_state_from_s3_bucket(filepath)
715
- else:
716
- logger.debug("Download state file from Recce Cloud...")
717
- return self._download_state_from_recce_cloud(filepath)
718
-
719
- def _purge_state_from_s3_bucket(self) -> (bool, str):
720
- import boto3
721
- from rich.console import Console
722
-
723
- console = Console()
724
- delete_objects = []
725
- logger.debug("Purging the state from AWS S3 bucket...")
726
- s3_client = boto3.client("s3")
727
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
728
- s3_key_prefix = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/"
729
- list_response = s3_client.list_objects_v2(Bucket=s3_bucket_name, Prefix=s3_key_prefix)
730
- if "Contents" in list_response:
731
- for obj in list_response["Contents"]:
732
- key = obj["Key"]
733
- delete_objects.append({"Key": key})
734
- console.print(f"[green]Deleted[/green]: {key}")
735
- else:
736
- return False, "No state file found in the S3 bucket."
737
-
738
- delete_response = s3_client.delete_objects(Bucket=s3_bucket_name, Delete={"Objects": delete_objects})
739
- if "Deleted" not in delete_response:
740
- return False, "Failed to delete the state file from the S3 bucket."
741
- RecceCloud(token=self.cloud_options.get("token")).update_github_pull_request_check(self.pr_info)
742
- return True, None
743
-
744
- def _purge_state_from_recce_cloud(self) -> (bool, str):
745
- try:
746
- RecceCloud(token=self.cloud_options.get("token")).purge_artifacts(self.pr_info)
747
- except RecceCloudException as e:
748
- return False, e.reason
749
- return True, None
750
-
751
- def purge_cloud_state(self) -> (bool, str):
752
- if self.cloud_options.get("host", "").startswith("s3://"):
753
- return self._purge_state_from_s3_bucket()
754
- else:
755
- return self._purge_state_from_recce_cloud()
756
-
757
-
758
- class RecceShareStateManager:
759
- error_message: str
760
- hint_message: str
761
-
762
- # It is a class to share state file on Recce Cloud.
763
-
764
- def __init__(self, auth_options: Optional[Dict[str, str]] = None):
765
- self.auth_options = auth_options or {}
766
- self.error_message = None
767
- self.hint_message = None
768
-
769
- def verify(self) -> bool:
770
- if get_recce_api_token() is None:
771
- self.error_message = RECCE_API_TOKEN_MISSING.error_message
772
- self.hint_message = RECCE_API_TOKEN_MISSING.hint_message
773
- return False
774
- return True
775
-
776
- @property
777
- def error_and_hint(self) -> (Union[str, None], Union[str, None]):
778
- return self.error_message, self.hint_message
779
-
780
- def share_state(self, file_name: str, state: RecceState) -> Dict:
781
- import tempfile
782
-
783
- with tempfile.NamedTemporaryFile() as tmp:
784
- state.to_file(tmp.name, file_type=SupportedFileTypes.FILE)
785
- response = RecceCloud(token=get_recce_api_token()).share_state(file_name, open(tmp.name, "rb"))
786
- return response