recce-nightly 1.9.0.20250623__py3-none-any.whl → 1.25.0.20251112a2066__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (169) hide show
  1. recce/VERSION +1 -1
  2. recce/__init__.py +5 -0
  3. recce/adapter/dbt_adapter/__init__.py +318 -240
  4. recce/artifact.py +76 -3
  5. recce/cli.py +703 -71
  6. recce/config.py +3 -3
  7. recce/connect_to_cloud.py +138 -0
  8. recce/core.py +3 -3
  9. recce/data/404.html +1 -22
  10. recce/data/__next.__PAGE__.txt +10 -0
  11. recce/data/__next._full.txt +23 -0
  12. recce/data/__next._index.txt +8 -0
  13. recce/data/__next._tree.txt +12 -0
  14. recce/data/_next/static/6LypcDXgyuSaiSCrsmUub/_buildManifest.js +11 -0
  15. recce/data/_next/static/6LypcDXgyuSaiSCrsmUub/_clientMiddlewareManifest.json +1 -0
  16. recce/data/_next/static/chunks/0a2b2dd4b57049c2.js +1 -0
  17. recce/data/_next/static/chunks/19c10d219a6a21ff.js +1 -0
  18. recce/data/_next/static/chunks/24fd885c7180a612.js +1 -0
  19. recce/data/_next/static/chunks/27e66b2eab4adc32.js +19 -0
  20. recce/data/_next/static/chunks/71f88fcc615bf282.js +1 -0
  21. recce/data/_next/static/chunks/917619ab62a32388.js +1 -0
  22. recce/data/_next/static/chunks/93ba5a62932b704f.js +4 -0
  23. recce/data/_next/static/chunks/a43a2a5e06d5a92b.js +1 -0
  24. recce/data/_next/static/chunks/a6c78b24bd8b84fc.js +1 -0
  25. recce/data/_next/static/chunks/b2610ba997ff8c4f.js +110 -0
  26. recce/data/_next/static/chunks/ba2d87265a68599d.css +2 -0
  27. recce/data/_next/static/chunks/c117fd1c1382dd83.js +11 -0
  28. recce/data/_next/static/chunks/c9425ca46eebdde9.js +1 -0
  29. recce/data/_next/static/chunks/cc8a9eadba012be0.css +6 -0
  30. recce/data/_next/static/chunks/e124bccf574a3361.css +1 -0
  31. recce/data/_next/static/chunks/e392ad92847c3e17.js +1 -0
  32. recce/data/_next/static/chunks/e4ce95efe88dae79.js +11 -0
  33. recce/data/_next/static/chunks/e69c777814fea6ed.js +2 -0
  34. recce/data/_next/static/chunks/turbopack-21cfd73037ff57ab.js +3 -0
  35. recce/data/_next/static/media/favicon.a8d38d84.ico +0 -0
  36. recce/data/_next/static/media/montserrat-cyrillic-800-normal.d80d830d.woff2 +0 -0
  37. recce/data/_next/static/media/{montserrat-cyrillic-800-normal.bd5c9f50.woff → montserrat-cyrillic-800-normal.f9d58125.woff} +0 -0
  38. recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.076c2a93.woff2 +0 -0
  39. recce/data/_next/static/media/montserrat-latin-800-normal.cde454cc.woff2 +0 -0
  40. recce/data/_next/static/media/{montserrat-latin-800-normal.fc315020.woff → montserrat-latin-800-normal.d5761935.woff} +0 -0
  41. recce/data/_next/static/media/montserrat-latin-ext-800-normal.40ec0659.woff2 +0 -0
  42. recce/data/_next/static/media/{montserrat-latin-ext-800-normal.2e5381b2.woff → montserrat-latin-ext-800-normal.b671449b.woff} +0 -0
  43. recce/data/_next/static/media/{montserrat-vietnamese-800-normal.20c545e6.woff → montserrat-vietnamese-800-normal.9f7b8541.woff} +0 -0
  44. recce/data/_next/static/media/montserrat-vietnamese-800-normal.f9eb854e.woff2 +0 -0
  45. recce/data/_not-found/__next._full.txt +17 -0
  46. recce/data/_not-found/__next._index.txt +8 -0
  47. recce/data/_not-found/__next._not-found.__PAGE__.txt +5 -0
  48. recce/data/_not-found/__next._not-found.txt +4 -0
  49. recce/data/_not-found/__next._tree.txt +10 -0
  50. recce/data/_not-found.html +1 -0
  51. recce/data/_not-found.txt +17 -0
  52. recce/data/auth_callback.html +68 -0
  53. recce/data/index.html +1 -27
  54. recce/data/index.txt +23 -8
  55. recce/event/__init__.py +9 -8
  56. recce/event/collector.py +6 -2
  57. recce/event/track.py +10 -0
  58. recce/github.py +1 -1
  59. recce/mcp_server.py +632 -0
  60. recce/models/types.py +23 -2
  61. recce/pull_request.py +1 -1
  62. recce/run.py +23 -16
  63. recce/server.py +194 -19
  64. recce/state/__init__.py +31 -0
  65. recce/state/cloud.py +632 -0
  66. recce/state/const.py +26 -0
  67. recce/state/local.py +56 -0
  68. recce/state/state.py +119 -0
  69. recce/state/state_loader.py +174 -0
  70. recce/summary.py +2 -1
  71. recce/tasks/dataframe.py +59 -2
  72. recce/tasks/rowcount.py +4 -1
  73. recce/tasks/schema.py +4 -1
  74. recce/tasks/valuediff.py +1 -1
  75. recce/util/api_token.py +11 -2
  76. recce/util/breaking.py +9 -0
  77. recce/util/cll.py +1 -2
  78. recce/util/io.py +2 -2
  79. recce/util/lineage.py +19 -18
  80. recce/util/perf_tracking.py +85 -0
  81. recce/util/recce_cloud.py +229 -5
  82. recce/yaml/__init__.py +2 -2
  83. recce_cloud/__init__.py +15 -0
  84. recce_cloud/api/__init__.py +17 -0
  85. recce_cloud/api/base.py +104 -0
  86. recce_cloud/api/client.py +150 -0
  87. recce_cloud/api/exceptions.py +26 -0
  88. recce_cloud/api/factory.py +63 -0
  89. recce_cloud/api/github.py +72 -0
  90. recce_cloud/api/gitlab.py +78 -0
  91. recce_cloud/artifact.py +57 -0
  92. recce_cloud/ci_providers/__init__.py +9 -0
  93. recce_cloud/ci_providers/base.py +82 -0
  94. recce_cloud/ci_providers/detector.py +147 -0
  95. recce_cloud/ci_providers/github_actions.py +136 -0
  96. recce_cloud/ci_providers/gitlab_ci.py +130 -0
  97. recce_cloud/cli.py +303 -0
  98. recce_cloud/upload.py +213 -0
  99. {recce_nightly-1.9.0.20250623.dist-info → recce_nightly-1.25.0.20251112a2066.dist-info}/METADATA +31 -27
  100. recce_nightly-1.25.0.20251112a2066.dist-info/RECORD +178 -0
  101. {recce_nightly-1.9.0.20250623.dist-info → recce_nightly-1.25.0.20251112a2066.dist-info}/top_level.txt +1 -0
  102. tests/adapter/dbt_adapter/test_dbt_cll.py +412 -79
  103. tests/recce_cloud/__init__.py +0 -0
  104. tests/recce_cloud/test_ci_providers.py +351 -0
  105. tests/recce_cloud/test_cli.py +372 -0
  106. tests/recce_cloud/test_client.py +273 -0
  107. tests/recce_cloud/test_platform_clients.py +279 -0
  108. tests/test_cli.py +106 -3
  109. tests/test_cli_mcp_optional.py +45 -0
  110. tests/test_cloud_listing_cli.py +324 -0
  111. tests/test_connect_to_cloud.py +82 -0
  112. tests/test_core.py +148 -3
  113. tests/test_mcp_server.py +332 -0
  114. tests/test_server.py +6 -6
  115. tests/test_summary.py +14 -6
  116. recce/data/_next/static/WrRUb3nV8BhAZG_R8kVma/_buildManifest.js +0 -1
  117. recce/data/_next/static/chunks/181-acc61ddada3bc0ca.js +0 -43
  118. recce/data/_next/static/chunks/1bff33f1-1ef85cf5e658a751.js +0 -1
  119. recce/data/_next/static/chunks/217-879a84d70f7a907c.js +0 -2
  120. recce/data/_next/static/chunks/29e3cc0d-60045b2e47aa3916.js +0 -1
  121. recce/data/_next/static/chunks/36e1c10d-8e7be4a6c1f6ab2d.js +0 -1
  122. recce/data/_next/static/chunks/3998a672-03adacad07b346ac.js +0 -1
  123. recce/data/_next/static/chunks/3a92ee20-1081c360214f9602.js +0 -1
  124. recce/data/_next/static/chunks/42-cd3c06533f5fd47c.js +0 -9
  125. recce/data/_next/static/chunks/450c323b-fd94e7ffaa4a5efa.js +0 -1
  126. recce/data/_next/static/chunks/47d8844f-929aed9b1c73a905.js +0 -1
  127. recce/data/_next/static/chunks/608-3b079b544e5d5f5e.js +0 -15
  128. recce/data/_next/static/chunks/6dc81886-adbfa45836061d79.js +0 -1
  129. recce/data/_next/static/chunks/7a8a3e83-edf6dc64b5d5f0a5.js +0 -1
  130. recce/data/_next/static/chunks/7f27ae6c-d5f0438edd5c2a5b.js +0 -1
  131. recce/data/_next/static/chunks/86730205-cfb14e3f051bab35.js +0 -1
  132. recce/data/_next/static/chunks/8d700b6a.8bb140898499c512.js +0 -1
  133. recce/data/_next/static/chunks/92-7ab55ae02606193c.js +0 -1
  134. recce/data/_next/static/chunks/9746af58-a42b7d169cacadf0.js +0 -1
  135. recce/data/_next/static/chunks/a30376cd-de84559016d7e133.js +0 -1
  136. recce/data/_next/static/chunks/app/_not-found/page-01ed58b7f971d311.js +0 -1
  137. recce/data/_next/static/chunks/app/layout-177a410a97e0d018.js +0 -1
  138. recce/data/_next/static/chunks/app/page-59241c42b7dd4fcf.js +0 -1
  139. recce/data/_next/static/chunks/b63b1b3f-4282bdcf459e075c.js +0 -1
  140. recce/data/_next/static/chunks/bbda5537-9ec25eb1dd62348a.js +0 -1
  141. recce/data/_next/static/chunks/c132bf7d-08cb668a789d6afd.js +0 -1
  142. recce/data/_next/static/chunks/ce84277d-2e5d1d46910cf052.js +0 -1
  143. recce/data/_next/static/chunks/febdd86e-c6b525341634b860.js +0 -54
  144. recce/data/_next/static/chunks/fee69bc6-2dbccaf9b90474e6.js +0 -1
  145. recce/data/_next/static/chunks/framework-ded83d71b51ce901.js +0 -1
  146. recce/data/_next/static/chunks/main-app-39061b0166c47f55.js +0 -1
  147. recce/data/_next/static/chunks/main-b5b3ae20a1405261.js +0 -1
  148. recce/data/_next/static/chunks/pages/_app-437c455677d62394.js +0 -1
  149. recce/data/_next/static/chunks/pages/_error-e7650df18ca04bde.js +0 -1
  150. recce/data/_next/static/chunks/webpack-7b49d5ba7e3a434d.js +0 -1
  151. recce/data/_next/static/css/17a96168e3a9db13.css +0 -1
  152. recce/data/_next/static/css/1b121dc4d36aeb4d.css +0 -3
  153. recce/data/_next/static/css/35c6679a098e1e34.css +0 -1
  154. recce/data/_next/static/css/951e2e0eea2d4a5b.css +0 -14
  155. recce/data/_next/static/media/montserrat-cyrillic-800-normal.22628180.woff2 +0 -0
  156. recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.94a63aea.woff2 +0 -0
  157. recce/data/_next/static/media/montserrat-latin-800-normal.6f8fa298.woff2 +0 -0
  158. recce/data/_next/static/media/montserrat-latin-ext-800-normal.013b84f9.woff2 +0 -0
  159. recce/data/_next/static/media/montserrat-vietnamese-800-normal.c0035377.woff2 +0 -0
  160. recce/state.py +0 -785
  161. recce_nightly-1.9.0.20250623.dist-info/RECORD +0 -151
  162. tests/test_state.py +0 -134
  163. /recce/data/_next/static/{WrRUb3nV8BhAZG_R8kVma → 6LypcDXgyuSaiSCrsmUub}/_ssgManifest.js +0 -0
  164. /recce/data/_next/static/chunks/{polyfills-42372ed130431b0a.js → a6dad97d9634a72d.js} +0 -0
  165. /recce/data/_next/static/media/{montserrat-cyrillic-ext-800-normal.e6e0d8d0.woff → montserrat-cyrillic-ext-800-normal.a4fa76b5.woff} +0 -0
  166. /recce/data/_next/static/media/{reload-image.79aabb7d.svg → reload-image.7aa931c7.svg} +0 -0
  167. {recce_nightly-1.9.0.20250623.dist-info → recce_nightly-1.25.0.20251112a2066.dist-info}/WHEEL +0 -0
  168. {recce_nightly-1.9.0.20250623.dist-info → recce_nightly-1.25.0.20251112a2066.dist-info}/entry_points.txt +0 -0
  169. {recce_nightly-1.9.0.20250623.dist-info → recce_nightly-1.25.0.20251112a2066.dist-info}/licenses/LICENSE +0 -0
recce/state.py DELETED
@@ -1,785 +0,0 @@
1
- """Define the type to serialize/de-serialize the state of the recce instance."""
2
-
3
- import json
4
- import logging
5
- import os
6
- import threading
7
- import time
8
- from base64 import b64encode
9
- from dataclasses import dataclass
10
- from datetime import datetime
11
- from hashlib import md5, sha256
12
- from typing import Dict, List, Optional, Tuple, Union
13
- from urllib.parse import urlencode
14
-
15
- import botocore.exceptions
16
- from pydantic import BaseModel, Field
17
-
18
- from recce import get_version
19
- from recce.git import current_branch
20
- from recce.models import CheckDAO
21
- from recce.models.types import Check, Run
22
- from recce.pull_request import PullRequestInfo, fetch_pr_metadata
23
- from recce.util.io import SupportedFileTypes, file_io_factory
24
- from recce.util.pydantic_model import pydantic_model_dump, pydantic_model_json_dump
25
- from recce.util.recce_cloud import PresignedUrlMethod, RecceCloud, RecceCloudException
26
-
27
- logger = logging.getLogger("uvicorn")
28
-
29
- RECCE_STATE_FILE = "recce-state.json"
30
- RECCE_STATE_COMPRESSED_FILE = f"{RECCE_STATE_FILE}.gz"
31
-
32
-
33
- @dataclass
34
- class ErrorMessage:
35
- error_message: str
36
- hint_message: str
37
-
38
-
39
- RECCE_CLOUD_TOKEN_MISSING = ErrorMessage(
40
- error_message="No GitHub token is provided to access the pull request information",
41
- hint_message="Please provide a GitHub token in the command argument",
42
- )
43
-
44
- RECCE_CLOUD_PASSWORD_MISSING = ErrorMessage(
45
- error_message="No password provided to access the state file in Recce Cloud",
46
- hint_message='Please provide a password with the option "--password <compress-password>"',
47
- )
48
-
49
- RECCE_API_TOKEN_MISSING = ErrorMessage(
50
- error_message="No Recc API token is provided",
51
- hint_message="Please login to Recce Cloud and copy the API token from the settings page",
52
- )
53
-
54
-
55
- def s3_sse_c_headers(password: str) -> Dict[str, str]:
56
- hashed_password = sha256()
57
- md5_hash = md5()
58
- hashed_password.update(password.encode())
59
- md5_hash.update(hashed_password.digest())
60
- encoded_passwd = b64encode(hashed_password.digest()).decode("utf-8")
61
- encoded_md5 = b64encode(md5_hash.digest()).decode("utf-8")
62
- return {
63
- "x-amz-server-side-encryption-customer-algorithm": "AES256",
64
- "x-amz-server-side-encryption-customer-key": encoded_passwd,
65
- "x-amz-server-side-encryption-customer-key-MD5": encoded_md5,
66
- }
67
-
68
-
69
- def check_s3_bucket(bucket_name: str):
70
- import boto3
71
-
72
- s3_client = boto3.client("s3")
73
- try:
74
- s3_client.head_bucket(Bucket=bucket_name)
75
- except botocore.exceptions.ClientError as e:
76
- error_code = e.response["Error"]["Code"]
77
- if error_code == "404":
78
- return False, f"Bucket '{bucket_name}' does not exist."
79
- elif error_code == "403":
80
- return False, f"Bucket '{bucket_name}' exists but you do not have permission to access it."
81
- else:
82
- return False, f"Failed to access the S3 bucket: '{bucket_name}'"
83
- return True, None
84
-
85
-
86
- class GitRepoInfo(BaseModel):
87
- branch: Optional[str] = None
88
-
89
- @staticmethod
90
- def from_current_repositroy():
91
- branch = current_branch()
92
- if branch is None:
93
- return None
94
-
95
- return GitRepoInfo(branch=branch)
96
-
97
- def to_dict(self):
98
- return pydantic_model_dump(self)
99
-
100
-
101
- class RecceStateMetadata(BaseModel):
102
- schema_version: str = "v0"
103
- recce_version: str = Field(default_factory=lambda: get_version())
104
- generated_at: str = Field(default_factory=lambda: datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"))
105
-
106
-
107
- class ArtifactsRoot(BaseModel):
108
- """
109
- Root of the artifacts.
110
-
111
- base: artifacts of the base env. key is file name, value is dict
112
- current: artifacts of the current env. key is file name, value is dict
113
- """
114
-
115
- base: Dict[str, Optional[dict]] = {}
116
- current: Dict[str, Optional[dict]] = {}
117
-
118
-
119
- class RecceState(BaseModel):
120
- metadata: Optional[RecceStateMetadata] = None
121
- runs: Optional[List[Run]] = Field(default_factory=list)
122
- checks: Optional[List[Check]] = Field(default_factory=list)
123
- artifacts: ArtifactsRoot = ArtifactsRoot(base={}, current={})
124
- git: Optional[GitRepoInfo] = None
125
- pull_request: Optional[PullRequestInfo] = None
126
-
127
- @staticmethod
128
- def from_json(json_content: str):
129
- dict_data = json.loads(json_content)
130
- state = RecceState(**dict_data)
131
- metadata = state.metadata
132
- if metadata:
133
- if metadata.schema_version is None:
134
- pass
135
- if metadata.schema_version == "v0":
136
- pass
137
- else:
138
- raise Exception(f"Unsupported state file version: {metadata.schema_version}")
139
- return state
140
-
141
- @staticmethod
142
- def from_file(file_path: str, file_type: SupportedFileTypes = SupportedFileTypes.FILE):
143
- """
144
- Load the state from a recce state file.
145
- """
146
- from pathlib import Path
147
-
148
- logger.debug(f"Load state file from: '{file_path}'")
149
- if not Path(file_path).is_file():
150
- return None
151
-
152
- io = file_io_factory(file_type)
153
- json_content = io.read(file_path)
154
- return RecceState.from_json(json_content)
155
-
156
- def to_json(self):
157
- return pydantic_model_json_dump(self)
158
-
159
- def to_file(self, file_path: str, file_type: SupportedFileTypes = SupportedFileTypes.FILE):
160
-
161
- json_data = self.to_json()
162
- io = file_io_factory(file_type)
163
-
164
- io.write(file_path, json_data)
165
- return f"The state file is stored at '{file_path}'"
166
-
167
- def _merge_run(self, run: Run):
168
- for r in self.runs:
169
- if r.run_id == run.run_id:
170
- break
171
- else:
172
- self.runs.append(run)
173
-
174
- def _merge_check(self, check: Check):
175
- for c in self.checks:
176
- if c.check_id == check.check_id:
177
- c.merge(check)
178
- break
179
- else:
180
- self.checks.append(check)
181
-
182
- def _merge_artifacts(self, artifacts: ArtifactsRoot):
183
- self.artifacts.merge(artifacts)
184
-
185
-
186
- class RecceStateLoader:
187
- def __init__(
188
- self,
189
- review_mode: bool = False,
190
- cloud_mode: bool = False,
191
- state_file: Optional[str] = None,
192
- cloud_options: Optional[Dict[str, str]] = None,
193
- initial_state: Optional[RecceState] = None,
194
- ):
195
- self.review_mode = review_mode
196
- self.cloud_mode = cloud_mode
197
- self.state_file = state_file
198
- self.cloud_options = cloud_options or {}
199
- self.error_message = None
200
- self.hint_message = None
201
- self.state: RecceState | None = initial_state
202
- self.state_lock = threading.Lock()
203
- self.state_etag = None
204
- self.pr_info = None
205
-
206
- if self.cloud_mode:
207
- if not self.cloud_options.get("token"):
208
- raise Exception(RECCE_CLOUD_TOKEN_MISSING.error_message)
209
- self.pr_info = fetch_pr_metadata(cloud=self.cloud_mode, github_token=self.cloud_options.get("token"))
210
- if self.pr_info.id is None:
211
- raise Exception("Cannot get the pull request information from GitHub.")
212
-
213
- # Load the state
214
- self.load()
215
-
216
- def verify(self) -> bool:
217
- if self.cloud_mode:
218
- if self.cloud_options.get("token") is None:
219
- self.error_message = RECCE_CLOUD_TOKEN_MISSING.error_message
220
- self.hint_message = RECCE_CLOUD_TOKEN_MISSING.hint_message
221
- return False
222
- if not self.cloud_options.get("host"):
223
- if self.cloud_options.get("password") is None:
224
- self.error_message = RECCE_CLOUD_PASSWORD_MISSING.error_message
225
- self.hint_message = RECCE_CLOUD_PASSWORD_MISSING.hint_message
226
- return False
227
- else:
228
- if self.review_mode is True and self.state_file is None:
229
- self.error_message = "Recce can not launch without a state file."
230
- self.hint_message = "Please provide a state file in the command argument."
231
- return False
232
- pass
233
- return True
234
-
235
- @property
236
- def error_and_hint(self) -> (Union[str, None], Union[str, None]):
237
- return self.error_message, self.hint_message
238
-
239
- def update(self, state: RecceState):
240
- self.state = state
241
-
242
- def load(self, refresh=False) -> RecceState:
243
- if self.state is not None and refresh is False:
244
- return self.state
245
- self.state_lock.acquire()
246
- try:
247
- if self.cloud_mode:
248
- self.state, self.state_etag = self._load_state_from_cloud()
249
- elif self.state_file:
250
- self.state = self._load_state_from_file()
251
- finally:
252
- self.state_lock.release()
253
- return self.state
254
-
255
- def save_as(self, state_file: str, state: RecceState = None):
256
- if self.cloud_mode:
257
- raise Exception("Cannot save the state to Recce Cloud.")
258
-
259
- self.state_file = state_file
260
- self.export(state)
261
-
262
- def export(self, state: RecceState = None) -> Union[str, None]:
263
- if state is not None:
264
- self.update(state)
265
- # TODO: Export the current Recce state to file or cloud storage
266
- start_time = time.time()
267
- self.state_lock.acquire()
268
- try:
269
- if self.cloud_mode:
270
- message, state_etag = self._export_state_to_cloud()
271
- self.state_etag = state_etag
272
- else:
273
- if self.state_file is None:
274
- return "No state file is provided. Skip storing the state."
275
- logger.info(f"Store recce state to '{self.state_file}'")
276
- message = self._export_state_to_file()
277
- end_time = time.time()
278
- elapsed_time = end_time - start_time
279
- finally:
280
- self.state_lock.release()
281
- logger.info(f"Store state completed in {elapsed_time:.2f} seconds")
282
- return message
283
-
284
- def refresh(self):
285
- new_state = self.load(refresh=True)
286
- return new_state
287
-
288
- def check_conflict(self) -> bool:
289
- if not self.cloud_mode:
290
- return False
291
-
292
- if self.cloud_options.get("host", "").startswith("s3://"):
293
- return False
294
-
295
- metadata = self._get_metadata_from_recce_cloud()
296
- if not metadata:
297
- return False
298
-
299
- state_etag = metadata.get("etag")
300
- return state_etag != self.state_etag
301
-
302
- def info(self):
303
- if self.state is None:
304
- self.error_message = "No state is loaded."
305
- return None
306
-
307
- state_info = {
308
- "mode": "cloud" if self.cloud_mode else "local",
309
- "source": None,
310
- }
311
- if self.cloud_mode:
312
- if self.cloud_options.get("host", "").startswith("s3://"):
313
- state_info["source"] = self.cloud_options.get("host")
314
- else:
315
- state_info["source"] = "Recce Cloud"
316
- state_info["pull_request"] = self.pr_info
317
- else:
318
- state_info["source"] = self.state_file
319
- return state_info
320
-
321
- def purge(self) -> bool:
322
- if self.cloud_mode is True:
323
- rc, err_msg = RecceCloudStateManager(self.cloud_options).purge_cloud_state()
324
- if err_msg:
325
- self.error_message = err_msg
326
- return rc
327
- else:
328
- if self.state_file is not None:
329
- try:
330
- os.remove(self.state_file)
331
- except Exception as e:
332
- self.error_message = f"Failed to remove the state file: {e}"
333
- return False
334
- else:
335
- self.error_message = "No state file is provided. Skip removing the state file."
336
- return False
337
-
338
- def _load_state_from_file(self, file_path: Optional[str] = None) -> RecceState:
339
- file_path = file_path or self.state_file
340
- return RecceState.from_file(file_path) if file_path else None
341
-
342
- def _load_state_from_cloud(self) -> Tuple[RecceState, str]:
343
- """
344
- Load the state from Recce Cloud.
345
-
346
- Returns:
347
- RecceState: The state object.
348
- str: The etag of the state file.
349
- """
350
- if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
351
- raise Exception("Cannot get the pull request information from GitHub.")
352
-
353
- if self.cloud_options.get("host", "").startswith("s3://"):
354
- logger.debug("Fetching state from AWS S3 bucket...")
355
- return self._load_state_from_s3_bucket(), None
356
- else:
357
- logger.debug("Fetching state from Recce Cloud...")
358
- metadata = self._get_metadata_from_recce_cloud()
359
- if metadata is None:
360
- return None, None
361
- state_etag = metadata.get("etag")
362
- if self.state_etag and state_etag == self.state_etag:
363
- return self.state, self.state_etag
364
-
365
- return self._load_state_from_recce_cloud(), state_etag
366
-
367
- def _get_metadata_from_recce_cloud(self) -> Union[dict, None]:
368
- recce_cloud = RecceCloud(token=self.cloud_options.get("token"))
369
- return recce_cloud.get_artifact_metadata(pr_info=self.pr_info)
370
-
371
- def _load_state_from_recce_cloud(self) -> Union[RecceState, None]:
372
- import tempfile
373
-
374
- import requests
375
-
376
- recce_cloud = RecceCloud(token=self.cloud_options.get("token"))
377
- presigned_url = recce_cloud.get_presigned_url(
378
- method=PresignedUrlMethod.DOWNLOAD,
379
- pr_id=self.pr_info.id,
380
- repository=self.pr_info.repository,
381
- artifact_name=RECCE_STATE_COMPRESSED_FILE,
382
- )
383
-
384
- password = self.cloud_options.get("password")
385
- if password is None:
386
- raise Exception(RECCE_CLOUD_PASSWORD_MISSING.error_message)
387
-
388
- with tempfile.NamedTemporaryFile() as tmp:
389
- headers = s3_sse_c_headers(password)
390
- response = requests.get(presigned_url, headers=headers)
391
- if response.status_code == 404:
392
- self.error_message = "The state file is not found in Recce Cloud."
393
- return None
394
- elif response.status_code != 200:
395
- self.error_message = response.text
396
- raise Exception(
397
- f"{response.status_code} Failed to download the state file from Recce Cloud. The password could be wrong."
398
- )
399
- with open(tmp.name, "wb") as f:
400
- f.write(response.content)
401
- return RecceState.from_file(tmp.name, file_type=SupportedFileTypes.GZIP)
402
-
403
- def _load_state_from_s3_bucket(self) -> Union[RecceState, None]:
404
- import tempfile
405
-
406
- import boto3
407
-
408
- s3_client = boto3.client("s3")
409
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
410
- s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
411
-
412
- rc, error_message = check_s3_bucket(s3_bucket_name)
413
- if rc is False:
414
- raise Exception(error_message)
415
-
416
- with tempfile.NamedTemporaryFile() as tmp:
417
- try:
418
- s3_client.download_file(s3_bucket_name, s3_bucket_key, tmp.name)
419
- except botocore.exceptions.ClientError as e:
420
- error_code = e.response.get("Error", {}).get("Code")
421
- if error_code == "404":
422
- self.error_message = "The state file is not found in the S3 bucket."
423
- return None
424
- else:
425
- raise e
426
- return RecceState.from_file(tmp.name, file_type=SupportedFileTypes.GZIP)
427
-
428
- def _export_state_to_cloud(self) -> Tuple[Union[str, None], str]:
429
- if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
430
- raise Exception("Cannot get the pull request information from GitHub.")
431
-
432
- check_status = CheckDAO().status()
433
- metadata = {
434
- "total_checks": check_status.get("total", 0),
435
- "approved_checks": check_status.get("approved", 0),
436
- }
437
-
438
- if self.cloud_options.get("host", "").startswith("s3://"):
439
- logger.info("Store recce state to AWS S3 bucket")
440
- return self._export_state_to_s3_bucket(metadata=metadata), None
441
- else:
442
- logger.info("Store recce state to Recce Cloud")
443
- message = self._export_state_to_recce_cloud(metadata=metadata)
444
- metadata = self._get_metadata_from_recce_cloud()
445
- if metadata is None:
446
- return None
447
- state_etag = metadata.get("etag")
448
- return message, state_etag
449
-
450
- def _export_state_to_recce_cloud(self, metadata: dict = None) -> Union[str, None]:
451
- import tempfile
452
-
453
- import requests
454
-
455
- presigned_url = RecceCloud(token=self.cloud_options.get("token")).get_presigned_url(
456
- method=PresignedUrlMethod.UPLOAD,
457
- repository=self.pr_info.repository,
458
- artifact_name=RECCE_STATE_COMPRESSED_FILE,
459
- pr_id=self.pr_info.id,
460
- metadata=metadata,
461
- )
462
- compress_passwd = self.cloud_options.get("password")
463
- headers = s3_sse_c_headers(compress_passwd)
464
- if metadata:
465
- headers["x-amz-tagging"] = urlencode(metadata)
466
- with tempfile.NamedTemporaryFile() as tmp:
467
- self._export_state_to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
468
- response = requests.put(presigned_url, data=open(tmp.name, "rb").read(), headers=headers)
469
- if response.status_code != 200:
470
- self.error_message = response.text
471
- return "Failed to upload the state file to Recce Cloud. Reason: " + response.text
472
- return "The state file is uploaded to Recce Cloud."
473
-
474
- def _export_state_to_s3_bucket(self, metadata: dict = None) -> Union[str, None]:
475
- import tempfile
476
-
477
- import boto3
478
-
479
- s3_client = boto3.client("s3")
480
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
481
- s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
482
-
483
- rc, error_message = check_s3_bucket(s3_bucket_name)
484
- if rc is False:
485
- raise Exception(error_message)
486
-
487
- with tempfile.NamedTemporaryFile() as tmp:
488
- self._export_state_to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
489
-
490
- s3_client.upload_file(
491
- tmp.name,
492
- s3_bucket_name,
493
- s3_bucket_key,
494
- # Casting all the values under metadata to string
495
- ExtraArgs={"Metadata": {k: str(v) for k, v in metadata.items()}},
496
- )
497
- RecceCloud(token=self.cloud_options.get("token")).update_github_pull_request_check(self.pr_info, metadata)
498
- return f"The state file is uploaded to ' s3://{s3_bucket_name}/{s3_bucket_key}'"
499
-
500
- def _get_artifact_metadata_from_s3_bucket(self, artifact_name: str) -> Union[dict, None]:
501
- import boto3
502
-
503
- s3_client = boto3.client("s3")
504
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
505
- s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{artifact_name}"
506
- try:
507
- response = s3_client.head_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
508
- metadata = response["Metadata"]
509
- return metadata
510
- except botocore.exceptions.ClientError as e:
511
- self.error_message = e.response.get("Error", {}).get("Message")
512
- raise Exception("Failed to get artifact metadata from Recce Cloud.")
513
-
514
- def _export_state_to_file(
515
- self, file_path: Optional[str] = None, file_type: SupportedFileTypes = SupportedFileTypes.FILE
516
- ) -> str:
517
- """
518
- Store the state to a file. Store happens when terminating the server or run instance.
519
- """
520
-
521
- file_path = file_path or self.state_file
522
- json_data = self.state.to_json()
523
- io = file_io_factory(file_type)
524
-
525
- io.write(file_path, json_data)
526
- return f"The state file is stored at '{file_path}'"
527
-
528
-
529
- class RecceCloudStateManager:
530
- error_message: str
531
- hint_message: str
532
-
533
- # It is a class to upload, download and purge the state file on Recce Cloud.
534
-
535
- def __init__(self, cloud_options: Optional[Dict[str, str]] = None):
536
- self.cloud_options = cloud_options or {}
537
- self.pr_info = None
538
- self.error_message = None
539
- self.hint_message = None
540
-
541
- if not self.cloud_options.get("token"):
542
- raise Exception(RECCE_CLOUD_TOKEN_MISSING.error_message)
543
- self.pr_info = fetch_pr_metadata(cloud=True, github_token=self.cloud_options.get("token"))
544
- if self.pr_info.id is None:
545
- raise Exception("Cannot get the pull request information from GitHub.")
546
-
547
- def verify(self) -> bool:
548
- if self.cloud_options.get("token") is None:
549
- self.error_message = RECCE_CLOUD_TOKEN_MISSING.error_message
550
- self.hint_message = RECCE_CLOUD_TOKEN_MISSING.hint_message
551
- return False
552
- if self.cloud_options.get("password") is None:
553
- self.error_message = RECCE_CLOUD_PASSWORD_MISSING.error_message
554
- self.hint_message = RECCE_CLOUD_PASSWORD_MISSING.hint_message
555
- return False
556
- return True
557
-
558
- @property
559
- def error_and_hint(self) -> (Union[str, None], Union[str, None]):
560
- return self.error_message, self.hint_message
561
-
562
- def _check_state_in_recce_cloud(self) -> bool:
563
- return RecceCloud(token=self.cloud_options.get("token")).check_artifacts_exists(self.pr_info)
564
-
565
- def _check_state_in_s3_bucket(self) -> bool:
566
- import boto3
567
-
568
- s3_client = boto3.client("s3")
569
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
570
- s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
571
- try:
572
- s3_client.head_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
573
- except botocore.exceptions.ClientError as e:
574
- error_code = e.response.get("Error", {}).get("Code")
575
- if error_code == "404":
576
- return False
577
- return True
578
-
579
- def check_cloud_state_exists(self) -> bool:
580
- if self.cloud_options.get("host", "").startswith("s3://"):
581
- return self._check_state_in_s3_bucket()
582
- else:
583
- return self._check_state_in_recce_cloud()
584
-
585
- def _upload_state_to_recce_cloud(self, state: RecceState, metadata: dict = None) -> Union[str, None]:
586
- import tempfile
587
-
588
- import requests
589
-
590
- presigned_url = RecceCloud(token=self.cloud_options.get("token")).get_presigned_url(
591
- method=PresignedUrlMethod.UPLOAD,
592
- repository=self.pr_info.repository,
593
- artifact_name=RECCE_STATE_COMPRESSED_FILE,
594
- pr_id=self.pr_info.id,
595
- metadata=metadata,
596
- )
597
-
598
- compress_passwd = self.cloud_options.get("password")
599
- headers = s3_sse_c_headers(compress_passwd)
600
- with tempfile.NamedTemporaryFile() as tmp:
601
- state.to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
602
- response = requests.put(presigned_url, data=open(tmp.name, "rb").read(), headers=headers)
603
- if response.status_code != 200:
604
- return f"Failed to upload the state file to Recce Cloud. Reason: {response.text}"
605
- return "The state file is uploaded to Recce Cloud."
606
-
607
- def _upload_state_to_s3_bucket(self, state: RecceState, metadata: dict = None) -> Union[str, None]:
608
- import tempfile
609
-
610
- import boto3
611
-
612
- s3_client = boto3.client("s3")
613
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
614
- s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
615
-
616
- rc, error_message = check_s3_bucket(s3_bucket_name)
617
- if rc is False:
618
- raise Exception(error_message)
619
-
620
- with tempfile.NamedTemporaryFile() as tmp:
621
- state.to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
622
-
623
- s3_client.upload_file(
624
- tmp.name,
625
- s3_bucket_name,
626
- s3_bucket_key,
627
- # Casting all the values under metadata to string
628
- ExtraArgs={"Metadata": {k: str(v) for k, v in metadata.items()}},
629
- )
630
- RecceCloud(token=self.cloud_options.get("token")).update_github_pull_request_check(self.pr_info, metadata)
631
- return f"The state file is uploaded to ' s3://{s3_bucket_name}/{s3_bucket_key}'"
632
-
633
- def upload_state_to_cloud(self, state: RecceState) -> Union[str, None]:
634
- if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
635
- raise Exception("Cannot get the pull request information from GitHub.")
636
-
637
- checks = state.checks
638
-
639
- metadata = {
640
- "total_checks": len(checks),
641
- "approved_checks": len([c for c in checks if c.is_checked]),
642
- }
643
-
644
- if self.cloud_options.get("host", "").startswith("s3://"):
645
- return self._upload_state_to_s3_bucket(state, metadata)
646
- else:
647
- return self._upload_state_to_recce_cloud(state, metadata)
648
-
649
- def _download_state_from_s3_bucket(self, filepath):
650
- import io
651
-
652
- import boto3
653
-
654
- s3_client = boto3.client("s3")
655
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
656
- s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
657
-
658
- rc, error_message = check_s3_bucket(s3_bucket_name)
659
- if rc is False:
660
- raise Exception(error_message)
661
-
662
- response = s3_client.get_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
663
- byte_stream = io.BytesIO(response["Body"].read())
664
- gzip_io = file_io_factory(SupportedFileTypes.GZIP)
665
- decompressed_content = gzip_io.read_fileobj(byte_stream)
666
-
667
- dirs = os.path.dirname(filepath)
668
- if dirs:
669
- os.makedirs(dirs, exist_ok=True)
670
- with open(filepath, "wb") as f:
671
- f.write(decompressed_content)
672
-
673
- def _download_state_from_recce_cloud(self, filepath):
674
- import io
675
-
676
- import requests
677
-
678
- presigned_url = RecceCloud(token=self.cloud_options.get("token")).get_presigned_url(
679
- method=PresignedUrlMethod.DOWNLOAD,
680
- repository=self.pr_info.repository,
681
- artifact_name=RECCE_STATE_COMPRESSED_FILE,
682
- pr_id=self.pr_info.id,
683
- )
684
-
685
- password = self.cloud_options.get("password")
686
- if password is None:
687
- raise Exception(RECCE_CLOUD_PASSWORD_MISSING.error_message)
688
-
689
- headers = s3_sse_c_headers(password)
690
- response = requests.get(presigned_url, headers=headers)
691
-
692
- if response.status_code != 200:
693
- raise Exception(
694
- f"{response.status_code} Failed to download the state file from Recce Cloud. The password could be wrong."
695
- )
696
-
697
- byte_stream = io.BytesIO(response.content)
698
- gzip_io = file_io_factory(SupportedFileTypes.GZIP)
699
- decompressed_content = gzip_io.read_fileobj(byte_stream)
700
-
701
- dirs = os.path.dirname(filepath)
702
- if dirs:
703
- os.makedirs(dirs, exist_ok=True)
704
- with open(filepath, "wb") as f:
705
- f.write(decompressed_content)
706
-
707
- def download_state_from_cloud(self, filepath: str) -> Union[str, None]:
708
- if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
709
- raise Exception("Cannot get the pull request information from GitHub.")
710
-
711
- if self.cloud_options.get("host", "").startswith("s3://"):
712
- logger.debug("Download state file from AWS S3 bucket...")
713
- return self._download_state_from_s3_bucket(filepath)
714
- else:
715
- logger.debug("Download state file from Recce Cloud...")
716
- return self._download_state_from_recce_cloud(filepath)
717
-
718
- def _purge_state_from_s3_bucket(self) -> (bool, str):
719
- import boto3
720
- from rich.console import Console
721
-
722
- console = Console()
723
- delete_objects = []
724
- logger.debug("Purging the state from AWS S3 bucket...")
725
- s3_client = boto3.client("s3")
726
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
727
- s3_key_prefix = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/"
728
- list_response = s3_client.list_objects_v2(Bucket=s3_bucket_name, Prefix=s3_key_prefix)
729
- if "Contents" in list_response:
730
- for obj in list_response["Contents"]:
731
- key = obj["Key"]
732
- delete_objects.append({"Key": key})
733
- console.print(f"[green]Deleted[/green]: {key}")
734
- else:
735
- return False, "No state file found in the S3 bucket."
736
-
737
- delete_response = s3_client.delete_objects(Bucket=s3_bucket_name, Delete={"Objects": delete_objects})
738
- if "Deleted" not in delete_response:
739
- return False, "Failed to delete the state file from the S3 bucket."
740
- RecceCloud(token=self.cloud_options.get("token")).update_github_pull_request_check(self.pr_info)
741
- return True, None
742
-
743
- def _purge_state_from_recce_cloud(self) -> (bool, str):
744
- try:
745
- RecceCloud(token=self.cloud_options.get("token")).purge_artifacts(self.pr_info)
746
- except RecceCloudException as e:
747
- return False, e.reason
748
- return True, None
749
-
750
- def purge_cloud_state(self) -> (bool, str):
751
- if self.cloud_options.get("host", "").startswith("s3://"):
752
- return self._purge_state_from_s3_bucket()
753
- else:
754
- return self._purge_state_from_recce_cloud()
755
-
756
-
757
- class RecceShareStateManager:
758
- error_message: str
759
- hint_message: str
760
-
761
- # It is a class to share state file on Recce Cloud.
762
-
763
- def __init__(self, auth_options: Optional[Dict[str, str]] = None):
764
- self.auth_options = auth_options or {}
765
- self.error_message = None
766
- self.hint_message = None
767
-
768
- def verify(self) -> bool:
769
- if self.auth_options.get("api_token") is None:
770
- self.error_message = RECCE_API_TOKEN_MISSING.error_message
771
- self.hint_message = RECCE_API_TOKEN_MISSING.hint_message
772
- return False
773
- return True
774
-
775
- @property
776
- def error_and_hint(self) -> (Union[str, None], Union[str, None]):
777
- return self.error_message, self.hint_message
778
-
779
- def share_state(self, file_name: str, state: RecceState) -> Dict:
780
- import tempfile
781
-
782
- with tempfile.NamedTemporaryFile() as tmp:
783
- state.to_file(tmp.name, file_type=SupportedFileTypes.FILE)
784
- response = RecceCloud(token=self.auth_options.get("api_token")).share_state(file_name, open(tmp.name, "rb"))
785
- return response