recce-nightly 1.15.0.20250806__py3-none-any.whl → 1.26.0.20251124__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of recce-nightly might be problematic. Click here for more details.

Files changed (167) hide show
  1. recce/VERSION +1 -1
  2. recce/__init__.py +5 -0
  3. recce/adapter/dbt_adapter/__init__.py +12 -3
  4. recce/artifact.py +74 -1
  5. recce/cli.py +642 -101
  6. recce/config.py +2 -2
  7. recce/connect_to_cloud.py +1 -1
  8. recce/core.py +2 -2
  9. recce/data/404.html +1 -1
  10. recce/data/__next.__PAGE__.txt +10 -0
  11. recce/data/__next._full.txt +23 -0
  12. recce/data/__next._head.txt +8 -0
  13. recce/data/__next._index.txt +8 -0
  14. recce/data/__next._tree.txt +5 -0
  15. recce/data/_next/static/52aV_JrNUZU6dMFgvTQEO/_buildManifest.js +11 -0
  16. recce/data/_next/static/52aV_JrNUZU6dMFgvTQEO/_clientMiddlewareManifest.json +1 -0
  17. recce/data/_next/static/chunks/02b996c7f6a29a06.js +4 -0
  18. recce/data/_next/static/chunks/19c10d219a6a21ff.js +1 -0
  19. recce/data/_next/static/chunks/2df9ec28a061971d.js +11 -0
  20. recce/data/_next/static/chunks/3098c987393bda15.js +1 -0
  21. recce/data/_next/static/chunks/393dc43e483f717a.css +2 -0
  22. recce/data/_next/static/chunks/399e8d91a7e45073.js +2 -0
  23. recce/data/_next/static/chunks/4d0186f631230245.js +1 -0
  24. recce/data/_next/static/chunks/5794ba9e10a9c060.js +11 -0
  25. recce/data/_next/static/chunks/715761c929a3f28b.js +110 -0
  26. recce/data/_next/static/chunks/71f88fcc615bf282.js +1 -0
  27. recce/data/_next/static/chunks/80d2a95eaf1201ea.js +1 -0
  28. recce/data/_next/static/chunks/9979c6109bbbee35.js +1 -0
  29. recce/data/_next/static/chunks/99d638224186c118.js +1 -0
  30. recce/data/_next/static/chunks/d003eb36240e92f3.js +1 -0
  31. recce/data/_next/static/chunks/d3167cdfec4fc351.js +1 -0
  32. recce/data/_next/static/chunks/e124bccf574a3361.css +1 -0
  33. recce/data/_next/static/chunks/f40141db1bdb46f0.css +6 -0
  34. recce/data/_next/static/chunks/fcc53a88741a52f9.js +1 -0
  35. recce/data/_next/static/chunks/turbopack-b1920d28cfb1f28d.js +3 -0
  36. recce/data/_next/static/media/favicon.a8d38d84.ico +0 -0
  37. recce/data/_next/static/media/montserrat-cyrillic-800-normal.d80d830d.woff2 +0 -0
  38. recce/data/_next/static/media/{montserrat-cyrillic-800-normal.bd5c9f50.woff → montserrat-cyrillic-800-normal.f9d58125.woff} +0 -0
  39. recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.076c2a93.woff2 +0 -0
  40. recce/data/_next/static/media/montserrat-latin-800-normal.cde454cc.woff2 +0 -0
  41. recce/data/_next/static/media/{montserrat-latin-800-normal.fc315020.woff → montserrat-latin-800-normal.d5761935.woff} +0 -0
  42. recce/data/_next/static/media/montserrat-latin-ext-800-normal.40ec0659.woff2 +0 -0
  43. recce/data/_next/static/media/{montserrat-latin-ext-800-normal.2e5381b2.woff → montserrat-latin-ext-800-normal.b671449b.woff} +0 -0
  44. recce/data/_next/static/media/{montserrat-vietnamese-800-normal.20c545e6.woff → montserrat-vietnamese-800-normal.9f7b8541.woff} +0 -0
  45. recce/data/_next/static/media/montserrat-vietnamese-800-normal.f9eb854e.woff2 +0 -0
  46. recce/data/_not-found/__next._full.txt +17 -0
  47. recce/data/_not-found/__next._head.txt +8 -0
  48. recce/data/_not-found/__next._index.txt +8 -0
  49. recce/data/_not-found/__next._not-found.__PAGE__.txt +5 -0
  50. recce/data/_not-found/__next._not-found.txt +4 -0
  51. recce/data/_not-found/__next._tree.txt +3 -0
  52. recce/data/_not-found.html +1 -0
  53. recce/data/_not-found.txt +17 -0
  54. recce/data/index.html +1 -1
  55. recce/data/index.txt +21 -23
  56. recce/event/__init__.py +9 -8
  57. recce/event/collector.py +3 -1
  58. recce/event/track.py +10 -0
  59. recce/github.py +1 -1
  60. recce/mcp_server.py +716 -0
  61. recce/models/types.py +35 -2
  62. recce/pull_request.py +1 -1
  63. recce/run.py +2 -2
  64. recce/server.py +105 -3
  65. recce/state/__init__.py +31 -0
  66. recce/state/cloud.py +632 -0
  67. recce/state/const.py +26 -0
  68. recce/state/local.py +56 -0
  69. recce/state/state.py +119 -0
  70. recce/state/state_loader.py +174 -0
  71. recce/summary.py +21 -1
  72. recce/tasks/dataframe.py +63 -1
  73. recce/tasks/rowcount.py +4 -1
  74. recce/tasks/schema.py +4 -1
  75. recce/util/api_token.py +9 -2
  76. recce/util/breaking.py +1 -1
  77. recce/util/io.py +2 -2
  78. recce/util/lineage.py +14 -18
  79. recce/util/recce_cloud.py +187 -7
  80. recce/yaml/__init__.py +2 -2
  81. recce_cloud/__init__.py +24 -0
  82. recce_cloud/api/__init__.py +17 -0
  83. recce_cloud/api/base.py +111 -0
  84. recce_cloud/api/client.py +150 -0
  85. recce_cloud/api/exceptions.py +26 -0
  86. recce_cloud/api/factory.py +63 -0
  87. recce_cloud/api/github.py +76 -0
  88. recce_cloud/api/gitlab.py +82 -0
  89. recce_cloud/artifact.py +57 -0
  90. recce_cloud/ci_providers/__init__.py +9 -0
  91. recce_cloud/ci_providers/base.py +82 -0
  92. recce_cloud/ci_providers/detector.py +147 -0
  93. recce_cloud/ci_providers/github_actions.py +136 -0
  94. recce_cloud/ci_providers/gitlab_ci.py +130 -0
  95. recce_cloud/cli.py +245 -0
  96. recce_cloud/upload.py +214 -0
  97. {recce_nightly-1.15.0.20250806.dist-info → recce_nightly-1.26.0.20251124.dist-info}/METADATA +54 -28
  98. recce_nightly-1.26.0.20251124.dist-info/RECORD +180 -0
  99. {recce_nightly-1.15.0.20250806.dist-info → recce_nightly-1.26.0.20251124.dist-info}/top_level.txt +1 -0
  100. tests/adapter/dbt_adapter/test_dbt_cll.py +4 -2
  101. tests/recce_cloud/__init__.py +0 -0
  102. tests/recce_cloud/test_ci_providers.py +351 -0
  103. tests/recce_cloud/test_cli.py +372 -0
  104. tests/recce_cloud/test_client.py +273 -0
  105. tests/recce_cloud/test_platform_clients.py +333 -0
  106. tests/test_cli.py +106 -3
  107. tests/test_cli_mcp_optional.py +45 -0
  108. tests/test_cloud_listing_cli.py +324 -0
  109. tests/test_core.py +147 -0
  110. tests/test_mcp_server.py +332 -0
  111. tests/test_server.py +6 -6
  112. tests/test_summary.py +14 -6
  113. recce/data/_next/static/Q_5ThPsmamd4VAGXuqwgi/_buildManifest.js +0 -1
  114. recce/data/_next/static/chunks/0376eeba-3db2196398d62270.js +0 -1
  115. recce/data/_next/static/chunks/068b80ea-833a129468ee1622.js +0 -1
  116. recce/data/_next/static/chunks/0ddaf06c-c7961285f66460f6.js +0 -1
  117. recce/data/_next/static/chunks/1268aea1-6dc1251c01bd724b.js +0 -54
  118. recce/data/_next/static/chunks/12f8fac4-16838e42d28d45c3.js +0 -1
  119. recce/data/_next/static/chunks/235b8375-8c84c51d7bd4f6aa.js +0 -1
  120. recce/data/_next/static/chunks/2541941f-2cd3a7c2d629bd33.js +0 -1
  121. recce/data/_next/static/chunks/273-f3fa401bd2b6fc91.js +0 -10
  122. recce/data/_next/static/chunks/2fc37c1e-910deebeb3d77c90.js +0 -1
  123. recce/data/_next/static/chunks/338-2e7eed5135c64550.js +0 -30
  124. recce/data/_next/static/chunks/367-ab8b16dd5f8586ca.js +0 -1
  125. recce/data/_next/static/chunks/3a92ee20-0400ffe460c7c803.js +0 -1
  126. recce/data/_next/static/chunks/62446465-423c03bb8c1f59b6.js +0 -1
  127. recce/data/_next/static/chunks/6af7f9e9-60aa8706f49dae45.js +0 -1
  128. recce/data/_next/static/chunks/6cf54382-49d52ae6e564e2ac.js +0 -1
  129. recce/data/_next/static/chunks/6dc81886-78e2efe4538794ae.js +0 -1
  130. recce/data/_next/static/chunks/715e4acc-9e2e6df4eb3809d1.js +0 -1
  131. recce/data/_next/static/chunks/72-181b430654230f0e.js +0 -1
  132. recce/data/_next/static/chunks/786-774e3e3ed70a41b3.js +0 -1
  133. recce/data/_next/static/chunks/8d700b6a.7fe2c8c3f4e333a6.js +0 -1
  134. recce/data/_next/static/chunks/a69d64b4-d6890125a87b0aba.js +0 -1
  135. recce/data/_next/static/chunks/ae307f12-01100009689ace61.js +0 -1
  136. recce/data/_next/static/chunks/app/_not-found/page-c7ef8ed6dc07aaeb.js +0 -1
  137. recce/data/_next/static/chunks/app/layout-744f0a78e9e50e60.js +0 -1
  138. recce/data/_next/static/chunks/app/page-e8f798c2ae3f59c2.js +0 -1
  139. recce/data/_next/static/chunks/c0015c5c-82c219792582c104.js +0 -1
  140. recce/data/_next/static/chunks/d90cfbaa-e7d779b3912afeec.js +0 -1
  141. recce/data/_next/static/chunks/e07c302e-cd170429646873e1.js +0 -1
  142. recce/data/_next/static/chunks/fa5fb511-15fb438349ad5b97.js +0 -1
  143. recce/data/_next/static/chunks/framework-7950757d31580329.js +0 -1
  144. recce/data/_next/static/chunks/main-app-4df79eb11c34d43c.js +0 -1
  145. recce/data/_next/static/chunks/main-cd6c104af638214a.js +0 -1
  146. recce/data/_next/static/chunks/pages/_app-73008661edbd5e05.js +0 -1
  147. recce/data/_next/static/chunks/pages/_error-cf8bbdc3cf76c83f.js +0 -1
  148. recce/data/_next/static/chunks/webpack-84df6dd5ae3cf908.js +0 -1
  149. recce/data/_next/static/css/188a3a1687e2a064.css +0 -1
  150. recce/data/_next/static/css/8edca58d4abcf908.css +0 -14
  151. recce/data/_next/static/css/abdb9814a3dd18bb.css +0 -1
  152. recce/data/_next/static/css/c21263c1520b615b.css +0 -1
  153. recce/data/_next/static/media/montserrat-cyrillic-800-normal.22628180.woff2 +0 -0
  154. recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.94a63aea.woff2 +0 -0
  155. recce/data/_next/static/media/montserrat-latin-800-normal.6f8fa298.woff2 +0 -0
  156. recce/data/_next/static/media/montserrat-latin-ext-800-normal.013b84f9.woff2 +0 -0
  157. recce/data/_next/static/media/montserrat-vietnamese-800-normal.c0035377.woff2 +0 -0
  158. recce/state.py +0 -865
  159. recce_nightly-1.15.0.20250806.dist-info/RECORD +0 -156
  160. tests/test_state.py +0 -134
  161. /recce/data/_next/static/{Q_5ThPsmamd4VAGXuqwgi → 52aV_JrNUZU6dMFgvTQEO}/_ssgManifest.js +0 -0
  162. /recce/data/_next/static/chunks/{polyfills-42372ed130431b0a.js → a6dad97d9634a72d.js} +0 -0
  163. /recce/data/_next/static/media/{montserrat-cyrillic-ext-800-normal.e6e0d8d0.woff → montserrat-cyrillic-ext-800-normal.a4fa76b5.woff} +0 -0
  164. /recce/data/_next/static/media/{reload-image.79aabb7d.svg → reload-image.7aa931c7.svg} +0 -0
  165. {recce_nightly-1.15.0.20250806.dist-info → recce_nightly-1.26.0.20251124.dist-info}/WHEEL +0 -0
  166. {recce_nightly-1.15.0.20250806.dist-info → recce_nightly-1.26.0.20251124.dist-info}/entry_points.txt +0 -0
  167. {recce_nightly-1.15.0.20250806.dist-info → recce_nightly-1.26.0.20251124.dist-info}/licenses/LICENSE +0 -0
recce/state.py DELETED
@@ -1,865 +0,0 @@
1
- """Define the type to serialize/de-serialize the state of the recce instance."""
2
-
3
- import json
4
- import logging
5
- import os
6
- import threading
7
- import time
8
- from base64 import b64encode
9
- from dataclasses import dataclass
10
- from datetime import datetime
11
- from hashlib import md5, sha256
12
- from typing import Dict, List, Literal, Optional, Tuple, Union
13
- from urllib.parse import urlencode
14
-
15
- import botocore.exceptions
16
- from pydantic import BaseModel, Field
17
-
18
- from recce import get_version
19
- from recce.event import get_recce_api_token
20
- from recce.exceptions import RecceException
21
- from recce.git import current_branch
22
- from recce.models import CheckDAO
23
- from recce.models.types import Check, Run
24
- from recce.pull_request import PullRequestInfo, fetch_pr_metadata
25
- from recce.util.io import SupportedFileTypes, file_io_factory
26
- from recce.util.pydantic_model import pydantic_model_dump, pydantic_model_json_dump
27
- from recce.util.recce_cloud import PresignedUrlMethod, RecceCloud, RecceCloudException
28
-
29
- logger = logging.getLogger("uvicorn")
30
-
31
- RECCE_STATE_FILE = "recce-state.json"
32
- RECCE_STATE_COMPRESSED_FILE = f"{RECCE_STATE_FILE}.gz"
33
-
34
-
35
- @dataclass
36
- class ErrorMessage:
37
- error_message: str
38
- hint_message: str
39
-
40
-
41
- RECCE_CLOUD_TOKEN_MISSING = ErrorMessage(
42
- error_message="No GitHub token is provided to access the pull request information",
43
- hint_message="Please provide a GitHub token in the command argument",
44
- )
45
-
46
- RECCE_CLOUD_PASSWORD_MISSING = ErrorMessage(
47
- error_message="No password provided to access the state file in Recce Cloud",
48
- hint_message='Please provide a password with the option "--password <compress-password>"',
49
- )
50
-
51
- RECCE_API_TOKEN_MISSING = ErrorMessage(
52
- error_message="No Recc API token is provided",
53
- hint_message="Please login to Recce Cloud and copy the API token from the settings page",
54
- )
55
-
56
-
57
- def s3_sse_c_headers(password: str) -> Dict[str, str]:
58
- hashed_password = sha256()
59
- md5_hash = md5()
60
- hashed_password.update(password.encode())
61
- md5_hash.update(hashed_password.digest())
62
- encoded_passwd = b64encode(hashed_password.digest()).decode("utf-8")
63
- encoded_md5 = b64encode(md5_hash.digest()).decode("utf-8")
64
- return {
65
- "x-amz-server-side-encryption-customer-algorithm": "AES256",
66
- "x-amz-server-side-encryption-customer-key": encoded_passwd,
67
- "x-amz-server-side-encryption-customer-key-MD5": encoded_md5,
68
- }
69
-
70
-
71
- def check_s3_bucket(bucket_name: str):
72
- import boto3
73
-
74
- s3_client = boto3.client("s3")
75
- try:
76
- s3_client.head_bucket(Bucket=bucket_name)
77
- except botocore.exceptions.ClientError as e:
78
- error_code = e.response["Error"]["Code"]
79
- if error_code == "404":
80
- return False, f"Bucket '{bucket_name}' does not exist."
81
- elif error_code == "403":
82
- return False, f"Bucket '{bucket_name}' exists but you do not have permission to access it."
83
- else:
84
- return False, f"Failed to access the S3 bucket: '{bucket_name}'"
85
- return True, None
86
-
87
-
88
- class GitRepoInfo(BaseModel):
89
- branch: Optional[str] = None
90
-
91
- @staticmethod
92
- def from_current_repositroy():
93
- branch = current_branch()
94
- if branch is None:
95
- return None
96
-
97
- return GitRepoInfo(branch=branch)
98
-
99
- def to_dict(self):
100
- return pydantic_model_dump(self)
101
-
102
-
103
- class RecceStateMetadata(BaseModel):
104
- schema_version: str = "v0"
105
- recce_version: str = Field(default_factory=lambda: get_version())
106
- generated_at: str = Field(default_factory=lambda: datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"))
107
-
108
-
109
- class ArtifactsRoot(BaseModel):
110
- """
111
- Root of the artifacts.
112
-
113
- base: artifacts of the base env. key is file name, value is dict
114
- current: artifacts of the current env. key is file name, value is dict
115
- """
116
-
117
- base: Dict[str, Optional[dict]] = {}
118
- current: Dict[str, Optional[dict]] = {}
119
-
120
-
121
- class RecceState(BaseModel):
122
- metadata: Optional[RecceStateMetadata] = None
123
- runs: Optional[List[Run]] = Field(default_factory=list)
124
- checks: Optional[List[Check]] = Field(default_factory=list)
125
- artifacts: ArtifactsRoot = ArtifactsRoot(base={}, current={})
126
- git: Optional[GitRepoInfo] = None
127
- pull_request: Optional[PullRequestInfo] = None
128
-
129
- @staticmethod
130
- def from_json(json_content: str):
131
- dict_data = json.loads(json_content)
132
- state = RecceState(**dict_data)
133
- metadata = state.metadata
134
- if metadata:
135
- if metadata.schema_version is None:
136
- pass
137
- if metadata.schema_version == "v0":
138
- pass
139
- else:
140
- raise RecceException(f"Unsupported state file version: {metadata.schema_version}")
141
- return state
142
-
143
- @staticmethod
144
- def from_file(file_path: str, file_type: SupportedFileTypes = SupportedFileTypes.FILE):
145
- """
146
- Load the state from a recce state file.
147
- """
148
- from pathlib import Path
149
-
150
- logger.debug(f"Load state file from: '{file_path}'")
151
- if not Path(file_path).is_file():
152
- return None
153
-
154
- io = file_io_factory(file_type)
155
- json_content = io.read(file_path)
156
- return RecceState.from_json(json_content)
157
-
158
- def to_json(self):
159
- return pydantic_model_json_dump(self)
160
-
161
- def to_file(self, file_path: str, file_type: SupportedFileTypes = SupportedFileTypes.FILE):
162
-
163
- json_data = self.to_json()
164
- io = file_io_factory(file_type)
165
-
166
- io.write(file_path, json_data)
167
- return f"The state file is stored at '{file_path}'"
168
-
169
- def _merge_run(self, run: Run):
170
- for r in self.runs:
171
- if r.run_id == run.run_id:
172
- break
173
- else:
174
- self.runs.append(run)
175
-
176
- def _merge_check(self, check: Check):
177
- for c in self.checks:
178
- if c.check_id == check.check_id:
179
- c.merge(check)
180
- break
181
- else:
182
- self.checks.append(check)
183
-
184
- def _merge_artifacts(self, artifacts: ArtifactsRoot):
185
- self.artifacts.merge(artifacts)
186
-
187
-
188
- class RecceStateLoader:
189
- def __init__(
190
- self,
191
- review_mode: bool = False,
192
- cloud_mode: bool = False,
193
- state_file: Optional[str] = None,
194
- cloud_options: Optional[Dict[str, str]] = None,
195
- initial_state: Optional[RecceState] = None,
196
- ):
197
- self.review_mode = review_mode
198
- self.cloud_mode = cloud_mode
199
- self.state_file = state_file
200
- self.cloud_options = cloud_options or {}
201
- self.error_message = None
202
- self.hint_message = None
203
- self.state: RecceState | None = initial_state
204
- self.state_lock = threading.Lock()
205
- self.state_etag = None
206
- self.pr_info = None
207
- self.catalog: Literal["github", "preview"] = "github"
208
- self.share_id = None
209
-
210
- if self.cloud_mode:
211
- if self.cloud_options.get("github_token"):
212
- self.catalog = "github"
213
- self.pr_info = fetch_pr_metadata(
214
- cloud=self.cloud_mode, github_token=self.cloud_options.get("github_token")
215
- )
216
- if self.pr_info.id is None:
217
- raise RecceException("Cannot get the pull request information from GitHub.")
218
- elif self.cloud_options.get("api_token"):
219
- self.catalog = "preview"
220
- self.share_id = self.cloud_options.get("share_id")
221
- else:
222
- raise RecceException(RECCE_CLOUD_TOKEN_MISSING.error_message)
223
-
224
- # Load the state
225
- self.load()
226
-
227
- def verify(self) -> bool:
228
- if self.cloud_mode:
229
- if self.catalog == "github":
230
- if self.cloud_options.get("github_token") is None:
231
- self.error_message = RECCE_CLOUD_TOKEN_MISSING.error_message
232
- self.hint_message = RECCE_CLOUD_TOKEN_MISSING.hint_message
233
- return False
234
- if not self.cloud_options.get("host"):
235
- if self.cloud_options.get("password") is None:
236
- self.error_message = RECCE_CLOUD_PASSWORD_MISSING.error_message
237
- self.hint_message = RECCE_CLOUD_PASSWORD_MISSING.hint_message
238
- return False
239
- elif self.catalog == "preview":
240
- if self.cloud_options.get("api_token") is None:
241
- self.error_message = RECCE_API_TOKEN_MISSING.error_message
242
- self.hint_message = RECCE_API_TOKEN_MISSING.hint_message
243
- return False
244
- if self.cloud_options.get("share_id") is None:
245
- self.error_message = "No share ID is provided for the preview catalog."
246
- self.hint_message = (
247
- 'Please provide a share URL in the command argument with option "--share-url <share-url>"'
248
- )
249
- return False
250
-
251
- else:
252
- if self.review_mode is True and self.state_file is None:
253
- self.error_message = "Recce can not launch without a state file."
254
- self.hint_message = "Please provide a state file in the command argument."
255
- return False
256
- pass
257
- return True
258
-
259
- @property
260
- def token(self):
261
- return self.cloud_options.get("github_token") or self.cloud_options.get("api_token")
262
-
263
- @property
264
- def error_and_hint(self) -> (Union[str, None], Union[str, None]):
265
- return self.error_message, self.hint_message
266
-
267
- def update(self, state: RecceState):
268
- self.state = state
269
-
270
- def load(self, refresh=False) -> RecceState:
271
- if self.state is not None and refresh is False:
272
- return self.state
273
- self.state_lock.acquire()
274
- try:
275
- if self.cloud_mode:
276
- self.state, self.state_etag = self._load_state_from_cloud()
277
- elif self.state_file:
278
- self.state = self._load_state_from_file()
279
- finally:
280
- self.state_lock.release()
281
- return self.state
282
-
283
- def save_as(self, state_file: str, state: RecceState = None):
284
- if self.cloud_mode:
285
- raise Exception("Cannot save the state to Recce Cloud.")
286
-
287
- self.state_file = state_file
288
- self.export(state)
289
-
290
- def export(self, state: RecceState = None) -> Union[str, None]:
291
- if state is not None:
292
- self.update(state)
293
- # TODO: Export the current Recce state to file or cloud storage
294
- start_time = time.time()
295
- self.state_lock.acquire()
296
- try:
297
- if self.cloud_mode:
298
- message, state_etag = self._export_state_to_cloud()
299
- self.state_etag = state_etag
300
- else:
301
- if self.state_file is None:
302
- return "No state file is provided. Skip storing the state."
303
- logger.info(f"Store recce state to '{self.state_file}'")
304
- message = self._export_state_to_file()
305
- end_time = time.time()
306
- elapsed_time = end_time - start_time
307
- finally:
308
- self.state_lock.release()
309
- logger.info(f"Store state completed in {elapsed_time:.2f} seconds")
310
- return message
311
-
312
- def refresh(self):
313
- new_state = self.load(refresh=True)
314
- return new_state
315
-
316
- def check_conflict(self) -> bool:
317
- if not self.cloud_mode:
318
- return False
319
-
320
- if self.cloud_options.get("host", "").startswith("s3://"):
321
- return False
322
-
323
- metadata = self._get_metadata_from_recce_cloud()
324
- if not metadata:
325
- return False
326
-
327
- state_etag = metadata.get("etag")
328
- return state_etag != self.state_etag
329
-
330
- def info(self):
331
- if self.state is None:
332
- self.error_message = "No state is loaded."
333
- return None
334
-
335
- state_info = {
336
- "mode": "cloud" if self.cloud_mode else "local",
337
- "source": None,
338
- }
339
- if self.cloud_mode:
340
- if self.cloud_options.get("host", "").startswith("s3://"):
341
- state_info["source"] = self.cloud_options.get("host")
342
- else:
343
- state_info["source"] = "Recce Cloud"
344
- state_info["pull_request"] = self.pr_info
345
- else:
346
- state_info["source"] = self.state_file
347
- return state_info
348
-
349
- def purge(self) -> bool:
350
- if self.cloud_mode is True:
351
- rc, err_msg = RecceCloudStateManager(self.cloud_options).purge_cloud_state()
352
- if err_msg:
353
- self.error_message = err_msg
354
- return rc
355
- else:
356
- if self.state_file is not None:
357
- try:
358
- os.remove(self.state_file)
359
- except Exception as e:
360
- self.error_message = f"Failed to remove the state file: {e}"
361
- return False
362
- else:
363
- self.error_message = "No state file is provided. Skip removing the state file."
364
- return False
365
-
366
- def _load_state_from_file(self, file_path: Optional[str] = None) -> RecceState:
367
- file_path = file_path or self.state_file
368
- return RecceState.from_file(file_path) if file_path else None
369
-
370
- def _load_state_from_cloud(self) -> Tuple[RecceState, str]:
371
- """
372
- Load the state from Recce Cloud.
373
-
374
- Returns:
375
- RecceState: The state object.
376
- str: The etag of the state file.
377
- """
378
- if self.catalog == "github":
379
- if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
380
- raise RecceException("Cannot get the pull request information from GitHub.")
381
- elif self.catalog == "preview":
382
- if self.share_id is None:
383
- raise RecceException("Cannot load the share state from Recce Cloud. No share ID is provided.")
384
-
385
- if self.cloud_options.get("host", "").startswith("s3://"):
386
- logger.debug("Fetching state from AWS S3 bucket...")
387
- return self._load_state_from_s3_bucket(), None
388
- else:
389
- logger.debug("Fetching state from Recce Cloud...")
390
- metadata = self._get_metadata_from_recce_cloud()
391
- if metadata:
392
- state_etag = metadata.get("etag")
393
- else:
394
- state_etag = None
395
- if self.state_etag and state_etag == self.state_etag:
396
- return self.state, self.state_etag
397
-
398
- return self._load_state_from_recce_cloud(), state_etag
399
-
400
- def _get_metadata_from_recce_cloud(self) -> Union[dict, None]:
401
- recce_cloud = RecceCloud(token=self.token)
402
- return recce_cloud.get_artifact_metadata(pr_info=self.pr_info) if self.pr_info else None
403
-
404
- def _load_state_from_recce_cloud(self) -> Union[RecceState, None]:
405
- import tempfile
406
-
407
- import requests
408
-
409
- recce_cloud = RecceCloud(token=self.token)
410
- password = None
411
-
412
- if self.catalog == "github":
413
- presigned_url = recce_cloud.get_presigned_url_by_github_repo(
414
- method=PresignedUrlMethod.DOWNLOAD,
415
- pr_id=self.pr_info.id,
416
- repository=self.pr_info.repository,
417
- artifact_name=RECCE_STATE_COMPRESSED_FILE,
418
- )
419
-
420
- password = self.cloud_options.get("password")
421
- if password is None:
422
- raise RecceException(RECCE_CLOUD_PASSWORD_MISSING.error_message)
423
- elif self.catalog == "preview":
424
- share_id = self.cloud_options.get("share_id")
425
- presigned_url = recce_cloud.get_presigned_url_by_share_id(
426
- method=PresignedUrlMethod.DOWNLOAD, share_id=share_id
427
- )
428
-
429
- with tempfile.NamedTemporaryFile() as tmp:
430
- headers = s3_sse_c_headers(password) if password else None
431
- response = requests.get(presigned_url, headers=headers)
432
- if response.status_code == 404:
433
- self.error_message = "The state file is not found in Recce Cloud."
434
- return None
435
- elif response.status_code != 200:
436
- self.error_message = response.text
437
- raise RecceException(
438
- f"{response.status_code} Failed to download the state file from Recce Cloud. The password could be wrong."
439
- )
440
- with open(tmp.name, "wb") as f:
441
- f.write(response.content)
442
-
443
- file_type = SupportedFileTypes.GZIP if self.catalog == "github" else SupportedFileTypes.FILE
444
- return RecceState.from_file(tmp.name, file_type=file_type)
445
-
446
- def _load_state_from_s3_bucket(self) -> Union[RecceState, None]:
447
- import tempfile
448
-
449
- import boto3
450
-
451
- s3_client = boto3.client("s3")
452
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
453
-
454
- if self.catalog == "github":
455
- s3_bucket_key = (
456
- f"{self.catalog}/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
457
- )
458
- elif self.catalog == "preview":
459
- s3_bucket_key = f"{self.catalog}/{self.share_id}/{RECCE_STATE_FILE}"
460
- else:
461
- raise RecceException(f"Unsupported catalog type. {self.catalog} is not supported.")
462
-
463
- rc, error_message = check_s3_bucket(s3_bucket_name)
464
- if rc is False:
465
- raise RecceException(error_message)
466
-
467
- with tempfile.NamedTemporaryFile() as tmp:
468
- try:
469
- s3_client.download_file(s3_bucket_name, s3_bucket_key, tmp.name)
470
- except botocore.exceptions.ClientError as e:
471
- error_code = e.response.get("Error", {}).get("Code")
472
- if error_code == "404":
473
- self.error_message = "The state file is not found in the S3 bucket."
474
- return None
475
- else:
476
- raise e
477
- return RecceState.from_file(tmp.name, file_type=SupportedFileTypes.GZIP)
478
-
479
- def _export_state_to_cloud(self) -> Tuple[Union[str, None], str]:
480
- if self.catalog == "github":
481
- if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
482
- raise RecceException("Cannot get the pull request information from GitHub.")
483
- elif self.catalog == "preview":
484
- pass
485
-
486
- check_status = CheckDAO().status()
487
- metadata = {
488
- "total_checks": check_status.get("total", 0),
489
- "approved_checks": check_status.get("approved", 0),
490
- }
491
-
492
- if self.cloud_options.get("host", "").startswith("s3://"):
493
- logger.info("Store recce state to AWS S3 bucket")
494
- return self._export_state_to_s3_bucket(metadata=metadata), None
495
- else:
496
- logger.info("Store recce state to Recce Cloud")
497
- message = self._export_state_to_recce_cloud(metadata=metadata)
498
- metadata = self._get_metadata_from_recce_cloud()
499
- state_etag = metadata.get("etag") if metadata else None
500
- if message:
501
- logger.warning(message)
502
- return message, state_etag
503
-
504
- def _export_state_to_recce_cloud(self, metadata: dict = None) -> Union[str, None]:
505
- import tempfile
506
-
507
- import requests
508
-
509
- if self.catalog == "github":
510
- presigned_url = RecceCloud(token=self.token).get_presigned_url_by_github_repo(
511
- method=PresignedUrlMethod.UPLOAD,
512
- repository=self.pr_info.repository,
513
- artifact_name=RECCE_STATE_COMPRESSED_FILE,
514
- pr_id=self.pr_info.id,
515
- metadata=metadata,
516
- )
517
- elif self.catalog == "preview":
518
- share_id = self.cloud_options.get("share_id")
519
- presigned_url = RecceCloud(token=self.token).get_presigned_url_by_share_id(
520
- method=PresignedUrlMethod.UPLOAD,
521
- share_id=share_id,
522
- metadata=metadata,
523
- )
524
- compress_passwd = self.cloud_options.get("password")
525
- if compress_passwd:
526
- headers = s3_sse_c_headers(compress_passwd)
527
- else:
528
- headers = {}
529
-
530
- if metadata:
531
- headers["x-amz-tagging"] = urlencode(metadata)
532
- with tempfile.NamedTemporaryFile() as tmp:
533
- if self.catalog == "github":
534
- file_type = SupportedFileTypes.GZIP
535
- elif self.catalog == "preview":
536
- file_type = SupportedFileTypes.FILE
537
- self._export_state_to_file(tmp.name, file_type=file_type)
538
-
539
- with open(tmp.name, "rb") as fd:
540
- response = requests.put(presigned_url, data=fd.read(), headers=headers)
541
- if response.status_code not in [200, 204]:
542
- self.error_message = response.text
543
- return "Failed to upload the state file to Recce Cloud. Reason: " + response.text
544
- return None
545
-
546
- def _export_state_to_s3_bucket(self, metadata: dict = None) -> Union[str, None]:
547
- import tempfile
548
-
549
- import boto3
550
-
551
- s3_client = boto3.client("s3")
552
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
553
- if self.catalog == "github":
554
- s3_bucket_key = (
555
- f"{self.catalog}/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
556
- )
557
- elif self.catalog == "preview":
558
- s3_bucket_key = f"{self.catalog}/{self.share_id}/{RECCE_STATE_FILE}"
559
- else:
560
- raise RecceException(f"Unsupported catalog type. {self.catalog} is not supported.")
561
-
562
- rc, error_message = check_s3_bucket(s3_bucket_name)
563
- if rc is False:
564
- raise RecceException(error_message)
565
-
566
- with tempfile.NamedTemporaryFile() as tmp:
567
- self._export_state_to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
568
-
569
- s3_client.upload_file(
570
- tmp.name,
571
- s3_bucket_name,
572
- s3_bucket_key,
573
- # Casting all the values under metadata to string
574
- ExtraArgs={"Metadata": {k: str(v) for k, v in metadata.items()}},
575
- )
576
- RecceCloud(token=self.token).update_github_pull_request_check(self.pr_info, metadata)
577
- return f"The state file is uploaded to ' s3://{s3_bucket_name}/{s3_bucket_key}'"
578
-
579
- def _get_artifact_metadata_from_s3_bucket(self, artifact_name: str) -> Union[dict, None]:
580
- import boto3
581
-
582
- s3_client = boto3.client("s3")
583
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
584
- s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{artifact_name}"
585
- try:
586
- response = s3_client.head_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
587
- metadata = response["Metadata"]
588
- return metadata
589
- except botocore.exceptions.ClientError as e:
590
- self.error_message = e.response.get("Error", {}).get("Message")
591
- raise RecceException("Failed to get artifact metadata from Recce Cloud.")
592
-
593
- def _export_state_to_file(
594
- self, file_path: Optional[str] = None, file_type: SupportedFileTypes = SupportedFileTypes.FILE
595
- ) -> str:
596
- """
597
- Store the state to a file. Store happens when terminating the server or run instance.
598
- """
599
-
600
- file_path = file_path or self.state_file
601
- json_data = self.state.to_json()
602
- io = file_io_factory(file_type)
603
-
604
- io.write(file_path, json_data)
605
- return f"The state file is stored at '{file_path}'"
606
-
607
-
608
- class RecceCloudStateManager:
609
- error_message: str
610
- hint_message: str
611
-
612
- # It is a class to upload, download and purge the state file on Recce Cloud.
613
-
614
- def __init__(self, cloud_options: Optional[Dict[str, str]] = None):
615
- self.cloud_options = cloud_options or {}
616
- self.pr_info = None
617
- self.error_message = None
618
- self.hint_message = None
619
- self.github_token = self.cloud_options.get("github_token")
620
-
621
- if not self.github_token:
622
- raise RecceException(RECCE_CLOUD_TOKEN_MISSING.error_message)
623
- self.pr_info = fetch_pr_metadata(cloud=True, github_token=self.github_token)
624
- if self.pr_info.id is None:
625
- raise RecceException("Cannot get the pull request information from GitHub.")
626
-
627
- def verify(self) -> bool:
628
- if self.github_token is None:
629
- self.error_message = RECCE_CLOUD_TOKEN_MISSING.error_message
630
- self.hint_message = RECCE_CLOUD_TOKEN_MISSING.hint_message
631
- return False
632
- if self.cloud_options.get("password") is None:
633
- self.error_message = RECCE_CLOUD_PASSWORD_MISSING.error_message
634
- self.hint_message = RECCE_CLOUD_PASSWORD_MISSING.hint_message
635
- return False
636
- return True
637
-
638
- @property
639
- def error_and_hint(self) -> (Union[str, None], Union[str, None]):
640
- return self.error_message, self.hint_message
641
-
642
- def _check_state_in_recce_cloud(self) -> bool:
643
- return RecceCloud(token=self.github_token).check_artifacts_exists(self.pr_info)
644
-
645
- def _check_state_in_s3_bucket(self) -> bool:
646
- import boto3
647
-
648
- s3_client = boto3.client("s3")
649
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
650
- s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
651
- try:
652
- s3_client.head_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
653
- except botocore.exceptions.ClientError as e:
654
- error_code = e.response.get("Error", {}).get("Code")
655
- if error_code == "404":
656
- return False
657
- return True
658
-
659
- def check_cloud_state_exists(self) -> bool:
660
- if self.cloud_options.get("host", "").startswith("s3://"):
661
- return self._check_state_in_s3_bucket()
662
- else:
663
- return self._check_state_in_recce_cloud()
664
-
665
- def _upload_state_to_recce_cloud(self, state: RecceState, metadata: dict = None) -> Union[str, None]:
666
- import tempfile
667
-
668
- import requests
669
-
670
- presigned_url = RecceCloud(token=self.github_token).get_presigned_url_by_github_repo(
671
- method=PresignedUrlMethod.UPLOAD,
672
- repository=self.pr_info.repository,
673
- artifact_name=RECCE_STATE_COMPRESSED_FILE,
674
- pr_id=self.pr_info.id,
675
- metadata=metadata,
676
- )
677
-
678
- compress_passwd = self.cloud_options.get("password")
679
- headers = s3_sse_c_headers(compress_passwd)
680
- with tempfile.NamedTemporaryFile() as tmp:
681
- state.to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
682
- response = requests.put(presigned_url, data=open(tmp.name, "rb").read(), headers=headers)
683
- if response.status_code != 200:
684
- return f"Failed to upload the state file to Recce Cloud. Reason: {response.text}"
685
- return "The state file is uploaded to Recce Cloud."
686
-
687
- def _upload_state_to_s3_bucket(self, state: RecceState, metadata: dict = None) -> Union[str, None]:
688
- import tempfile
689
-
690
- import boto3
691
-
692
- s3_client = boto3.client("s3")
693
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
694
- s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
695
-
696
- rc, error_message = check_s3_bucket(s3_bucket_name)
697
- if rc is False:
698
- raise RecceException(error_message)
699
-
700
- with tempfile.NamedTemporaryFile() as tmp:
701
- state.to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
702
-
703
- s3_client.upload_file(
704
- tmp.name,
705
- s3_bucket_name,
706
- s3_bucket_key,
707
- # Casting all the values under metadata to string
708
- ExtraArgs={"Metadata": {k: str(v) for k, v in metadata.items()}},
709
- )
710
- RecceCloud(token=self.github_token).update_github_pull_request_check(self.pr_info, metadata)
711
- return f"The state file is uploaded to ' s3://{s3_bucket_name}/{s3_bucket_key}'"
712
-
713
- def upload_state_to_cloud(self, state: RecceState) -> Union[str, None]:
714
- if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
715
- raise RecceException("Cannot get the pull request information from GitHub.")
716
-
717
- checks = state.checks
718
-
719
- metadata = {
720
- "total_checks": len(checks),
721
- "approved_checks": len([c for c in checks if c.is_checked]),
722
- }
723
-
724
- if self.cloud_options.get("host", "").startswith("s3://"):
725
- return self._upload_state_to_s3_bucket(state, metadata)
726
- else:
727
- return self._upload_state_to_recce_cloud(state, metadata)
728
-
729
- def _download_state_from_s3_bucket(self, filepath):
730
- import io
731
-
732
- import boto3
733
-
734
- s3_client = boto3.client("s3")
735
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
736
- s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
737
-
738
- rc, error_message = check_s3_bucket(s3_bucket_name)
739
- if rc is False:
740
- raise RecceException(error_message)
741
-
742
- response = s3_client.get_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
743
- byte_stream = io.BytesIO(response["Body"].read())
744
- gzip_io = file_io_factory(SupportedFileTypes.GZIP)
745
- decompressed_content = gzip_io.read_fileobj(byte_stream)
746
-
747
- dirs = os.path.dirname(filepath)
748
- if dirs:
749
- os.makedirs(dirs, exist_ok=True)
750
- with open(filepath, "wb") as f:
751
- f.write(decompressed_content)
752
-
753
- def _download_state_from_recce_cloud(self, filepath):
754
- import io
755
-
756
- import requests
757
-
758
- presigned_url = RecceCloud(token=self.github_token).get_presigned_url_by_github_repo(
759
- method=PresignedUrlMethod.DOWNLOAD,
760
- repository=self.pr_info.repository,
761
- artifact_name=RECCE_STATE_COMPRESSED_FILE,
762
- pr_id=self.pr_info.id,
763
- )
764
-
765
- password = self.cloud_options.get("password")
766
- if password is None:
767
- raise RecceException(RECCE_CLOUD_PASSWORD_MISSING.error_message)
768
-
769
- headers = s3_sse_c_headers(password)
770
- response = requests.get(presigned_url, headers=headers)
771
-
772
- if response.status_code != 200:
773
- raise RecceException(
774
- f"{response.status_code} Failed to download the state file from Recce Cloud. The password could be wrong."
775
- )
776
-
777
- byte_stream = io.BytesIO(response.content)
778
- gzip_io = file_io_factory(SupportedFileTypes.GZIP)
779
- decompressed_content = gzip_io.read_fileobj(byte_stream)
780
-
781
- dirs = os.path.dirname(filepath)
782
- if dirs:
783
- os.makedirs(dirs, exist_ok=True)
784
- with open(filepath, "wb") as f:
785
- f.write(decompressed_content)
786
-
787
- def download_state_from_cloud(self, filepath: str) -> Union[str, None]:
788
- if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
789
- raise RecceException("Cannot get the pull request information from GitHub.")
790
-
791
- if self.cloud_options.get("host", "").startswith("s3://"):
792
- logger.debug("Download state file from AWS S3 bucket...")
793
- return self._download_state_from_s3_bucket(filepath)
794
- else:
795
- logger.debug("Download state file from Recce Cloud...")
796
- return self._download_state_from_recce_cloud(filepath)
797
-
798
- def _purge_state_from_s3_bucket(self) -> (bool, str):
799
- import boto3
800
- from rich.console import Console
801
-
802
- console = Console()
803
- delete_objects = []
804
- logger.debug("Purging the state from AWS S3 bucket...")
805
- s3_client = boto3.client("s3")
806
- s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
807
- s3_key_prefix = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/"
808
- list_response = s3_client.list_objects_v2(Bucket=s3_bucket_name, Prefix=s3_key_prefix)
809
- if "Contents" in list_response:
810
- for obj in list_response["Contents"]:
811
- key = obj["Key"]
812
- delete_objects.append({"Key": key})
813
- console.print(f"[green]Deleted[/green]: {key}")
814
- else:
815
- return False, "No state file found in the S3 bucket."
816
-
817
- delete_response = s3_client.delete_objects(Bucket=s3_bucket_name, Delete={"Objects": delete_objects})
818
- if "Deleted" not in delete_response:
819
- return False, "Failed to delete the state file from the S3 bucket."
820
- RecceCloud(token=self.github_token).update_github_pull_request_check(self.pr_info)
821
- return True, None
822
-
823
- def _purge_state_from_recce_cloud(self) -> (bool, str):
824
- try:
825
- RecceCloud(token=self.github_token).purge_artifacts(self.pr_info)
826
- except RecceCloudException as e:
827
- return False, e.reason
828
- return True, None
829
-
830
- def purge_cloud_state(self) -> (bool, str):
831
- if self.cloud_options.get("host", "").startswith("s3://"):
832
- return self._purge_state_from_s3_bucket()
833
- else:
834
- return self._purge_state_from_recce_cloud()
835
-
836
-
837
- class RecceShareStateManager:
838
- error_message: str
839
- hint_message: str
840
-
841
- # It is a class to share state file on Recce Cloud.
842
-
843
- def __init__(self, auth_options: Optional[Dict[str, str]] = None):
844
- self.auth_options = auth_options or {}
845
- self.error_message = None
846
- self.hint_message = None
847
-
848
- def verify(self) -> bool:
849
- if get_recce_api_token() is None:
850
- self.error_message = RECCE_API_TOKEN_MISSING.error_message
851
- self.hint_message = RECCE_API_TOKEN_MISSING.hint_message
852
- return False
853
- return True
854
-
855
- @property
856
- def error_and_hint(self) -> (Union[str, None], Union[str, None]):
857
- return self.error_message, self.hint_message
858
-
859
- def share_state(self, file_name: str, state: RecceState) -> Dict:
860
- import tempfile
861
-
862
- with tempfile.NamedTemporaryFile() as tmp:
863
- state.to_file(tmp.name, file_type=SupportedFileTypes.FILE)
864
- response = RecceCloud(token=get_recce_api_token()).share_state(file_name, open(tmp.name, "rb"))
865
- return response