recce-nightly 1.3.0.20250507__py3-none-any.whl → 1.4.0.20250515__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of recce-nightly might be problematic. Click here for more details.

Files changed (93) hide show
  1. recce/VERSION +1 -1
  2. recce/__init__.py +22 -22
  3. recce/adapter/base.py +11 -14
  4. recce/adapter/dbt_adapter/__init__.py +355 -316
  5. recce/adapter/dbt_adapter/dbt_version.py +3 -0
  6. recce/adapter/sqlmesh_adapter.py +24 -35
  7. recce/apis/check_api.py +39 -28
  8. recce/apis/check_func.py +33 -27
  9. recce/apis/run_api.py +25 -19
  10. recce/apis/run_func.py +29 -23
  11. recce/artifact.py +44 -49
  12. recce/cli.py +484 -285
  13. recce/config.py +42 -33
  14. recce/core.py +52 -44
  15. recce/data/404.html +1 -1
  16. recce/data/_next/static/chunks/{368-7587b306577df275.js → 778-aef312bffb4c0312.js} +15 -15
  17. recce/data/_next/static/chunks/8d700b6a.ed11a130057c7a47.js +1 -0
  18. recce/data/_next/static/chunks/app/layout-c713a2829d3279e4.js +1 -0
  19. recce/data/_next/static/chunks/app/page-7086764277331fcb.js +1 -0
  20. recce/data/_next/static/chunks/{cd9f8d63-cf0d5a7b0f7a92e8.js → cd9f8d63-e020f408095ed77c.js} +3 -3
  21. recce/data/_next/static/chunks/webpack-b787cb1a4f2293de.js +1 -0
  22. recce/data/_next/static/css/88b8abc134cfd59a.css +3 -0
  23. recce/data/index.html +2 -2
  24. recce/data/index.txt +2 -2
  25. recce/diff.py +6 -12
  26. recce/event/__init__.py +74 -72
  27. recce/event/collector.py +27 -20
  28. recce/event/track.py +39 -27
  29. recce/exceptions.py +1 -1
  30. recce/git.py +7 -7
  31. recce/github.py +57 -53
  32. recce/models/__init__.py +1 -1
  33. recce/models/check.py +6 -7
  34. recce/models/run.py +1 -0
  35. recce/models/types.py +27 -27
  36. recce/pull_request.py +26 -24
  37. recce/run.py +148 -111
  38. recce/server.py +103 -89
  39. recce/state.py +209 -177
  40. recce/summary.py +168 -143
  41. recce/tasks/__init__.py +3 -3
  42. recce/tasks/core.py +11 -13
  43. recce/tasks/dataframe.py +19 -17
  44. recce/tasks/histogram.py +69 -34
  45. recce/tasks/lineage.py +2 -2
  46. recce/tasks/profile.py +147 -86
  47. recce/tasks/query.py +139 -87
  48. recce/tasks/rowcount.py +33 -30
  49. recce/tasks/schema.py +14 -14
  50. recce/tasks/top_k.py +35 -35
  51. recce/tasks/valuediff.py +216 -152
  52. recce/util/breaking.py +77 -84
  53. recce/util/cll.py +55 -51
  54. recce/util/io.py +19 -17
  55. recce/util/logger.py +1 -1
  56. recce/util/recce_cloud.py +70 -72
  57. recce/util/singleton.py +4 -4
  58. recce/yaml/__init__.py +7 -10
  59. {recce_nightly-1.3.0.20250507.dist-info → recce_nightly-1.4.0.20250515.dist-info}/METADATA +5 -2
  60. recce_nightly-1.4.0.20250515.dist-info/RECORD +143 -0
  61. {recce_nightly-1.3.0.20250507.dist-info → recce_nightly-1.4.0.20250515.dist-info}/WHEEL +1 -1
  62. tests/adapter/dbt_adapter/conftest.py +1 -0
  63. tests/adapter/dbt_adapter/dbt_test_helper.py +28 -18
  64. tests/adapter/dbt_adapter/test_dbt_adapter.py +0 -15
  65. tests/adapter/dbt_adapter/test_dbt_cll.py +39 -32
  66. tests/adapter/dbt_adapter/test_selector.py +22 -21
  67. tests/tasks/test_histogram.py +58 -66
  68. tests/tasks/test_lineage.py +36 -23
  69. tests/tasks/test_preset_checks.py +45 -31
  70. tests/tasks/test_profile.py +340 -15
  71. tests/tasks/test_query.py +40 -40
  72. tests/tasks/test_row_count.py +65 -46
  73. tests/tasks/test_schema.py +65 -42
  74. tests/tasks/test_top_k.py +22 -18
  75. tests/tasks/test_valuediff.py +43 -32
  76. tests/test_cli.py +71 -58
  77. tests/test_config.py +7 -9
  78. tests/test_core.py +5 -3
  79. tests/test_dbt.py +7 -7
  80. tests/test_pull_request.py +1 -1
  81. tests/test_server.py +19 -13
  82. tests/test_state.py +40 -27
  83. tests/test_summary.py +18 -14
  84. recce/data/_next/static/chunks/8d700b6a-f0b1f6b9e0d97ce2.js +0 -1
  85. recce/data/_next/static/chunks/app/layout-9102e22cb73f74d6.js +0 -1
  86. recce/data/_next/static/chunks/app/page-92f13c8fad9fae3d.js +0 -1
  87. recce/data/_next/static/chunks/webpack-567d72f0bc0820d5.js +0 -1
  88. recce_nightly-1.3.0.20250507.dist-info/RECORD +0 -142
  89. /recce/data/_next/static/{K5iKlCYhdcpq8Ea6ck9J_ → q0Xsc9Sd6PDuo1lshYpLu}/_buildManifest.js +0 -0
  90. /recce/data/_next/static/{K5iKlCYhdcpq8Ea6ck9J_ → q0Xsc9Sd6PDuo1lshYpLu}/_ssgManifest.js +0 -0
  91. {recce_nightly-1.3.0.20250507.dist-info → recce_nightly-1.4.0.20250515.dist-info}/entry_points.txt +0 -0
  92. {recce_nightly-1.3.0.20250507.dist-info → recce_nightly-1.4.0.20250515.dist-info}/licenses/LICENSE +0 -0
  93. {recce_nightly-1.3.0.20250507.dist-info → recce_nightly-1.4.0.20250515.dist-info}/top_level.txt +0 -0
recce/state.py CHANGED
@@ -1,4 +1,5 @@
1
1
  """Define the type to serialize/de-serialize the state of the recce instance."""
2
+
2
3
  import json
3
4
  import logging
4
5
  import os
@@ -8,26 +9,25 @@ from base64 import b64encode
8
9
  from dataclasses import dataclass
9
10
  from datetime import datetime
10
11
  from hashlib import md5, sha256
11
- from typing import List, Optional, Dict, Union, Tuple
12
+ from typing import Dict, List, Optional, Tuple, Union
12
13
  from urllib.parse import urlencode
13
14
 
14
15
  import botocore.exceptions
15
- from pydantic import BaseModel
16
- from pydantic import Field
16
+ from pydantic import BaseModel, Field
17
17
 
18
18
  from recce import get_version
19
19
  from recce.git import current_branch
20
20
  from recce.models import CheckDAO
21
- from recce.models.types import Run, Check
22
- from recce.pull_request import fetch_pr_metadata, PullRequestInfo
21
+ from recce.models.types import Check, Run
22
+ from recce.pull_request import PullRequestInfo, fetch_pr_metadata
23
23
  from recce.util.io import SupportedFileTypes, file_io_factory
24
- from recce.util.pydantic_model import pydantic_model_json_dump, pydantic_model_dump
25
- from recce.util.recce_cloud import RecceCloud, PresignedUrlMethod, RecceCloudException
24
+ from recce.util.pydantic_model import pydantic_model_dump, pydantic_model_json_dump
25
+ from recce.util.recce_cloud import PresignedUrlMethod, RecceCloud, RecceCloudException
26
26
 
27
- logger = logging.getLogger('uvicorn')
27
+ logger = logging.getLogger("uvicorn")
28
28
 
29
- RECCE_STATE_FILE = 'recce-state.json'
30
- RECCE_STATE_COMPRESSED_FILE = f'{RECCE_STATE_FILE}.gz'
29
+ RECCE_STATE_FILE = "recce-state.json"
30
+ RECCE_STATE_COMPRESSED_FILE = f"{RECCE_STATE_FILE}.gz"
31
31
 
32
32
 
33
33
  @dataclass
@@ -37,18 +37,18 @@ class ErrorMessage:
37
37
 
38
38
 
39
39
  RECCE_CLOUD_TOKEN_MISSING = ErrorMessage(
40
- error_message='No GitHub token is provided to access the pull request information',
41
- hint_message='Please provide a GitHub token in the command argument'
40
+ error_message="No GitHub token is provided to access the pull request information",
41
+ hint_message="Please provide a GitHub token in the command argument",
42
42
  )
43
43
 
44
44
  RECCE_CLOUD_PASSWORD_MISSING = ErrorMessage(
45
- error_message='No password provided to access the state file in Recce Cloud',
46
- hint_message='Please provide a password with the option "--password <compress-password>"'
45
+ error_message="No password provided to access the state file in Recce Cloud",
46
+ hint_message='Please provide a password with the option "--password <compress-password>"',
47
47
  )
48
48
 
49
49
  RECCE_API_TOKEN_MISSING = ErrorMessage(
50
- error_message='No Recc API token is provided',
51
- hint_message='Please login to Recce Cloud and copy the API token from the settings page'
50
+ error_message="No Recc API token is provided",
51
+ hint_message="Please login to Recce Cloud and copy the API token from the settings page",
52
52
  )
53
53
 
54
54
 
@@ -57,25 +57,26 @@ def s3_sse_c_headers(password: str) -> Dict[str, str]:
57
57
  md5_hash = md5()
58
58
  hashed_password.update(password.encode())
59
59
  md5_hash.update(hashed_password.digest())
60
- encoded_passwd = b64encode(hashed_password.digest()).decode('utf-8')
61
- encoded_md5 = b64encode(md5_hash.digest()).decode('utf-8')
60
+ encoded_passwd = b64encode(hashed_password.digest()).decode("utf-8")
61
+ encoded_md5 = b64encode(md5_hash.digest()).decode("utf-8")
62
62
  return {
63
- 'x-amz-server-side-encryption-customer-algorithm': 'AES256',
64
- 'x-amz-server-side-encryption-customer-key': encoded_passwd,
65
- 'x-amz-server-side-encryption-customer-key-MD5': encoded_md5,
63
+ "x-amz-server-side-encryption-customer-algorithm": "AES256",
64
+ "x-amz-server-side-encryption-customer-key": encoded_passwd,
65
+ "x-amz-server-side-encryption-customer-key-MD5": encoded_md5,
66
66
  }
67
67
 
68
68
 
69
69
  def check_s3_bucket(bucket_name: str):
70
70
  import boto3
71
- s3_client = boto3.client('s3')
71
+
72
+ s3_client = boto3.client("s3")
72
73
  try:
73
74
  s3_client.head_bucket(Bucket=bucket_name)
74
75
  except botocore.exceptions.ClientError as e:
75
- error_code = e.response['Error']['Code']
76
- if error_code == '404':
76
+ error_code = e.response["Error"]["Code"]
77
+ if error_code == "404":
77
78
  return False, f"Bucket '{bucket_name}' does not exist."
78
- elif error_code == '403':
79
+ elif error_code == "403":
79
80
  return False, f"Bucket '{bucket_name}' exists but you do not have permission to access it."
80
81
  else:
81
82
  return False, f"Failed to access the S3 bucket: '{bucket_name}'"
@@ -98,7 +99,7 @@ class GitRepoInfo(BaseModel):
98
99
 
99
100
 
100
101
  class RecceStateMetadata(BaseModel):
101
- schema_version: str = 'v0'
102
+ schema_version: str = "v0"
102
103
  recce_version: str = Field(default_factory=lambda: get_version())
103
104
  generated_at: str = Field(default_factory=lambda: datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"))
104
105
 
@@ -110,6 +111,7 @@ class ArtifactsRoot(BaseModel):
110
111
  base: artifacts of the base env. key is file name, value is dict
111
112
  current: artifacts of the current env. key is file name, value is dict
112
113
  """
114
+
113
115
  base: Dict[str, Optional[dict]] = {}
114
116
  current: Dict[str, Optional[dict]] = {}
115
117
 
@@ -130,7 +132,7 @@ class RecceState(BaseModel):
130
132
  if metadata:
131
133
  if metadata.schema_version is None:
132
134
  pass
133
- if metadata.schema_version == 'v0':
135
+ if metadata.schema_version == "v0":
134
136
  pass
135
137
  else:
136
138
  raise Exception(f"Unsupported state file version: {metadata.schema_version}")
@@ -160,7 +162,7 @@ class RecceState(BaseModel):
160
162
  io = file_io_factory(file_type)
161
163
 
162
164
  io.write(file_path, json_data)
163
- return f'The state file is stored at \'{file_path}\''
165
+ return f"The state file is stored at '{file_path}'"
164
166
 
165
167
  def _merge_run(self, run: Run):
166
168
  for r in self.runs:
@@ -182,13 +184,14 @@ class RecceState(BaseModel):
182
184
 
183
185
 
184
186
  class RecceStateLoader:
185
- def __init__(self,
186
- review_mode: bool = False,
187
- cloud_mode: bool = False,
188
- state_file: Optional[str] = None,
189
- cloud_options: Optional[Dict[str, str]] = None,
190
- initial_state: Optional[RecceState] = None,
191
- ):
187
+ def __init__(
188
+ self,
189
+ review_mode: bool = False,
190
+ cloud_mode: bool = False,
191
+ state_file: Optional[str] = None,
192
+ cloud_options: Optional[Dict[str, str]] = None,
193
+ initial_state: Optional[RecceState] = None,
194
+ ):
192
195
  self.review_mode = review_mode
193
196
  self.cloud_mode = cloud_mode
194
197
  self.state_file = state_file
@@ -201,30 +204,30 @@ class RecceStateLoader:
201
204
  self.pr_info = None
202
205
 
203
206
  if self.cloud_mode:
204
- if not self.cloud_options.get('token'):
207
+ if not self.cloud_options.get("token"):
205
208
  raise Exception(RECCE_CLOUD_TOKEN_MISSING.error_message)
206
- self.pr_info = fetch_pr_metadata(cloud=self.cloud_mode, github_token=self.cloud_options.get('token'))
209
+ self.pr_info = fetch_pr_metadata(cloud=self.cloud_mode, github_token=self.cloud_options.get("token"))
207
210
  if self.pr_info.id is None:
208
- raise Exception('Cannot get the pull request information from GitHub.')
211
+ raise Exception("Cannot get the pull request information from GitHub.")
209
212
 
210
213
  # Load the state
211
214
  self.load()
212
215
 
213
216
  def verify(self) -> bool:
214
217
  if self.cloud_mode:
215
- if self.cloud_options.get('token') is None:
218
+ if self.cloud_options.get("token") is None:
216
219
  self.error_message = RECCE_CLOUD_TOKEN_MISSING.error_message
217
220
  self.hint_message = RECCE_CLOUD_TOKEN_MISSING.hint_message
218
221
  return False
219
- if not self.cloud_options.get('host'):
220
- if self.cloud_options.get('password') is None:
222
+ if not self.cloud_options.get("host"):
223
+ if self.cloud_options.get("password") is None:
221
224
  self.error_message = RECCE_CLOUD_PASSWORD_MISSING.error_message
222
225
  self.hint_message = RECCE_CLOUD_PASSWORD_MISSING.hint_message
223
226
  return False
224
227
  else:
225
228
  if self.review_mode is True and self.state_file is None:
226
- self.error_message = 'Recce can not launch without a state file.'
227
- self.hint_message = 'Please provide a state file in the command argument.'
229
+ self.error_message = "Recce can not launch without a state file."
230
+ self.hint_message = "Please provide a state file in the command argument."
228
231
  return False
229
232
  pass
230
233
  return True
@@ -251,7 +254,7 @@ class RecceStateLoader:
251
254
 
252
255
  def save_as(self, state_file: str, state: RecceState = None):
253
256
  if self.cloud_mode:
254
- raise Exception('Cannot save the state to Recce Cloud.')
257
+ raise Exception("Cannot save the state to Recce Cloud.")
255
258
 
256
259
  self.state_file = state_file
257
260
  self.export(state)
@@ -268,14 +271,14 @@ class RecceStateLoader:
268
271
  self.state_etag = state_etag
269
272
  else:
270
273
  if self.state_file is None:
271
- return 'No state file is provided. Skip storing the state.'
274
+ return "No state file is provided. Skip storing the state."
272
275
  logger.info(f"Store recce state to '{self.state_file}'")
273
276
  message = self._export_state_to_file()
274
277
  end_time = time.time()
275
278
  elapsed_time = end_time - start_time
276
279
  finally:
277
280
  self.state_lock.release()
278
- logger.info(f'Store state completed in {elapsed_time:.2f} seconds')
281
+ logger.info(f"Store state completed in {elapsed_time:.2f} seconds")
279
282
  return message
280
283
 
281
284
  def refresh(self):
@@ -286,33 +289,33 @@ class RecceStateLoader:
286
289
  if not self.cloud_mode:
287
290
  return False
288
291
 
289
- if self.cloud_options.get('host', '').startswith('s3://'):
292
+ if self.cloud_options.get("host", "").startswith("s3://"):
290
293
  return False
291
294
 
292
295
  metadata = self._get_metadata_from_recce_cloud()
293
296
  if not metadata:
294
297
  return False
295
298
 
296
- state_etag = metadata.get('etag')
299
+ state_etag = metadata.get("etag")
297
300
  return state_etag != self.state_etag
298
301
 
299
302
  def info(self):
300
303
  if self.state is None:
301
- self.error_message = 'No state is loaded.'
304
+ self.error_message = "No state is loaded."
302
305
  return None
303
306
 
304
307
  state_info = {
305
- 'mode': 'cloud' if self.cloud_mode else 'local',
306
- 'source': None,
308
+ "mode": "cloud" if self.cloud_mode else "local",
309
+ "source": None,
307
310
  }
308
311
  if self.cloud_mode:
309
- if self.cloud_options.get('host', '').startswith('s3://'):
310
- state_info['source'] = self.cloud_options.get('host')
312
+ if self.cloud_options.get("host", "").startswith("s3://"):
313
+ state_info["source"] = self.cloud_options.get("host")
311
314
  else:
312
- state_info['source'] = 'Recce Cloud'
313
- state_info['pull_request'] = self.pr_info
315
+ state_info["source"] = "Recce Cloud"
316
+ state_info["pull_request"] = self.pr_info
314
317
  else:
315
- state_info['source'] = self.state_file
318
+ state_info["source"] = self.state_file
316
319
  return state_info
317
320
 
318
321
  def purge(self) -> bool:
@@ -326,10 +329,10 @@ class RecceStateLoader:
326
329
  try:
327
330
  os.remove(self.state_file)
328
331
  except Exception as e:
329
- self.error_message = f'Failed to remove the state file: {e}'
332
+ self.error_message = f"Failed to remove the state file: {e}"
330
333
  return False
331
334
  else:
332
- self.error_message = 'No state file is provided. Skip removing the state file.'
335
+ self.error_message = "No state file is provided. Skip removing the state file."
333
336
  return False
334
337
 
335
338
  def _load_state_from_file(self, file_path: Optional[str] = None) -> RecceState:
@@ -337,71 +340,74 @@ class RecceStateLoader:
337
340
  return RecceState.from_file(file_path) if file_path else None
338
341
 
339
342
  def _load_state_from_cloud(self) -> Tuple[RecceState, str]:
340
- '''
343
+ """
341
344
  Load the state from Recce Cloud.
342
345
 
343
346
  Returns:
344
347
  RecceState: The state object.
345
348
  str: The etag of the state file.
346
- '''
349
+ """
347
350
  if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
348
- raise Exception('Cannot get the pull request information from GitHub.')
351
+ raise Exception("Cannot get the pull request information from GitHub.")
349
352
 
350
- if self.cloud_options.get('host', '').startswith('s3://'):
351
- logger.debug('Fetching state from AWS S3 bucket...')
353
+ if self.cloud_options.get("host", "").startswith("s3://"):
354
+ logger.debug("Fetching state from AWS S3 bucket...")
352
355
  return self._load_state_from_s3_bucket(), None
353
356
  else:
354
- logger.debug('Fetching state from Recce Cloud...')
357
+ logger.debug("Fetching state from Recce Cloud...")
355
358
  metadata = self._get_metadata_from_recce_cloud()
356
359
  if metadata is None:
357
360
  return None, None
358
- state_etag = metadata.get('etag')
361
+ state_etag = metadata.get("etag")
359
362
  if self.state_etag and state_etag == self.state_etag:
360
363
  return self.state, self.state_etag
361
364
 
362
365
  return self._load_state_from_recce_cloud(), state_etag
363
366
 
364
367
  def _get_metadata_from_recce_cloud(self) -> Union[dict, None]:
365
- recce_cloud = RecceCloud(token=self.cloud_options.get('token'))
368
+ recce_cloud = RecceCloud(token=self.cloud_options.get("token"))
366
369
  return recce_cloud.get_artifact_metadata(pr_info=self.pr_info)
367
370
 
368
371
  def _load_state_from_recce_cloud(self) -> Union[RecceState, None]:
369
372
  import tempfile
373
+
370
374
  import requests
371
375
 
372
- recce_cloud = RecceCloud(token=self.cloud_options.get('token'))
376
+ recce_cloud = RecceCloud(token=self.cloud_options.get("token"))
373
377
  presigned_url = recce_cloud.get_presigned_url(
374
378
  method=PresignedUrlMethod.DOWNLOAD,
375
379
  pr_id=self.pr_info.id,
376
380
  repository=self.pr_info.repository,
377
- artifact_name=RECCE_STATE_COMPRESSED_FILE
381
+ artifact_name=RECCE_STATE_COMPRESSED_FILE,
378
382
  )
379
383
 
380
- password = self.cloud_options.get('password')
384
+ password = self.cloud_options.get("password")
381
385
  if password is None:
382
386
  raise Exception(RECCE_CLOUD_PASSWORD_MISSING.error_message)
383
387
 
384
388
  with tempfile.NamedTemporaryFile() as tmp:
385
389
  headers = s3_sse_c_headers(password)
386
- response = requests.get(presigned_url,
387
- headers=headers)
390
+ response = requests.get(presigned_url, headers=headers)
388
391
  if response.status_code == 404:
389
- self.error_message = 'The state file is not found in Recce Cloud.'
392
+ self.error_message = "The state file is not found in Recce Cloud."
390
393
  return None
391
394
  elif response.status_code != 200:
392
395
  self.error_message = response.text
393
396
  raise Exception(
394
- f'{response.status_code} Failed to download the state file from Recce Cloud. The password could be wrong.')
395
- with open(tmp.name, 'wb') as f:
397
+ f"{response.status_code} Failed to download the state file from Recce Cloud. The password could be wrong."
398
+ )
399
+ with open(tmp.name, "wb") as f:
396
400
  f.write(response.content)
397
401
  return RecceState.from_file(tmp.name, file_type=SupportedFileTypes.GZIP)
398
402
 
399
403
  def _load_state_from_s3_bucket(self) -> Union[RecceState, None]:
400
- import boto3
401
404
  import tempfile
402
- s3_client = boto3.client('s3')
403
- s3_bucket_name = self.cloud_options.get('host').replace('s3://', '')
404
- s3_bucket_key = f'github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}'
405
+
406
+ import boto3
407
+
408
+ s3_client = boto3.client("s3")
409
+ s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
410
+ s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
405
411
 
406
412
  rc, error_message = check_s3_bucket(s3_bucket_name)
407
413
  if rc is False:
@@ -411,9 +417,9 @@ class RecceStateLoader:
411
417
  try:
412
418
  s3_client.download_file(s3_bucket_name, s3_bucket_key, tmp.name)
413
419
  except botocore.exceptions.ClientError as e:
414
- error_code = e.response.get('Error', {}).get('Code')
415
- if error_code == '404':
416
- self.error_message = 'The state file is not found in the S3 bucket.'
420
+ error_code = e.response.get("Error", {}).get("Code")
421
+ if error_code == "404":
422
+ self.error_message = "The state file is not found in the S3 bucket."
417
423
  return None
418
424
  else:
419
425
  raise e
@@ -421,15 +427,15 @@ class RecceStateLoader:
421
427
 
422
428
  def _export_state_to_cloud(self) -> Tuple[Union[str, None], str]:
423
429
  if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
424
- raise Exception('Cannot get the pull request information from GitHub.')
430
+ raise Exception("Cannot get the pull request information from GitHub.")
425
431
 
426
432
  check_status = CheckDAO().status()
427
433
  metadata = {
428
- 'total_checks': check_status.get('total', 0),
429
- 'approved_checks': check_status.get('approved', 0),
434
+ "total_checks": check_status.get("total", 0),
435
+ "approved_checks": check_status.get("approved", 0),
430
436
  }
431
437
 
432
- if self.cloud_options.get('host', '').startswith('s3://'):
438
+ if self.cloud_options.get("host", "").startswith("s3://"):
433
439
  logger.info("Store recce state to AWS S3 bucket")
434
440
  return self._export_state_to_s3_bucket(metadata=metadata), None
435
441
  else:
@@ -438,36 +444,41 @@ class RecceStateLoader:
438
444
  metadata = self._get_metadata_from_recce_cloud()
439
445
  if metadata is None:
440
446
  return None
441
- state_etag = metadata.get('etag')
447
+ state_etag = metadata.get("etag")
442
448
  return message, state_etag
443
449
 
444
450
  def _export_state_to_recce_cloud(self, metadata: dict = None) -> Union[str, None]:
445
451
  import tempfile
452
+
446
453
  import requests
447
454
 
448
- presigned_url = RecceCloud(token=self.cloud_options.get('token')).get_presigned_url(
449
- method=PresignedUrlMethod.UPLOAD, repository=self.pr_info.repository,
455
+ presigned_url = RecceCloud(token=self.cloud_options.get("token")).get_presigned_url(
456
+ method=PresignedUrlMethod.UPLOAD,
457
+ repository=self.pr_info.repository,
450
458
  artifact_name=RECCE_STATE_COMPRESSED_FILE,
451
459
  pr_id=self.pr_info.id,
452
- metadata=metadata)
453
- compress_passwd = self.cloud_options.get('password')
460
+ metadata=metadata,
461
+ )
462
+ compress_passwd = self.cloud_options.get("password")
454
463
  headers = s3_sse_c_headers(compress_passwd)
455
464
  if metadata:
456
- headers['x-amz-tagging'] = urlencode(metadata)
465
+ headers["x-amz-tagging"] = urlencode(metadata)
457
466
  with tempfile.NamedTemporaryFile() as tmp:
458
467
  self._export_state_to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
459
- response = requests.put(presigned_url, data=open(tmp.name, 'rb').read(), headers=headers)
468
+ response = requests.put(presigned_url, data=open(tmp.name, "rb").read(), headers=headers)
460
469
  if response.status_code != 200:
461
470
  self.error_message = response.text
462
- return 'Failed to upload the state file to Recce Cloud. Reason: ' + response.text
463
- return 'The state file is uploaded to Recce Cloud.'
471
+ return "Failed to upload the state file to Recce Cloud. Reason: " + response.text
472
+ return "The state file is uploaded to Recce Cloud."
464
473
 
465
474
  def _export_state_to_s3_bucket(self, metadata: dict = None) -> Union[str, None]:
466
- import boto3
467
475
  import tempfile
468
- s3_client = boto3.client('s3')
469
- s3_bucket_name = self.cloud_options.get('host').replace('s3://', '')
470
- s3_bucket_key = f'github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}'
476
+
477
+ import boto3
478
+
479
+ s3_client = boto3.client("s3")
480
+ s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
481
+ s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
471
482
 
472
483
  rc, error_message = check_s3_bucket(s3_bucket_name)
473
484
  if rc is False:
@@ -476,27 +487,33 @@ class RecceStateLoader:
476
487
  with tempfile.NamedTemporaryFile() as tmp:
477
488
  self._export_state_to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
478
489
 
479
- s3_client.upload_file(tmp.name, s3_bucket_name, s3_bucket_key,
480
- # Casting all the values under metadata to string
481
- ExtraArgs={'Metadata': {k: str(v) for k, v in metadata.items()}})
482
- RecceCloud(token=self.cloud_options.get('token')).update_github_pull_request_check(self.pr_info, metadata)
483
- return f'The state file is uploaded to \' s3://{s3_bucket_name}/{s3_bucket_key}\''
490
+ s3_client.upload_file(
491
+ tmp.name,
492
+ s3_bucket_name,
493
+ s3_bucket_key,
494
+ # Casting all the values under metadata to string
495
+ ExtraArgs={"Metadata": {k: str(v) for k, v in metadata.items()}},
496
+ )
497
+ RecceCloud(token=self.cloud_options.get("token")).update_github_pull_request_check(self.pr_info, metadata)
498
+ return f"The state file is uploaded to ' s3://{s3_bucket_name}/{s3_bucket_key}'"
484
499
 
485
500
  def _get_artifact_metadata_from_s3_bucket(self, artifact_name: str) -> Union[dict, None]:
486
501
  import boto3
487
- s3_client = boto3.client('s3')
488
- s3_bucket_name = self.cloud_options.get('host').replace('s3://', '')
489
- s3_bucket_key = f'github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{artifact_name}'
502
+
503
+ s3_client = boto3.client("s3")
504
+ s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
505
+ s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{artifact_name}"
490
506
  try:
491
507
  response = s3_client.head_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
492
- metadata = response['Metadata']
508
+ metadata = response["Metadata"]
493
509
  return metadata
494
510
  except botocore.exceptions.ClientError as e:
495
- self.error_message = e.response.get('Error', {}).get('Message')
496
- raise Exception('Failed to get artifact metadata from Recce Cloud.')
511
+ self.error_message = e.response.get("Error", {}).get("Message")
512
+ raise Exception("Failed to get artifact metadata from Recce Cloud.")
497
513
 
498
- def _export_state_to_file(self, file_path: Optional[str] = None,
499
- file_type: SupportedFileTypes = SupportedFileTypes.FILE) -> str:
514
+ def _export_state_to_file(
515
+ self, file_path: Optional[str] = None, file_type: SupportedFileTypes = SupportedFileTypes.FILE
516
+ ) -> str:
500
517
  """
501
518
  Store the state to a file. Store happens when terminating the server or run instance.
502
519
  """
@@ -506,7 +523,7 @@ class RecceStateLoader:
506
523
  io = file_io_factory(file_type)
507
524
 
508
525
  io.write(file_path, json_data)
509
- return f'The state file is stored at \'{file_path}\''
526
+ return f"The state file is stored at '{file_path}'"
510
527
 
511
528
 
512
529
  class RecceCloudStateManager:
@@ -521,18 +538,18 @@ class RecceCloudStateManager:
521
538
  self.error_message = None
522
539
  self.hint_message = None
523
540
 
524
- if not self.cloud_options.get('token'):
541
+ if not self.cloud_options.get("token"):
525
542
  raise Exception(RECCE_CLOUD_TOKEN_MISSING.error_message)
526
- self.pr_info = fetch_pr_metadata(cloud=True, github_token=self.cloud_options.get('token'))
543
+ self.pr_info = fetch_pr_metadata(cloud=True, github_token=self.cloud_options.get("token"))
527
544
  if self.pr_info.id is None:
528
- raise Exception('Cannot get the pull request information from GitHub.')
545
+ raise Exception("Cannot get the pull request information from GitHub.")
529
546
 
530
547
  def verify(self) -> bool:
531
- if self.cloud_options.get('token') is None:
548
+ if self.cloud_options.get("token") is None:
532
549
  self.error_message = RECCE_CLOUD_TOKEN_MISSING.error_message
533
550
  self.hint_message = RECCE_CLOUD_TOKEN_MISSING.hint_message
534
551
  return False
535
- if self.cloud_options.get('password') is None:
552
+ if self.cloud_options.get("password") is None:
536
553
  self.error_message = RECCE_CLOUD_PASSWORD_MISSING.error_message
537
554
  self.hint_message = RECCE_CLOUD_PASSWORD_MISSING.hint_message
538
555
  return False
@@ -543,54 +560,58 @@ class RecceCloudStateManager:
543
560
  return self.error_message, self.hint_message
544
561
 
545
562
  def _check_state_in_recce_cloud(self) -> bool:
546
- return RecceCloud(token=self.cloud_options.get('token')).check_artifacts_exists(self.pr_info)
563
+ return RecceCloud(token=self.cloud_options.get("token")).check_artifacts_exists(self.pr_info)
547
564
 
548
565
  def _check_state_in_s3_bucket(self) -> bool:
549
566
  import boto3
550
567
 
551
- s3_client = boto3.client('s3')
552
- s3_bucket_name = self.cloud_options.get('host').replace('s3://', '')
553
- s3_bucket_key = f'github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}'
568
+ s3_client = boto3.client("s3")
569
+ s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
570
+ s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
554
571
  try:
555
572
  s3_client.head_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
556
573
  except botocore.exceptions.ClientError as e:
557
- error_code = e.response.get('Error', {}).get('Code')
558
- if error_code == '404':
574
+ error_code = e.response.get("Error", {}).get("Code")
575
+ if error_code == "404":
559
576
  return False
560
577
  return True
561
578
 
562
579
  def check_cloud_state_exists(self) -> bool:
563
- if self.cloud_options.get('host', '').startswith('s3://'):
580
+ if self.cloud_options.get("host", "").startswith("s3://"):
564
581
  return self._check_state_in_s3_bucket()
565
582
  else:
566
583
  return self._check_state_in_recce_cloud()
567
584
 
568
585
  def _upload_state_to_recce_cloud(self, state: RecceState, metadata: dict = None) -> Union[str, None]:
569
586
  import tempfile
587
+
570
588
  import requests
571
589
 
572
- presigned_url = RecceCloud(token=self.cloud_options.get('token')).get_presigned_url(
573
- method=PresignedUrlMethod.UPLOAD, repository=self.pr_info.repository,
590
+ presigned_url = RecceCloud(token=self.cloud_options.get("token")).get_presigned_url(
591
+ method=PresignedUrlMethod.UPLOAD,
592
+ repository=self.pr_info.repository,
574
593
  artifact_name=RECCE_STATE_COMPRESSED_FILE,
575
594
  pr_id=self.pr_info.id,
576
- metadata=metadata)
595
+ metadata=metadata,
596
+ )
577
597
 
578
- compress_passwd = self.cloud_options.get('password')
598
+ compress_passwd = self.cloud_options.get("password")
579
599
  headers = s3_sse_c_headers(compress_passwd)
580
600
  with tempfile.NamedTemporaryFile() as tmp:
581
601
  state.to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
582
- response = requests.put(presigned_url, data=open(tmp.name, 'rb').read(), headers=headers)
602
+ response = requests.put(presigned_url, data=open(tmp.name, "rb").read(), headers=headers)
583
603
  if response.status_code != 200:
584
- return f'Failed to upload the state file to Recce Cloud. Reason: {response.text}'
585
- return 'The state file is uploaded to Recce Cloud.'
604
+ return f"Failed to upload the state file to Recce Cloud. Reason: {response.text}"
605
+ return "The state file is uploaded to Recce Cloud."
586
606
 
587
607
  def _upload_state_to_s3_bucket(self, state: RecceState, metadata: dict = None) -> Union[str, None]:
588
- import boto3
589
608
  import tempfile
590
609
 
591
- s3_client = boto3.client('s3')
592
- s3_bucket_name = self.cloud_options.get('host').replace('s3://', '')
593
- s3_bucket_key = f'github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}'
610
+ import boto3
611
+
612
+ s3_client = boto3.client("s3")
613
+ s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
614
+ s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
594
615
 
595
616
  rc, error_message = check_s3_bucket(s3_bucket_name)
596
617
  if rc is False:
@@ -599,60 +620,69 @@ class RecceCloudStateManager:
599
620
  with tempfile.NamedTemporaryFile() as tmp:
600
621
  state.to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
601
622
 
602
- s3_client.upload_file(tmp.name, s3_bucket_name, s3_bucket_key,
603
- # Casting all the values under metadata to string
604
- ExtraArgs={'Metadata': {k: str(v) for k, v in metadata.items()}})
605
- RecceCloud(token=self.cloud_options.get('token')).update_github_pull_request_check(self.pr_info, metadata)
606
- return f'The state file is uploaded to \' s3://{s3_bucket_name}/{s3_bucket_key}\''
623
+ s3_client.upload_file(
624
+ tmp.name,
625
+ s3_bucket_name,
626
+ s3_bucket_key,
627
+ # Casting all the values under metadata to string
628
+ ExtraArgs={"Metadata": {k: str(v) for k, v in metadata.items()}},
629
+ )
630
+ RecceCloud(token=self.cloud_options.get("token")).update_github_pull_request_check(self.pr_info, metadata)
631
+ return f"The state file is uploaded to ' s3://{s3_bucket_name}/{s3_bucket_key}'"
607
632
 
608
633
  def upload_state_to_cloud(self, state: RecceState) -> Union[str, None]:
609
634
  if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
610
- raise Exception('Cannot get the pull request information from GitHub.')
635
+ raise Exception("Cannot get the pull request information from GitHub.")
611
636
 
612
637
  checks = state.checks
613
638
 
614
639
  metadata = {
615
- 'total_checks': len(checks),
616
- 'approved_checks': len([c for c in checks if c.is_checked]),
640
+ "total_checks": len(checks),
641
+ "approved_checks": len([c for c in checks if c.is_checked]),
617
642
  }
618
643
 
619
- if self.cloud_options.get('host', '').startswith('s3://'):
644
+ if self.cloud_options.get("host", "").startswith("s3://"):
620
645
  return self._upload_state_to_s3_bucket(state, metadata)
621
646
  else:
622
647
  return self._upload_state_to_recce_cloud(state, metadata)
623
648
 
624
649
  def _download_state_from_s3_bucket(self, filepath):
625
650
  import io
651
+
626
652
  import boto3
627
653
 
628
- s3_client = boto3.client('s3')
629
- s3_bucket_name = self.cloud_options.get('host').replace('s3://', '')
630
- s3_bucket_key = f'github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}'
654
+ s3_client = boto3.client("s3")
655
+ s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
656
+ s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
631
657
 
632
658
  rc, error_message = check_s3_bucket(s3_bucket_name)
633
659
  if rc is False:
634
660
  raise Exception(error_message)
635
661
 
636
662
  response = s3_client.get_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
637
- byte_stream = io.BytesIO(response['Body'].read())
663
+ byte_stream = io.BytesIO(response["Body"].read())
638
664
  gzip_io = file_io_factory(SupportedFileTypes.GZIP)
639
665
  decompressed_content = gzip_io.read_fileobj(byte_stream)
640
666
 
641
667
  dirs = os.path.dirname(filepath)
642
668
  if dirs:
643
669
  os.makedirs(dirs, exist_ok=True)
644
- with open(filepath, 'wb') as f:
670
+ with open(filepath, "wb") as f:
645
671
  f.write(decompressed_content)
646
672
 
647
673
  def _download_state_from_recce_cloud(self, filepath):
648
674
  import io
675
+
649
676
  import requests
650
677
 
651
- presigned_url = RecceCloud(token=self.cloud_options.get('token')).get_presigned_url(
652
- method=PresignedUrlMethod.DOWNLOAD, repository=self.pr_info.repository,
653
- artifact_name=RECCE_STATE_COMPRESSED_FILE, pr_id=self.pr_info.id)
678
+ presigned_url = RecceCloud(token=self.cloud_options.get("token")).get_presigned_url(
679
+ method=PresignedUrlMethod.DOWNLOAD,
680
+ repository=self.pr_info.repository,
681
+ artifact_name=RECCE_STATE_COMPRESSED_FILE,
682
+ pr_id=self.pr_info.id,
683
+ )
654
684
 
655
- password = self.cloud_options.get('password')
685
+ password = self.cloud_options.get("password")
656
686
  if password is None:
657
687
  raise Exception(RECCE_CLOUD_PASSWORD_MISSING.error_message)
658
688
 
@@ -661,7 +691,8 @@ class RecceCloudStateManager:
661
691
 
662
692
  if response.status_code != 200:
663
693
  raise Exception(
664
- f'{response.status_code} Failed to download the state file from Recce Cloud. The password could be wrong.')
694
+ f"{response.status_code} Failed to download the state file from Recce Cloud. The password could be wrong."
695
+ )
665
696
 
666
697
  byte_stream = io.BytesIO(response.content)
667
698
  gzip_io = file_io_factory(SupportedFileTypes.GZIP)
@@ -670,53 +701,54 @@ class RecceCloudStateManager:
670
701
  dirs = os.path.dirname(filepath)
671
702
  if dirs:
672
703
  os.makedirs(dirs, exist_ok=True)
673
- with open(filepath, 'wb') as f:
704
+ with open(filepath, "wb") as f:
674
705
  f.write(decompressed_content)
675
706
 
676
707
  def download_state_from_cloud(self, filepath: str) -> Union[str, None]:
677
708
  if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
678
- raise Exception('Cannot get the pull request information from GitHub.')
709
+ raise Exception("Cannot get the pull request information from GitHub.")
679
710
 
680
- if self.cloud_options.get('host', '').startswith('s3://'):
681
- logger.debug('Download state file from AWS S3 bucket...')
711
+ if self.cloud_options.get("host", "").startswith("s3://"):
712
+ logger.debug("Download state file from AWS S3 bucket...")
682
713
  return self._download_state_from_s3_bucket(filepath)
683
714
  else:
684
- logger.debug('Download state file from Recce Cloud...')
715
+ logger.debug("Download state file from Recce Cloud...")
685
716
  return self._download_state_from_recce_cloud(filepath)
686
717
 
687
718
  def _purge_state_from_s3_bucket(self) -> (bool, str):
688
719
  import boto3
689
720
  from rich.console import Console
721
+
690
722
  console = Console()
691
723
  delete_objects = []
692
- logger.debug('Purging the state from AWS S3 bucket...')
693
- s3_client = boto3.client('s3')
694
- s3_bucket_name = self.cloud_options.get('host').replace('s3://', '')
695
- s3_key_prefix = f'github/{self.pr_info.repository}/pulls/{self.pr_info.id}/'
724
+ logger.debug("Purging the state from AWS S3 bucket...")
725
+ s3_client = boto3.client("s3")
726
+ s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
727
+ s3_key_prefix = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/"
696
728
  list_response = s3_client.list_objects_v2(Bucket=s3_bucket_name, Prefix=s3_key_prefix)
697
- if 'Contents' in list_response:
698
- for obj in list_response['Contents']:
699
- key = obj['Key']
700
- delete_objects.append({'Key': key})
701
- console.print(f'[green]Deleted[/green]: {key}')
729
+ if "Contents" in list_response:
730
+ for obj in list_response["Contents"]:
731
+ key = obj["Key"]
732
+ delete_objects.append({"Key": key})
733
+ console.print(f"[green]Deleted[/green]: {key}")
702
734
  else:
703
- return False, 'No state file found in the S3 bucket.'
735
+ return False, "No state file found in the S3 bucket."
704
736
 
705
- delete_response = s3_client.delete_objects(Bucket=s3_bucket_name, Delete={'Objects': delete_objects})
706
- if 'Deleted' not in delete_response:
707
- return False, 'Failed to delete the state file from the S3 bucket.'
708
- RecceCloud(token=self.cloud_options.get('token')).update_github_pull_request_check(self.pr_info)
737
+ delete_response = s3_client.delete_objects(Bucket=s3_bucket_name, Delete={"Objects": delete_objects})
738
+ if "Deleted" not in delete_response:
739
+ return False, "Failed to delete the state file from the S3 bucket."
740
+ RecceCloud(token=self.cloud_options.get("token")).update_github_pull_request_check(self.pr_info)
709
741
  return True, None
710
742
 
711
743
  def _purge_state_from_recce_cloud(self) -> (bool, str):
712
744
  try:
713
- RecceCloud(token=self.cloud_options.get('token')).purge_artifacts(self.pr_info)
745
+ RecceCloud(token=self.cloud_options.get("token")).purge_artifacts(self.pr_info)
714
746
  except RecceCloudException as e:
715
747
  return False, e.reason
716
748
  return True, None
717
749
 
718
750
  def purge_cloud_state(self) -> (bool, str):
719
- if self.cloud_options.get('host', '').startswith('s3://'):
751
+ if self.cloud_options.get("host", "").startswith("s3://"):
720
752
  return self._purge_state_from_s3_bucket()
721
753
  else:
722
754
  return self._purge_state_from_recce_cloud()
@@ -734,7 +766,7 @@ class RecceShareStateManager:
734
766
  self.hint_message = None
735
767
 
736
768
  def verify(self) -> bool:
737
- if self.auth_options.get('api_token') is None:
769
+ if self.auth_options.get("api_token") is None:
738
770
  self.error_message = RECCE_API_TOKEN_MISSING.error_message
739
771
  self.hint_message = RECCE_API_TOKEN_MISSING.hint_message
740
772
  return False
@@ -749,5 +781,5 @@ class RecceShareStateManager:
749
781
 
750
782
  with tempfile.NamedTemporaryFile() as tmp:
751
783
  state.to_file(tmp.name, file_type=SupportedFileTypes.FILE)
752
- response = RecceCloud(token=self.auth_options.get('api_token')).share_state(file_name, open(tmp.name, 'rb'))
784
+ response = RecceCloud(token=self.auth_options.get("api_token")).share_state(file_name, open(tmp.name, "rb"))
753
785
  return response