recce-nightly 1.2.0.20250506__py3-none-any.whl → 1.26.0.20251124__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of recce-nightly might be problematic. Click here for more details.

Files changed (213) hide show
  1. recce/VERSION +1 -1
  2. recce/__init__.py +27 -22
  3. recce/adapter/base.py +11 -14
  4. recce/adapter/dbt_adapter/__init__.py +810 -480
  5. recce/adapter/dbt_adapter/dbt_version.py +3 -0
  6. recce/adapter/sqlmesh_adapter.py +24 -35
  7. recce/apis/check_api.py +39 -28
  8. recce/apis/check_func.py +33 -27
  9. recce/apis/run_api.py +25 -19
  10. recce/apis/run_func.py +29 -23
  11. recce/artifact.py +119 -51
  12. recce/cli.py +1299 -323
  13. recce/config.py +42 -33
  14. recce/connect_to_cloud.py +138 -0
  15. recce/core.py +55 -47
  16. recce/data/404.html +1 -1
  17. recce/data/__next.__PAGE__.txt +10 -0
  18. recce/data/__next._full.txt +23 -0
  19. recce/data/__next._head.txt +8 -0
  20. recce/data/__next._index.txt +8 -0
  21. recce/data/__next._tree.txt +5 -0
  22. recce/data/_next/static/52aV_JrNUZU6dMFgvTQEO/_buildManifest.js +11 -0
  23. recce/data/_next/static/52aV_JrNUZU6dMFgvTQEO/_clientMiddlewareManifest.json +1 -0
  24. recce/data/_next/static/chunks/02b996c7f6a29a06.js +4 -0
  25. recce/data/_next/static/chunks/19c10d219a6a21ff.js +1 -0
  26. recce/data/_next/static/chunks/2df9ec28a061971d.js +11 -0
  27. recce/data/_next/static/chunks/3098c987393bda15.js +1 -0
  28. recce/data/_next/static/chunks/393dc43e483f717a.css +2 -0
  29. recce/data/_next/static/chunks/399e8d91a7e45073.js +2 -0
  30. recce/data/_next/static/chunks/4d0186f631230245.js +1 -0
  31. recce/data/_next/static/chunks/5794ba9e10a9c060.js +11 -0
  32. recce/data/_next/static/chunks/715761c929a3f28b.js +110 -0
  33. recce/data/_next/static/chunks/71f88fcc615bf282.js +1 -0
  34. recce/data/_next/static/chunks/80d2a95eaf1201ea.js +1 -0
  35. recce/data/_next/static/chunks/9979c6109bbbee35.js +1 -0
  36. recce/data/_next/static/chunks/99d638224186c118.js +1 -0
  37. recce/data/_next/static/chunks/d003eb36240e92f3.js +1 -0
  38. recce/data/_next/static/chunks/d3167cdfec4fc351.js +1 -0
  39. recce/data/_next/static/chunks/e124bccf574a3361.css +1 -0
  40. recce/data/_next/static/chunks/f40141db1bdb46f0.css +6 -0
  41. recce/data/_next/static/chunks/fcc53a88741a52f9.js +1 -0
  42. recce/data/_next/static/chunks/turbopack-b1920d28cfb1f28d.js +3 -0
  43. recce/data/_next/static/media/favicon.a8d38d84.ico +0 -0
  44. recce/data/_next/static/media/montserrat-cyrillic-800-normal.d80d830d.woff2 +0 -0
  45. recce/data/_next/static/media/montserrat-cyrillic-800-normal.f9d58125.woff +0 -0
  46. recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.076c2a93.woff2 +0 -0
  47. recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.a4fa76b5.woff +0 -0
  48. recce/data/_next/static/media/montserrat-latin-800-normal.cde454cc.woff2 +0 -0
  49. recce/data/_next/static/media/montserrat-latin-800-normal.d5761935.woff +0 -0
  50. recce/data/_next/static/media/montserrat-latin-ext-800-normal.40ec0659.woff2 +0 -0
  51. recce/data/_next/static/media/montserrat-latin-ext-800-normal.b671449b.woff +0 -0
  52. recce/data/_next/static/media/montserrat-vietnamese-800-normal.9f7b8541.woff +0 -0
  53. recce/data/_next/static/media/montserrat-vietnamese-800-normal.f9eb854e.woff2 +0 -0
  54. recce/data/_next/static/media/reload-image.7aa931c7.svg +4 -0
  55. recce/data/_not-found/__next._full.txt +17 -0
  56. recce/data/_not-found/__next._head.txt +8 -0
  57. recce/data/_not-found/__next._index.txt +8 -0
  58. recce/data/_not-found/__next._not-found.__PAGE__.txt +5 -0
  59. recce/data/_not-found/__next._not-found.txt +4 -0
  60. recce/data/_not-found/__next._tree.txt +3 -0
  61. recce/data/_not-found.html +1 -0
  62. recce/data/_not-found.txt +17 -0
  63. recce/data/auth_callback.html +68 -0
  64. recce/data/imgs/reload-image.svg +4 -0
  65. recce/data/index.html +1 -27
  66. recce/data/index.txt +23 -7
  67. recce/diff.py +6 -12
  68. recce/event/__init__.py +86 -74
  69. recce/event/collector.py +33 -22
  70. recce/event/track.py +49 -27
  71. recce/exceptions.py +1 -1
  72. recce/git.py +7 -7
  73. recce/github.py +57 -53
  74. recce/mcp_server.py +716 -0
  75. recce/models/__init__.py +4 -1
  76. recce/models/check.py +6 -7
  77. recce/models/run.py +1 -0
  78. recce/models/types.py +131 -28
  79. recce/pull_request.py +27 -25
  80. recce/run.py +165 -121
  81. recce/server.py +303 -111
  82. recce/state/__init__.py +31 -0
  83. recce/state/cloud.py +632 -0
  84. recce/state/const.py +26 -0
  85. recce/state/local.py +56 -0
  86. recce/state/state.py +119 -0
  87. recce/state/state_loader.py +174 -0
  88. recce/summary.py +188 -143
  89. recce/tasks/__init__.py +19 -3
  90. recce/tasks/core.py +11 -13
  91. recce/tasks/dataframe.py +82 -18
  92. recce/tasks/histogram.py +69 -34
  93. recce/tasks/lineage.py +2 -2
  94. recce/tasks/profile.py +152 -86
  95. recce/tasks/query.py +139 -87
  96. recce/tasks/rowcount.py +37 -31
  97. recce/tasks/schema.py +18 -15
  98. recce/tasks/top_k.py +35 -35
  99. recce/tasks/valuediff.py +216 -152
  100. recce/util/__init__.py +3 -0
  101. recce/util/api_token.py +80 -0
  102. recce/util/breaking.py +87 -85
  103. recce/util/cll.py +274 -219
  104. recce/util/io.py +22 -17
  105. recce/util/lineage.py +65 -16
  106. recce/util/logger.py +1 -1
  107. recce/util/onboarding_state.py +45 -0
  108. recce/util/perf_tracking.py +85 -0
  109. recce/util/recce_cloud.py +322 -72
  110. recce/util/singleton.py +4 -4
  111. recce/yaml/__init__.py +7 -10
  112. recce_cloud/__init__.py +24 -0
  113. recce_cloud/api/__init__.py +17 -0
  114. recce_cloud/api/base.py +111 -0
  115. recce_cloud/api/client.py +150 -0
  116. recce_cloud/api/exceptions.py +26 -0
  117. recce_cloud/api/factory.py +63 -0
  118. recce_cloud/api/github.py +76 -0
  119. recce_cloud/api/gitlab.py +82 -0
  120. recce_cloud/artifact.py +57 -0
  121. recce_cloud/ci_providers/__init__.py +9 -0
  122. recce_cloud/ci_providers/base.py +82 -0
  123. recce_cloud/ci_providers/detector.py +147 -0
  124. recce_cloud/ci_providers/github_actions.py +136 -0
  125. recce_cloud/ci_providers/gitlab_ci.py +130 -0
  126. recce_cloud/cli.py +245 -0
  127. recce_cloud/upload.py +214 -0
  128. {recce_nightly-1.2.0.20250506.dist-info → recce_nightly-1.26.0.20251124.dist-info}/METADATA +68 -37
  129. recce_nightly-1.26.0.20251124.dist-info/RECORD +180 -0
  130. {recce_nightly-1.2.0.20250506.dist-info → recce_nightly-1.26.0.20251124.dist-info}/WHEEL +1 -1
  131. {recce_nightly-1.2.0.20250506.dist-info → recce_nightly-1.26.0.20251124.dist-info}/top_level.txt +1 -0
  132. tests/adapter/dbt_adapter/conftest.py +9 -5
  133. tests/adapter/dbt_adapter/dbt_test_helper.py +37 -22
  134. tests/adapter/dbt_adapter/test_dbt_adapter.py +0 -15
  135. tests/adapter/dbt_adapter/test_dbt_cll.py +656 -41
  136. tests/adapter/dbt_adapter/test_selector.py +22 -21
  137. tests/recce_cloud/__init__.py +0 -0
  138. tests/recce_cloud/test_ci_providers.py +351 -0
  139. tests/recce_cloud/test_cli.py +372 -0
  140. tests/recce_cloud/test_client.py +273 -0
  141. tests/recce_cloud/test_platform_clients.py +333 -0
  142. tests/tasks/conftest.py +1 -1
  143. tests/tasks/test_histogram.py +58 -66
  144. tests/tasks/test_lineage.py +36 -23
  145. tests/tasks/test_preset_checks.py +45 -31
  146. tests/tasks/test_profile.py +339 -15
  147. tests/tasks/test_query.py +46 -46
  148. tests/tasks/test_row_count.py +65 -46
  149. tests/tasks/test_schema.py +65 -42
  150. tests/tasks/test_top_k.py +22 -18
  151. tests/tasks/test_valuediff.py +43 -32
  152. tests/test_cli.py +174 -60
  153. tests/test_cli_mcp_optional.py +45 -0
  154. tests/test_cloud_listing_cli.py +324 -0
  155. tests/test_config.py +7 -9
  156. tests/test_connect_to_cloud.py +82 -0
  157. tests/test_core.py +151 -4
  158. tests/test_dbt.py +7 -7
  159. tests/test_mcp_server.py +332 -0
  160. tests/test_pull_request.py +1 -1
  161. tests/test_server.py +25 -19
  162. tests/test_summary.py +29 -17
  163. recce/data/_next/static/Kcbs3GEIyH2LxgLYat0es/_buildManifest.js +0 -1
  164. recce/data/_next/static/chunks/1f229bf6-d9fe92e56db8d93b.js +0 -1
  165. recce/data/_next/static/chunks/29e3cc0d-8c150e37dff9631b.js +0 -1
  166. recce/data/_next/static/chunks/368-7587b306577df275.js +0 -65
  167. recce/data/_next/static/chunks/36e1c10d-bb0210cbd6573a8d.js +0 -1
  168. recce/data/_next/static/chunks/3998a672-eaad84bdd88cc73e.js +0 -1
  169. recce/data/_next/static/chunks/3a92ee20-3b5d922d4157af5e.js +0 -1
  170. recce/data/_next/static/chunks/450c323b-1bb5db526e54435a.js +0 -1
  171. recce/data/_next/static/chunks/47d8844f-79a1b53c66a7d7ec.js +0 -1
  172. recce/data/_next/static/chunks/6dc81886-c94b9b91bc2c3caf.js +0 -1
  173. recce/data/_next/static/chunks/6ef81909-694dc38134099299.js +0 -1
  174. recce/data/_next/static/chunks/700-3b65fc3666820d00.js +0 -2
  175. recce/data/_next/static/chunks/7a8a3e83-d7fa409d97b38b2b.js +0 -1
  176. recce/data/_next/static/chunks/7f27ae6c-413f6b869a04183a.js +0 -1
  177. recce/data/_next/static/chunks/8d700b6a-f0b1f6b9e0d97ce2.js +0 -1
  178. recce/data/_next/static/chunks/9746af58-d74bef4d03eea6ab.js +0 -1
  179. recce/data/_next/static/chunks/a30376cd-7d806e1602f2dc3a.js +0 -1
  180. recce/data/_next/static/chunks/app/_not-found/page-8a886fa0855c3105.js +0 -1
  181. recce/data/_next/static/chunks/app/layout-9102e22cb73f74d6.js +0 -1
  182. recce/data/_next/static/chunks/app/page-cee661090afbd6aa.js +0 -1
  183. recce/data/_next/static/chunks/b63b1b3f-7395c74e11a14e95.js +0 -1
  184. recce/data/_next/static/chunks/c132bf7d-8102037f9ccf372a.js +0 -1
  185. recce/data/_next/static/chunks/c1ceaa8b-a1e442154d23515e.js +0 -1
  186. recce/data/_next/static/chunks/cd9f8d63-cf0d5a7b0f7a92e8.js +0 -54
  187. recce/data/_next/static/chunks/ce84277d-f42c2c58049cea2d.js +0 -1
  188. recce/data/_next/static/chunks/e24bf851-0f8cbc99656833e7.js +0 -1
  189. recce/data/_next/static/chunks/fee69bc6-f17d36c080742e74.js +0 -1
  190. recce/data/_next/static/chunks/framework-ded83d71b51ce901.js +0 -1
  191. recce/data/_next/static/chunks/main-a0859f1f36d0aa6c.js +0 -1
  192. recce/data/_next/static/chunks/main-app-0225a2255968e566.js +0 -1
  193. recce/data/_next/static/chunks/pages/_app-d5672bf3d8b6371b.js +0 -1
  194. recce/data/_next/static/chunks/pages/_error-ed75be3f25588548.js +0 -1
  195. recce/data/_next/static/chunks/webpack-567d72f0bc0820d5.js +0 -1
  196. recce/data/_next/static/css/c9ecb46a4b21c126.css +0 -14
  197. recce/data/_next/static/media/montserrat-cyrillic-800-normal.22628180.woff2 +0 -0
  198. recce/data/_next/static/media/montserrat-cyrillic-800-normal.31d693bb.woff +0 -0
  199. recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.7e2c1e62.woff +0 -0
  200. recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.94a63aea.woff2 +0 -0
  201. recce/data/_next/static/media/montserrat-latin-800-normal.6f8fa298.woff2 +0 -0
  202. recce/data/_next/static/media/montserrat-latin-800-normal.97e20d5e.woff +0 -0
  203. recce/data/_next/static/media/montserrat-latin-ext-800-normal.013b84f9.woff2 +0 -0
  204. recce/data/_next/static/media/montserrat-latin-ext-800-normal.aff52ab0.woff +0 -0
  205. recce/data/_next/static/media/montserrat-vietnamese-800-normal.5f21869b.woff +0 -0
  206. recce/data/_next/static/media/montserrat-vietnamese-800-normal.c0035377.woff2 +0 -0
  207. recce/state.py +0 -753
  208. recce_nightly-1.2.0.20250506.dist-info/RECORD +0 -142
  209. tests/test_state.py +0 -123
  210. /recce/data/_next/static/{Kcbs3GEIyH2LxgLYat0es → 52aV_JrNUZU6dMFgvTQEO}/_ssgManifest.js +0 -0
  211. /recce/data/_next/static/chunks/{polyfills-42372ed130431b0a.js → a6dad97d9634a72d.js} +0 -0
  212. {recce_nightly-1.2.0.20250506.dist-info → recce_nightly-1.26.0.20251124.dist-info}/entry_points.txt +0 -0
  213. {recce_nightly-1.2.0.20250506.dist-info → recce_nightly-1.26.0.20251124.dist-info}/licenses/LICENSE +0 -0
recce/state.py DELETED
@@ -1,753 +0,0 @@
1
- """Define the type to serialize/de-serialize the state of the recce instance."""
2
- import json
3
- import logging
4
- import os
5
- import threading
6
- import time
7
- from base64 import b64encode
8
- from dataclasses import dataclass
9
- from datetime import datetime
10
- from hashlib import md5, sha256
11
- from typing import List, Optional, Dict, Union, Tuple
12
- from urllib.parse import urlencode
13
-
14
- import botocore.exceptions
15
- from pydantic import BaseModel
16
- from pydantic import Field
17
-
18
- from recce import get_version
19
- from recce.git import current_branch
20
- from recce.models import CheckDAO
21
- from recce.models.types import Run, Check
22
- from recce.pull_request import fetch_pr_metadata, PullRequestInfo
23
- from recce.util.io import SupportedFileTypes, file_io_factory
24
- from recce.util.pydantic_model import pydantic_model_json_dump, pydantic_model_dump
25
- from recce.util.recce_cloud import RecceCloud, PresignedUrlMethod, RecceCloudException
26
-
27
- logger = logging.getLogger('uvicorn')
28
-
29
- RECCE_STATE_FILE = 'recce-state.json'
30
- RECCE_STATE_COMPRESSED_FILE = f'{RECCE_STATE_FILE}.gz'
31
-
32
-
33
- @dataclass
34
- class ErrorMessage:
35
- error_message: str
36
- hint_message: str
37
-
38
-
39
- RECCE_CLOUD_TOKEN_MISSING = ErrorMessage(
40
- error_message='No GitHub token is provided to access the pull request information',
41
- hint_message='Please provide a GitHub token in the command argument'
42
- )
43
-
44
- RECCE_CLOUD_PASSWORD_MISSING = ErrorMessage(
45
- error_message='No password provided to access the state file in Recce Cloud',
46
- hint_message='Please provide a password with the option "--password <compress-password>"'
47
- )
48
-
49
- RECCE_API_TOKEN_MISSING = ErrorMessage(
50
- error_message='No Recc API token is provided',
51
- hint_message='Please login to Recce Cloud and copy the API token from the settings page'
52
- )
53
-
54
-
55
- def s3_sse_c_headers(password: str) -> Dict[str, str]:
56
- hashed_password = sha256()
57
- md5_hash = md5()
58
- hashed_password.update(password.encode())
59
- md5_hash.update(hashed_password.digest())
60
- encoded_passwd = b64encode(hashed_password.digest()).decode('utf-8')
61
- encoded_md5 = b64encode(md5_hash.digest()).decode('utf-8')
62
- return {
63
- 'x-amz-server-side-encryption-customer-algorithm': 'AES256',
64
- 'x-amz-server-side-encryption-customer-key': encoded_passwd,
65
- 'x-amz-server-side-encryption-customer-key-MD5': encoded_md5,
66
- }
67
-
68
-
69
- def check_s3_bucket(bucket_name: str):
70
- import boto3
71
- s3_client = boto3.client('s3')
72
- try:
73
- s3_client.head_bucket(Bucket=bucket_name)
74
- except botocore.exceptions.ClientError as e:
75
- error_code = e.response['Error']['Code']
76
- if error_code == '404':
77
- return False, f"Bucket '{bucket_name}' does not exist."
78
- elif error_code == '403':
79
- return False, f"Bucket '{bucket_name}' exists but you do not have permission to access it."
80
- else:
81
- return False, f"Failed to access the S3 bucket: '{bucket_name}'"
82
- return True, None
83
-
84
-
85
- class GitRepoInfo(BaseModel):
86
- branch: Optional[str] = None
87
-
88
- @staticmethod
89
- def from_current_repositroy():
90
- branch = current_branch()
91
- if branch is None:
92
- return None
93
-
94
- return GitRepoInfo(branch=branch)
95
-
96
- def to_dict(self):
97
- return pydantic_model_dump(self)
98
-
99
-
100
- class RecceStateMetadata(BaseModel):
101
- schema_version: str = 'v0'
102
- recce_version: str = Field(default_factory=lambda: get_version())
103
- generated_at: str = Field(default_factory=lambda: datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"))
104
-
105
-
106
- class ArtifactsRoot(BaseModel):
107
- """
108
- Root of the artifacts.
109
-
110
- base: artifacts of the base env. key is file name, value is dict
111
- current: artifacts of the current env. key is file name, value is dict
112
- """
113
- base: Dict[str, Optional[dict]] = {}
114
- current: Dict[str, Optional[dict]] = {}
115
-
116
-
117
- class RecceState(BaseModel):
118
- metadata: Optional[RecceStateMetadata] = None
119
- runs: Optional[List[Run]] = Field(default_factory=list)
120
- checks: Optional[List[Check]] = Field(default_factory=list)
121
- artifacts: ArtifactsRoot = ArtifactsRoot(base={}, current={})
122
- git: Optional[GitRepoInfo] = None
123
- pull_request: Optional[PullRequestInfo] = None
124
-
125
- @staticmethod
126
- def from_json(json_content: str):
127
- dict_data = json.loads(json_content)
128
- state = RecceState(**dict_data)
129
- metadata = state.metadata
130
- if metadata:
131
- if metadata.schema_version is None:
132
- pass
133
- if metadata.schema_version == 'v0':
134
- pass
135
- else:
136
- raise Exception(f"Unsupported state file version: {metadata.schema_version}")
137
- return state
138
-
139
- @staticmethod
140
- def from_file(file_path: str, file_type: SupportedFileTypes = SupportedFileTypes.FILE):
141
- """
142
- Load the state from a recce state file.
143
- """
144
- from pathlib import Path
145
-
146
- logger.debug(f"Load state file from: '{file_path}'")
147
- if not Path(file_path).is_file():
148
- return None
149
-
150
- io = file_io_factory(file_type)
151
- json_content = io.read(file_path)
152
- return RecceState.from_json(json_content)
153
-
154
- def to_json(self):
155
- return pydantic_model_json_dump(self)
156
-
157
- def to_file(self, file_path: str, file_type: SupportedFileTypes = SupportedFileTypes.FILE):
158
-
159
- json_data = self.to_json()
160
- io = file_io_factory(file_type)
161
-
162
- io.write(file_path, json_data)
163
- return f'The state file is stored at \'{file_path}\''
164
-
165
- def _merge_run(self, run: Run):
166
- for r in self.runs:
167
- if r.run_id == run.run_id:
168
- break
169
- else:
170
- self.runs.append(run)
171
-
172
- def _merge_check(self, check: Check):
173
- for c in self.checks:
174
- if c.check_id == check.check_id:
175
- c.merge(check)
176
- break
177
- else:
178
- self.checks.append(check)
179
-
180
- def _merge_artifacts(self, artifacts: ArtifactsRoot):
181
- self.artifacts.merge(artifacts)
182
-
183
-
184
- class RecceStateLoader:
185
- def __init__(self,
186
- review_mode: bool = False,
187
- cloud_mode: bool = False,
188
- state_file: Optional[str] = None,
189
- cloud_options: Optional[Dict[str, str]] = None,
190
- initial_state: Optional[RecceState] = None,
191
- ):
192
- self.review_mode = review_mode
193
- self.cloud_mode = cloud_mode
194
- self.state_file = state_file
195
- self.cloud_options = cloud_options or {}
196
- self.error_message = None
197
- self.hint_message = None
198
- self.state: RecceState | None = initial_state
199
- self.state_lock = threading.Lock()
200
- self.state_etag = None
201
- self.pr_info = None
202
-
203
- if self.cloud_mode:
204
- if not self.cloud_options.get('token'):
205
- raise Exception(RECCE_CLOUD_TOKEN_MISSING.error_message)
206
- self.pr_info = fetch_pr_metadata(cloud=self.cloud_mode, github_token=self.cloud_options.get('token'))
207
- if self.pr_info.id is None:
208
- raise Exception('Cannot get the pull request information from GitHub.')
209
-
210
- # Load the state
211
- self.load()
212
-
213
- def verify(self) -> bool:
214
- if self.cloud_mode:
215
- if self.cloud_options.get('token') is None:
216
- self.error_message = RECCE_CLOUD_TOKEN_MISSING.error_message
217
- self.hint_message = RECCE_CLOUD_TOKEN_MISSING.hint_message
218
- return False
219
- if not self.cloud_options.get('host'):
220
- if self.cloud_options.get('password') is None:
221
- self.error_message = RECCE_CLOUD_PASSWORD_MISSING.error_message
222
- self.hint_message = RECCE_CLOUD_PASSWORD_MISSING.hint_message
223
- return False
224
- else:
225
- if self.review_mode is True and self.state_file is None:
226
- self.error_message = 'Recce can not launch without a state file.'
227
- self.hint_message = 'Please provide a state file in the command argument.'
228
- return False
229
- pass
230
- return True
231
-
232
- @property
233
- def error_and_hint(self) -> (Union[str, None], Union[str, None]):
234
- return self.error_message, self.hint_message
235
-
236
- def update(self, state: RecceState):
237
- self.state = state
238
-
239
- def load(self, refresh=False) -> RecceState:
240
- if self.state is not None and refresh is False:
241
- return self.state
242
- self.state_lock.acquire()
243
- try:
244
- if self.cloud_mode:
245
- self.state, self.state_etag = self._load_state_from_cloud()
246
- elif self.state_file:
247
- self.state = self._load_state_from_file()
248
- finally:
249
- self.state_lock.release()
250
- return self.state
251
-
252
- def save_as(self, state_file: str, state: RecceState = None):
253
- if self.cloud_mode:
254
- raise Exception('Cannot save the state to Recce Cloud.')
255
-
256
- self.state_file = state_file
257
- self.export(state)
258
-
259
- def export(self, state: RecceState = None) -> Union[str, None]:
260
- if state is not None:
261
- self.update(state)
262
- # TODO: Export the current Recce state to file or cloud storage
263
- start_time = time.time()
264
- self.state_lock.acquire()
265
- try:
266
- if self.cloud_mode:
267
- message, state_etag = self._export_state_to_cloud()
268
- self.state_etag = state_etag
269
- else:
270
- if self.state_file is None:
271
- return 'No state file is provided. Skip storing the state.'
272
- logger.info(f"Store recce state to '{self.state_file}'")
273
- message = self._export_state_to_file()
274
- end_time = time.time()
275
- elapsed_time = end_time - start_time
276
- finally:
277
- self.state_lock.release()
278
- logger.info(f'Store state completed in {elapsed_time:.2f} seconds')
279
- return message
280
-
281
- def refresh(self):
282
- new_state = self.load(refresh=True)
283
- return new_state
284
-
285
- def check_conflict(self) -> bool:
286
- if not self.cloud_mode:
287
- return False
288
-
289
- if self.cloud_options.get('host', '').startswith('s3://'):
290
- return False
291
-
292
- metadata = self._get_metadata_from_recce_cloud()
293
- if not metadata:
294
- return False
295
-
296
- state_etag = metadata.get('etag')
297
- return state_etag != self.state_etag
298
-
299
- def info(self):
300
- if self.state is None:
301
- self.error_message = 'No state is loaded.'
302
- return None
303
-
304
- state_info = {
305
- 'mode': 'cloud' if self.cloud_mode else 'local',
306
- 'source': None,
307
- }
308
- if self.cloud_mode:
309
- if self.cloud_options.get('host', '').startswith('s3://'):
310
- state_info['source'] = self.cloud_options.get('host')
311
- else:
312
- state_info['source'] = 'Recce Cloud'
313
- state_info['pull_request'] = self.pr_info
314
- else:
315
- state_info['source'] = self.state_file
316
- return state_info
317
-
318
- def purge(self) -> bool:
319
- if self.cloud_mode is True:
320
- rc, err_msg = RecceCloudStateManager(self.cloud_options).purge_cloud_state()
321
- if err_msg:
322
- self.error_message = err_msg
323
- return rc
324
- else:
325
- if self.state_file is not None:
326
- try:
327
- os.remove(self.state_file)
328
- except Exception as e:
329
- self.error_message = f'Failed to remove the state file: {e}'
330
- return False
331
- else:
332
- self.error_message = 'No state file is provided. Skip removing the state file.'
333
- return False
334
-
335
- def _load_state_from_file(self, file_path: Optional[str] = None) -> RecceState:
336
- file_path = file_path or self.state_file
337
- return RecceState.from_file(file_path) if file_path else None
338
-
339
- def _load_state_from_cloud(self) -> Tuple[RecceState, str]:
340
- '''
341
- Load the state from Recce Cloud.
342
-
343
- Returns:
344
- RecceState: The state object.
345
- str: The etag of the state file.
346
- '''
347
- if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
348
- raise Exception('Cannot get the pull request information from GitHub.')
349
-
350
- if self.cloud_options.get('host', '').startswith('s3://'):
351
- logger.debug('Fetching state from AWS S3 bucket...')
352
- return self._load_state_from_s3_bucket(), None
353
- else:
354
- logger.debug('Fetching state from Recce Cloud...')
355
- metadata = self._get_metadata_from_recce_cloud()
356
- if metadata is None:
357
- return None, None
358
- state_etag = metadata.get('etag')
359
- if self.state_etag and state_etag == self.state_etag:
360
- return self.state, self.state_etag
361
-
362
- return self._load_state_from_recce_cloud(), state_etag
363
-
364
- def _get_metadata_from_recce_cloud(self) -> Union[dict, None]:
365
- recce_cloud = RecceCloud(token=self.cloud_options.get('token'))
366
- return recce_cloud.get_artifact_metadata(pr_info=self.pr_info)
367
-
368
- def _load_state_from_recce_cloud(self) -> Union[RecceState, None]:
369
- import tempfile
370
- import requests
371
-
372
- recce_cloud = RecceCloud(token=self.cloud_options.get('token'))
373
- presigned_url = recce_cloud.get_presigned_url(
374
- method=PresignedUrlMethod.DOWNLOAD,
375
- pr_id=self.pr_info.id,
376
- repository=self.pr_info.repository,
377
- artifact_name=RECCE_STATE_COMPRESSED_FILE
378
- )
379
-
380
- password = self.cloud_options.get('password')
381
- if password is None:
382
- raise Exception(RECCE_CLOUD_PASSWORD_MISSING.error_message)
383
-
384
- with tempfile.NamedTemporaryFile() as tmp:
385
- headers = s3_sse_c_headers(password)
386
- response = requests.get(presigned_url,
387
- headers=headers)
388
- if response.status_code == 404:
389
- self.error_message = 'The state file is not found in Recce Cloud.'
390
- return None
391
- elif response.status_code != 200:
392
- self.error_message = response.text
393
- raise Exception(
394
- f'{response.status_code} Failed to download the state file from Recce Cloud. The password could be wrong.')
395
- with open(tmp.name, 'wb') as f:
396
- f.write(response.content)
397
- return RecceState.from_file(tmp.name, file_type=SupportedFileTypes.GZIP)
398
-
399
- def _load_state_from_s3_bucket(self) -> Union[RecceState, None]:
400
- import boto3
401
- import tempfile
402
- s3_client = boto3.client('s3')
403
- s3_bucket_name = self.cloud_options.get('host').replace('s3://', '')
404
- s3_bucket_key = f'github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}'
405
-
406
- rc, error_message = check_s3_bucket(s3_bucket_name)
407
- if rc is False:
408
- raise Exception(error_message)
409
-
410
- with tempfile.NamedTemporaryFile() as tmp:
411
- try:
412
- s3_client.download_file(s3_bucket_name, s3_bucket_key, tmp.name)
413
- except botocore.exceptions.ClientError as e:
414
- error_code = e.response.get('Error', {}).get('Code')
415
- if error_code == '404':
416
- self.error_message = 'The state file is not found in the S3 bucket.'
417
- return None
418
- else:
419
- raise e
420
- return RecceState.from_file(tmp.name, file_type=SupportedFileTypes.GZIP)
421
-
422
- def _export_state_to_cloud(self) -> Tuple[Union[str, None], str]:
423
- if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
424
- raise Exception('Cannot get the pull request information from GitHub.')
425
-
426
- check_status = CheckDAO().status()
427
- metadata = {
428
- 'total_checks': check_status.get('total', 0),
429
- 'approved_checks': check_status.get('approved', 0),
430
- }
431
-
432
- if self.cloud_options.get('host', '').startswith('s3://'):
433
- logger.info("Store recce state to AWS S3 bucket")
434
- return self._export_state_to_s3_bucket(metadata=metadata), None
435
- else:
436
- logger.info("Store recce state to Recce Cloud")
437
- message = self._export_state_to_recce_cloud(metadata=metadata)
438
- metadata = self._get_metadata_from_recce_cloud()
439
- if metadata is None:
440
- return None
441
- state_etag = metadata.get('etag')
442
- return message, state_etag
443
-
444
- def _export_state_to_recce_cloud(self, metadata: dict = None) -> Union[str, None]:
445
- import tempfile
446
- import requests
447
-
448
- presigned_url = RecceCloud(token=self.cloud_options.get('token')).get_presigned_url(
449
- method=PresignedUrlMethod.UPLOAD, repository=self.pr_info.repository,
450
- artifact_name=RECCE_STATE_COMPRESSED_FILE,
451
- pr_id=self.pr_info.id,
452
- metadata=metadata)
453
- compress_passwd = self.cloud_options.get('password')
454
- headers = s3_sse_c_headers(compress_passwd)
455
- if metadata:
456
- headers['x-amz-tagging'] = urlencode(metadata)
457
- with tempfile.NamedTemporaryFile() as tmp:
458
- self._export_state_to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
459
- response = requests.put(presigned_url, data=open(tmp.name, 'rb').read(), headers=headers)
460
- if response.status_code != 200:
461
- self.error_message = response.text
462
- return 'Failed to upload the state file to Recce Cloud. Reason: ' + response.text
463
- return 'The state file is uploaded to Recce Cloud.'
464
-
465
- def _export_state_to_s3_bucket(self, metadata: dict = None) -> Union[str, None]:
466
- import boto3
467
- import tempfile
468
- s3_client = boto3.client('s3')
469
- s3_bucket_name = self.cloud_options.get('host').replace('s3://', '')
470
- s3_bucket_key = f'github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}'
471
-
472
- rc, error_message = check_s3_bucket(s3_bucket_name)
473
- if rc is False:
474
- raise Exception(error_message)
475
-
476
- with tempfile.NamedTemporaryFile() as tmp:
477
- self._export_state_to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
478
-
479
- s3_client.upload_file(tmp.name, s3_bucket_name, s3_bucket_key,
480
- # Casting all the values under metadata to string
481
- ExtraArgs={'Metadata': {k: str(v) for k, v in metadata.items()}})
482
- RecceCloud(token=self.cloud_options.get('token')).update_github_pull_request_check(self.pr_info, metadata)
483
- return f'The state file is uploaded to \' s3://{s3_bucket_name}/{s3_bucket_key}\''
484
-
485
- def _get_artifact_metadata_from_s3_bucket(self, artifact_name: str) -> Union[dict, None]:
486
- import boto3
487
- s3_client = boto3.client('s3')
488
- s3_bucket_name = self.cloud_options.get('host').replace('s3://', '')
489
- s3_bucket_key = f'github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{artifact_name}'
490
- try:
491
- response = s3_client.head_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
492
- metadata = response['Metadata']
493
- return metadata
494
- except botocore.exceptions.ClientError as e:
495
- self.error_message = e.response.get('Error', {}).get('Message')
496
- raise Exception('Failed to get artifact metadata from Recce Cloud.')
497
-
498
- def _export_state_to_file(self, file_path: Optional[str] = None,
499
- file_type: SupportedFileTypes = SupportedFileTypes.FILE) -> str:
500
- """
501
- Store the state to a file. Store happens when terminating the server or run instance.
502
- """
503
-
504
- file_path = file_path or self.state_file
505
- json_data = self.state.to_json()
506
- io = file_io_factory(file_type)
507
-
508
- io.write(file_path, json_data)
509
- return f'The state file is stored at \'{file_path}\''
510
-
511
-
512
- class RecceCloudStateManager:
513
- error_message: str
514
- hint_message: str
515
-
516
- # It is a class to upload, download and purge the state file on Recce Cloud.
517
-
518
- def __init__(self, cloud_options: Optional[Dict[str, str]] = None):
519
- self.cloud_options = cloud_options or {}
520
- self.pr_info = None
521
- self.error_message = None
522
- self.hint_message = None
523
-
524
- if not self.cloud_options.get('token'):
525
- raise Exception(RECCE_CLOUD_TOKEN_MISSING.error_message)
526
- self.pr_info = fetch_pr_metadata(cloud=True, github_token=self.cloud_options.get('token'))
527
- if self.pr_info.id is None:
528
- raise Exception('Cannot get the pull request information from GitHub.')
529
-
530
- def verify(self) -> bool:
531
- if self.cloud_options.get('token') is None:
532
- self.error_message = RECCE_CLOUD_TOKEN_MISSING.error_message
533
- self.hint_message = RECCE_CLOUD_TOKEN_MISSING.hint_message
534
- return False
535
- if self.cloud_options.get('password') is None:
536
- self.error_message = RECCE_CLOUD_PASSWORD_MISSING.error_message
537
- self.hint_message = RECCE_CLOUD_PASSWORD_MISSING.hint_message
538
- return False
539
- return True
540
-
541
- @property
542
- def error_and_hint(self) -> (Union[str, None], Union[str, None]):
543
- return self.error_message, self.hint_message
544
-
545
- def _check_state_in_recce_cloud(self) -> bool:
546
- return RecceCloud(token=self.cloud_options.get('token')).check_artifacts_exists(self.pr_info)
547
-
548
- def _check_state_in_s3_bucket(self) -> bool:
549
- import boto3
550
-
551
- s3_client = boto3.client('s3')
552
- s3_bucket_name = self.cloud_options.get('host').replace('s3://', '')
553
- s3_bucket_key = f'github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}'
554
- try:
555
- s3_client.head_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
556
- except botocore.exceptions.ClientError as e:
557
- error_code = e.response.get('Error', {}).get('Code')
558
- if error_code == '404':
559
- return False
560
- return True
561
-
562
- def check_cloud_state_exists(self) -> bool:
563
- if self.cloud_options.get('host', '').startswith('s3://'):
564
- return self._check_state_in_s3_bucket()
565
- else:
566
- return self._check_state_in_recce_cloud()
567
-
568
- def _upload_state_to_recce_cloud(self, state: RecceState, metadata: dict = None) -> Union[str, None]:
569
- import tempfile
570
- import requests
571
-
572
- presigned_url = RecceCloud(token=self.cloud_options.get('token')).get_presigned_url(
573
- method=PresignedUrlMethod.UPLOAD, repository=self.pr_info.repository,
574
- artifact_name=RECCE_STATE_COMPRESSED_FILE,
575
- pr_id=self.pr_info.id,
576
- metadata=metadata)
577
-
578
- compress_passwd = self.cloud_options.get('password')
579
- headers = s3_sse_c_headers(compress_passwd)
580
- with tempfile.NamedTemporaryFile() as tmp:
581
- state.to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
582
- response = requests.put(presigned_url, data=open(tmp.name, 'rb').read(), headers=headers)
583
- if response.status_code != 200:
584
- return f'Failed to upload the state file to Recce Cloud. Reason: {response.text}'
585
- return 'The state file is uploaded to Recce Cloud.'
586
-
587
- def _upload_state_to_s3_bucket(self, state: RecceState, metadata: dict = None) -> Union[str, None]:
588
- import boto3
589
- import tempfile
590
-
591
- s3_client = boto3.client('s3')
592
- s3_bucket_name = self.cloud_options.get('host').replace('s3://', '')
593
- s3_bucket_key = f'github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}'
594
-
595
- rc, error_message = check_s3_bucket(s3_bucket_name)
596
- if rc is False:
597
- raise Exception(error_message)
598
-
599
- with tempfile.NamedTemporaryFile() as tmp:
600
- state.to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
601
-
602
- s3_client.upload_file(tmp.name, s3_bucket_name, s3_bucket_key,
603
- # Casting all the values under metadata to string
604
- ExtraArgs={'Metadata': {k: str(v) for k, v in metadata.items()}})
605
- RecceCloud(token=self.cloud_options.get('token')).update_github_pull_request_check(self.pr_info, metadata)
606
- return f'The state file is uploaded to \' s3://{s3_bucket_name}/{s3_bucket_key}\''
607
-
608
- def upload_state_to_cloud(self, state: RecceState) -> Union[str, None]:
609
- if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
610
- raise Exception('Cannot get the pull request information from GitHub.')
611
-
612
- checks = state.checks
613
-
614
- metadata = {
615
- 'total_checks': len(checks),
616
- 'approved_checks': len([c for c in checks if c.is_checked]),
617
- }
618
-
619
- if self.cloud_options.get('host', '').startswith('s3://'):
620
- return self._upload_state_to_s3_bucket(state, metadata)
621
- else:
622
- return self._upload_state_to_recce_cloud(state, metadata)
623
-
624
- def _download_state_from_s3_bucket(self, filepath):
625
- import io
626
- import boto3
627
-
628
- s3_client = boto3.client('s3')
629
- s3_bucket_name = self.cloud_options.get('host').replace('s3://', '')
630
- s3_bucket_key = f'github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}'
631
-
632
- rc, error_message = check_s3_bucket(s3_bucket_name)
633
- if rc is False:
634
- raise Exception(error_message)
635
-
636
- response = s3_client.get_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
637
- byte_stream = io.BytesIO(response['Body'].read())
638
- gzip_io = file_io_factory(SupportedFileTypes.GZIP)
639
- decompressed_content = gzip_io.read_fileobj(byte_stream)
640
-
641
- dirs = os.path.dirname(filepath)
642
- if dirs:
643
- os.makedirs(dirs, exist_ok=True)
644
- with open(filepath, 'wb') as f:
645
- f.write(decompressed_content)
646
-
647
- def _download_state_from_recce_cloud(self, filepath):
648
- import io
649
- import requests
650
-
651
- presigned_url = RecceCloud(token=self.cloud_options.get('token')).get_presigned_url(
652
- method=PresignedUrlMethod.DOWNLOAD, repository=self.pr_info.repository,
653
- artifact_name=RECCE_STATE_COMPRESSED_FILE, pr_id=self.pr_info.id)
654
-
655
- password = self.cloud_options.get('password')
656
- if password is None:
657
- raise Exception(RECCE_CLOUD_PASSWORD_MISSING.error_message)
658
-
659
- headers = s3_sse_c_headers(password)
660
- response = requests.get(presigned_url, headers=headers)
661
-
662
- if response.status_code != 200:
663
- raise Exception(
664
- f'{response.status_code} Failed to download the state file from Recce Cloud. The password could be wrong.')
665
-
666
- byte_stream = io.BytesIO(response.content)
667
- gzip_io = file_io_factory(SupportedFileTypes.GZIP)
668
- decompressed_content = gzip_io.read_fileobj(byte_stream)
669
-
670
- dirs = os.path.dirname(filepath)
671
- if dirs:
672
- os.makedirs(dirs, exist_ok=True)
673
- with open(filepath, 'wb') as f:
674
- f.write(decompressed_content)
675
-
676
- def download_state_from_cloud(self, filepath: str) -> Union[str, None]:
677
- if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
678
- raise Exception('Cannot get the pull request information from GitHub.')
679
-
680
- if self.cloud_options.get('host', '').startswith('s3://'):
681
- logger.debug('Download state file from AWS S3 bucket...')
682
- return self._download_state_from_s3_bucket(filepath)
683
- else:
684
- logger.debug('Download state file from Recce Cloud...')
685
- return self._download_state_from_recce_cloud(filepath)
686
-
687
- def _purge_state_from_s3_bucket(self) -> (bool, str):
688
- import boto3
689
- from rich.console import Console
690
- console = Console()
691
- delete_objects = []
692
- logger.debug('Purging the state from AWS S3 bucket...')
693
- s3_client = boto3.client('s3')
694
- s3_bucket_name = self.cloud_options.get('host').replace('s3://', '')
695
- s3_key_prefix = f'github/{self.pr_info.repository}/pulls/{self.pr_info.id}/'
696
- list_response = s3_client.list_objects_v2(Bucket=s3_bucket_name, Prefix=s3_key_prefix)
697
- if 'Contents' in list_response:
698
- for obj in list_response['Contents']:
699
- key = obj['Key']
700
- delete_objects.append({'Key': key})
701
- console.print(f'[green]Deleted[/green]: {key}')
702
- else:
703
- return False, 'No state file found in the S3 bucket.'
704
-
705
- delete_response = s3_client.delete_objects(Bucket=s3_bucket_name, Delete={'Objects': delete_objects})
706
- if 'Deleted' not in delete_response:
707
- return False, 'Failed to delete the state file from the S3 bucket.'
708
- RecceCloud(token=self.cloud_options.get('token')).update_github_pull_request_check(self.pr_info)
709
- return True, None
710
-
711
- def _purge_state_from_recce_cloud(self) -> (bool, str):
712
- try:
713
- RecceCloud(token=self.cloud_options.get('token')).purge_artifacts(self.pr_info)
714
- except RecceCloudException as e:
715
- return False, e.reason
716
- return True, None
717
-
718
- def purge_cloud_state(self) -> (bool, str):
719
- if self.cloud_options.get('host', '').startswith('s3://'):
720
- return self._purge_state_from_s3_bucket()
721
- else:
722
- return self._purge_state_from_recce_cloud()
723
-
724
-
725
- class RecceShareStateManager:
726
- error_message: str
727
- hint_message: str
728
-
729
- # It is a class to share state file on Recce Cloud.
730
-
731
- def __init__(self, auth_options: Optional[Dict[str, str]] = None):
732
- self.auth_options = auth_options or {}
733
- self.error_message = None
734
- self.hint_message = None
735
-
736
- def verify(self) -> bool:
737
- if self.auth_options.get('api_token') is None:
738
- self.error_message = RECCE_API_TOKEN_MISSING.error_message
739
- self.hint_message = RECCE_API_TOKEN_MISSING.hint_message
740
- return False
741
- return True
742
-
743
- @property
744
- def error_and_hint(self) -> (Union[str, None], Union[str, None]):
745
- return self.error_message, self.hint_message
746
-
747
- def share_state(self, file_name: str, state: RecceState) -> Dict:
748
- import tempfile
749
-
750
- with tempfile.NamedTemporaryFile() as tmp:
751
- state.to_file(tmp.name, file_type=SupportedFileTypes.FILE)
752
- response = RecceCloud(token=self.auth_options.get('api_token')).share_state(file_name, open(tmp.name, 'rb'))
753
- return response