recce-nightly 1.9.0.20250623__py3-none-any.whl → 1.25.0.20251112a2066__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- recce/VERSION +1 -1
- recce/__init__.py +5 -0
- recce/adapter/dbt_adapter/__init__.py +318 -240
- recce/artifact.py +76 -3
- recce/cli.py +703 -71
- recce/config.py +3 -3
- recce/connect_to_cloud.py +138 -0
- recce/core.py +3 -3
- recce/data/404.html +1 -22
- recce/data/__next.__PAGE__.txt +10 -0
- recce/data/__next._full.txt +23 -0
- recce/data/__next._index.txt +8 -0
- recce/data/__next._tree.txt +12 -0
- recce/data/_next/static/6LypcDXgyuSaiSCrsmUub/_buildManifest.js +11 -0
- recce/data/_next/static/6LypcDXgyuSaiSCrsmUub/_clientMiddlewareManifest.json +1 -0
- recce/data/_next/static/chunks/0a2b2dd4b57049c2.js +1 -0
- recce/data/_next/static/chunks/19c10d219a6a21ff.js +1 -0
- recce/data/_next/static/chunks/24fd885c7180a612.js +1 -0
- recce/data/_next/static/chunks/27e66b2eab4adc32.js +19 -0
- recce/data/_next/static/chunks/71f88fcc615bf282.js +1 -0
- recce/data/_next/static/chunks/917619ab62a32388.js +1 -0
- recce/data/_next/static/chunks/93ba5a62932b704f.js +4 -0
- recce/data/_next/static/chunks/a43a2a5e06d5a92b.js +1 -0
- recce/data/_next/static/chunks/a6c78b24bd8b84fc.js +1 -0
- recce/data/_next/static/chunks/b2610ba997ff8c4f.js +110 -0
- recce/data/_next/static/chunks/ba2d87265a68599d.css +2 -0
- recce/data/_next/static/chunks/c117fd1c1382dd83.js +11 -0
- recce/data/_next/static/chunks/c9425ca46eebdde9.js +1 -0
- recce/data/_next/static/chunks/cc8a9eadba012be0.css +6 -0
- recce/data/_next/static/chunks/e124bccf574a3361.css +1 -0
- recce/data/_next/static/chunks/e392ad92847c3e17.js +1 -0
- recce/data/_next/static/chunks/e4ce95efe88dae79.js +11 -0
- recce/data/_next/static/chunks/e69c777814fea6ed.js +2 -0
- recce/data/_next/static/chunks/turbopack-21cfd73037ff57ab.js +3 -0
- recce/data/_next/static/media/favicon.a8d38d84.ico +0 -0
- recce/data/_next/static/media/montserrat-cyrillic-800-normal.d80d830d.woff2 +0 -0
- recce/data/_next/static/media/{montserrat-cyrillic-800-normal.bd5c9f50.woff → montserrat-cyrillic-800-normal.f9d58125.woff} +0 -0
- recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.076c2a93.woff2 +0 -0
- recce/data/_next/static/media/montserrat-latin-800-normal.cde454cc.woff2 +0 -0
- recce/data/_next/static/media/{montserrat-latin-800-normal.fc315020.woff → montserrat-latin-800-normal.d5761935.woff} +0 -0
- recce/data/_next/static/media/montserrat-latin-ext-800-normal.40ec0659.woff2 +0 -0
- recce/data/_next/static/media/{montserrat-latin-ext-800-normal.2e5381b2.woff → montserrat-latin-ext-800-normal.b671449b.woff} +0 -0
- recce/data/_next/static/media/{montserrat-vietnamese-800-normal.20c545e6.woff → montserrat-vietnamese-800-normal.9f7b8541.woff} +0 -0
- recce/data/_next/static/media/montserrat-vietnamese-800-normal.f9eb854e.woff2 +0 -0
- recce/data/_not-found/__next._full.txt +17 -0
- recce/data/_not-found/__next._index.txt +8 -0
- recce/data/_not-found/__next._not-found.__PAGE__.txt +5 -0
- recce/data/_not-found/__next._not-found.txt +4 -0
- recce/data/_not-found/__next._tree.txt +10 -0
- recce/data/_not-found.html +1 -0
- recce/data/_not-found.txt +17 -0
- recce/data/auth_callback.html +68 -0
- recce/data/index.html +1 -27
- recce/data/index.txt +23 -8
- recce/event/__init__.py +9 -8
- recce/event/collector.py +6 -2
- recce/event/track.py +10 -0
- recce/github.py +1 -1
- recce/mcp_server.py +632 -0
- recce/models/types.py +23 -2
- recce/pull_request.py +1 -1
- recce/run.py +23 -16
- recce/server.py +194 -19
- recce/state/__init__.py +31 -0
- recce/state/cloud.py +632 -0
- recce/state/const.py +26 -0
- recce/state/local.py +56 -0
- recce/state/state.py +119 -0
- recce/state/state_loader.py +174 -0
- recce/summary.py +2 -1
- recce/tasks/dataframe.py +59 -2
- recce/tasks/rowcount.py +4 -1
- recce/tasks/schema.py +4 -1
- recce/tasks/valuediff.py +1 -1
- recce/util/api_token.py +11 -2
- recce/util/breaking.py +9 -0
- recce/util/cll.py +1 -2
- recce/util/io.py +2 -2
- recce/util/lineage.py +19 -18
- recce/util/perf_tracking.py +85 -0
- recce/util/recce_cloud.py +229 -5
- recce/yaml/__init__.py +2 -2
- recce_cloud/__init__.py +15 -0
- recce_cloud/api/__init__.py +17 -0
- recce_cloud/api/base.py +104 -0
- recce_cloud/api/client.py +150 -0
- recce_cloud/api/exceptions.py +26 -0
- recce_cloud/api/factory.py +63 -0
- recce_cloud/api/github.py +72 -0
- recce_cloud/api/gitlab.py +78 -0
- recce_cloud/artifact.py +57 -0
- recce_cloud/ci_providers/__init__.py +9 -0
- recce_cloud/ci_providers/base.py +82 -0
- recce_cloud/ci_providers/detector.py +147 -0
- recce_cloud/ci_providers/github_actions.py +136 -0
- recce_cloud/ci_providers/gitlab_ci.py +130 -0
- recce_cloud/cli.py +303 -0
- recce_cloud/upload.py +213 -0
- {recce_nightly-1.9.0.20250623.dist-info → recce_nightly-1.25.0.20251112a2066.dist-info}/METADATA +31 -27
- recce_nightly-1.25.0.20251112a2066.dist-info/RECORD +178 -0
- {recce_nightly-1.9.0.20250623.dist-info → recce_nightly-1.25.0.20251112a2066.dist-info}/top_level.txt +1 -0
- tests/adapter/dbt_adapter/test_dbt_cll.py +412 -79
- tests/recce_cloud/__init__.py +0 -0
- tests/recce_cloud/test_ci_providers.py +351 -0
- tests/recce_cloud/test_cli.py +372 -0
- tests/recce_cloud/test_client.py +273 -0
- tests/recce_cloud/test_platform_clients.py +279 -0
- tests/test_cli.py +106 -3
- tests/test_cli_mcp_optional.py +45 -0
- tests/test_cloud_listing_cli.py +324 -0
- tests/test_connect_to_cloud.py +82 -0
- tests/test_core.py +148 -3
- tests/test_mcp_server.py +332 -0
- tests/test_server.py +6 -6
- tests/test_summary.py +14 -6
- recce/data/_next/static/WrRUb3nV8BhAZG_R8kVma/_buildManifest.js +0 -1
- recce/data/_next/static/chunks/181-acc61ddada3bc0ca.js +0 -43
- recce/data/_next/static/chunks/1bff33f1-1ef85cf5e658a751.js +0 -1
- recce/data/_next/static/chunks/217-879a84d70f7a907c.js +0 -2
- recce/data/_next/static/chunks/29e3cc0d-60045b2e47aa3916.js +0 -1
- recce/data/_next/static/chunks/36e1c10d-8e7be4a6c1f6ab2d.js +0 -1
- recce/data/_next/static/chunks/3998a672-03adacad07b346ac.js +0 -1
- recce/data/_next/static/chunks/3a92ee20-1081c360214f9602.js +0 -1
- recce/data/_next/static/chunks/42-cd3c06533f5fd47c.js +0 -9
- recce/data/_next/static/chunks/450c323b-fd94e7ffaa4a5efa.js +0 -1
- recce/data/_next/static/chunks/47d8844f-929aed9b1c73a905.js +0 -1
- recce/data/_next/static/chunks/608-3b079b544e5d5f5e.js +0 -15
- recce/data/_next/static/chunks/6dc81886-adbfa45836061d79.js +0 -1
- recce/data/_next/static/chunks/7a8a3e83-edf6dc64b5d5f0a5.js +0 -1
- recce/data/_next/static/chunks/7f27ae6c-d5f0438edd5c2a5b.js +0 -1
- recce/data/_next/static/chunks/86730205-cfb14e3f051bab35.js +0 -1
- recce/data/_next/static/chunks/8d700b6a.8bb140898499c512.js +0 -1
- recce/data/_next/static/chunks/92-7ab55ae02606193c.js +0 -1
- recce/data/_next/static/chunks/9746af58-a42b7d169cacadf0.js +0 -1
- recce/data/_next/static/chunks/a30376cd-de84559016d7e133.js +0 -1
- recce/data/_next/static/chunks/app/_not-found/page-01ed58b7f971d311.js +0 -1
- recce/data/_next/static/chunks/app/layout-177a410a97e0d018.js +0 -1
- recce/data/_next/static/chunks/app/page-59241c42b7dd4fcf.js +0 -1
- recce/data/_next/static/chunks/b63b1b3f-4282bdcf459e075c.js +0 -1
- recce/data/_next/static/chunks/bbda5537-9ec25eb1dd62348a.js +0 -1
- recce/data/_next/static/chunks/c132bf7d-08cb668a789d6afd.js +0 -1
- recce/data/_next/static/chunks/ce84277d-2e5d1d46910cf052.js +0 -1
- recce/data/_next/static/chunks/febdd86e-c6b525341634b860.js +0 -54
- recce/data/_next/static/chunks/fee69bc6-2dbccaf9b90474e6.js +0 -1
- recce/data/_next/static/chunks/framework-ded83d71b51ce901.js +0 -1
- recce/data/_next/static/chunks/main-app-39061b0166c47f55.js +0 -1
- recce/data/_next/static/chunks/main-b5b3ae20a1405261.js +0 -1
- recce/data/_next/static/chunks/pages/_app-437c455677d62394.js +0 -1
- recce/data/_next/static/chunks/pages/_error-e7650df18ca04bde.js +0 -1
- recce/data/_next/static/chunks/webpack-7b49d5ba7e3a434d.js +0 -1
- recce/data/_next/static/css/17a96168e3a9db13.css +0 -1
- recce/data/_next/static/css/1b121dc4d36aeb4d.css +0 -3
- recce/data/_next/static/css/35c6679a098e1e34.css +0 -1
- recce/data/_next/static/css/951e2e0eea2d4a5b.css +0 -14
- recce/data/_next/static/media/montserrat-cyrillic-800-normal.22628180.woff2 +0 -0
- recce/data/_next/static/media/montserrat-cyrillic-ext-800-normal.94a63aea.woff2 +0 -0
- recce/data/_next/static/media/montserrat-latin-800-normal.6f8fa298.woff2 +0 -0
- recce/data/_next/static/media/montserrat-latin-ext-800-normal.013b84f9.woff2 +0 -0
- recce/data/_next/static/media/montserrat-vietnamese-800-normal.c0035377.woff2 +0 -0
- recce/state.py +0 -785
- recce_nightly-1.9.0.20250623.dist-info/RECORD +0 -151
- tests/test_state.py +0 -134
- /recce/data/_next/static/{WrRUb3nV8BhAZG_R8kVma → 6LypcDXgyuSaiSCrsmUub}/_ssgManifest.js +0 -0
- /recce/data/_next/static/chunks/{polyfills-42372ed130431b0a.js → a6dad97d9634a72d.js} +0 -0
- /recce/data/_next/static/media/{montserrat-cyrillic-ext-800-normal.e6e0d8d0.woff → montserrat-cyrillic-ext-800-normal.a4fa76b5.woff} +0 -0
- /recce/data/_next/static/media/{reload-image.79aabb7d.svg → reload-image.7aa931c7.svg} +0 -0
- {recce_nightly-1.9.0.20250623.dist-info → recce_nightly-1.25.0.20251112a2066.dist-info}/WHEEL +0 -0
- {recce_nightly-1.9.0.20250623.dist-info → recce_nightly-1.25.0.20251112a2066.dist-info}/entry_points.txt +0 -0
- {recce_nightly-1.9.0.20250623.dist-info → recce_nightly-1.25.0.20251112a2066.dist-info}/licenses/LICENSE +0 -0
recce/state.py
DELETED
|
@@ -1,785 +0,0 @@
|
|
|
1
|
-
"""Define the type to serialize/de-serialize the state of the recce instance."""
|
|
2
|
-
|
|
3
|
-
import json
|
|
4
|
-
import logging
|
|
5
|
-
import os
|
|
6
|
-
import threading
|
|
7
|
-
import time
|
|
8
|
-
from base64 import b64encode
|
|
9
|
-
from dataclasses import dataclass
|
|
10
|
-
from datetime import datetime
|
|
11
|
-
from hashlib import md5, sha256
|
|
12
|
-
from typing import Dict, List, Optional, Tuple, Union
|
|
13
|
-
from urllib.parse import urlencode
|
|
14
|
-
|
|
15
|
-
import botocore.exceptions
|
|
16
|
-
from pydantic import BaseModel, Field
|
|
17
|
-
|
|
18
|
-
from recce import get_version
|
|
19
|
-
from recce.git import current_branch
|
|
20
|
-
from recce.models import CheckDAO
|
|
21
|
-
from recce.models.types import Check, Run
|
|
22
|
-
from recce.pull_request import PullRequestInfo, fetch_pr_metadata
|
|
23
|
-
from recce.util.io import SupportedFileTypes, file_io_factory
|
|
24
|
-
from recce.util.pydantic_model import pydantic_model_dump, pydantic_model_json_dump
|
|
25
|
-
from recce.util.recce_cloud import PresignedUrlMethod, RecceCloud, RecceCloudException
|
|
26
|
-
|
|
27
|
-
logger = logging.getLogger("uvicorn")
|
|
28
|
-
|
|
29
|
-
RECCE_STATE_FILE = "recce-state.json"
|
|
30
|
-
RECCE_STATE_COMPRESSED_FILE = f"{RECCE_STATE_FILE}.gz"
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
@dataclass
|
|
34
|
-
class ErrorMessage:
|
|
35
|
-
error_message: str
|
|
36
|
-
hint_message: str
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
RECCE_CLOUD_TOKEN_MISSING = ErrorMessage(
|
|
40
|
-
error_message="No GitHub token is provided to access the pull request information",
|
|
41
|
-
hint_message="Please provide a GitHub token in the command argument",
|
|
42
|
-
)
|
|
43
|
-
|
|
44
|
-
RECCE_CLOUD_PASSWORD_MISSING = ErrorMessage(
|
|
45
|
-
error_message="No password provided to access the state file in Recce Cloud",
|
|
46
|
-
hint_message='Please provide a password with the option "--password <compress-password>"',
|
|
47
|
-
)
|
|
48
|
-
|
|
49
|
-
RECCE_API_TOKEN_MISSING = ErrorMessage(
|
|
50
|
-
error_message="No Recc API token is provided",
|
|
51
|
-
hint_message="Please login to Recce Cloud and copy the API token from the settings page",
|
|
52
|
-
)
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
def s3_sse_c_headers(password: str) -> Dict[str, str]:
|
|
56
|
-
hashed_password = sha256()
|
|
57
|
-
md5_hash = md5()
|
|
58
|
-
hashed_password.update(password.encode())
|
|
59
|
-
md5_hash.update(hashed_password.digest())
|
|
60
|
-
encoded_passwd = b64encode(hashed_password.digest()).decode("utf-8")
|
|
61
|
-
encoded_md5 = b64encode(md5_hash.digest()).decode("utf-8")
|
|
62
|
-
return {
|
|
63
|
-
"x-amz-server-side-encryption-customer-algorithm": "AES256",
|
|
64
|
-
"x-amz-server-side-encryption-customer-key": encoded_passwd,
|
|
65
|
-
"x-amz-server-side-encryption-customer-key-MD5": encoded_md5,
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
def check_s3_bucket(bucket_name: str):
|
|
70
|
-
import boto3
|
|
71
|
-
|
|
72
|
-
s3_client = boto3.client("s3")
|
|
73
|
-
try:
|
|
74
|
-
s3_client.head_bucket(Bucket=bucket_name)
|
|
75
|
-
except botocore.exceptions.ClientError as e:
|
|
76
|
-
error_code = e.response["Error"]["Code"]
|
|
77
|
-
if error_code == "404":
|
|
78
|
-
return False, f"Bucket '{bucket_name}' does not exist."
|
|
79
|
-
elif error_code == "403":
|
|
80
|
-
return False, f"Bucket '{bucket_name}' exists but you do not have permission to access it."
|
|
81
|
-
else:
|
|
82
|
-
return False, f"Failed to access the S3 bucket: '{bucket_name}'"
|
|
83
|
-
return True, None
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
class GitRepoInfo(BaseModel):
|
|
87
|
-
branch: Optional[str] = None
|
|
88
|
-
|
|
89
|
-
@staticmethod
|
|
90
|
-
def from_current_repositroy():
|
|
91
|
-
branch = current_branch()
|
|
92
|
-
if branch is None:
|
|
93
|
-
return None
|
|
94
|
-
|
|
95
|
-
return GitRepoInfo(branch=branch)
|
|
96
|
-
|
|
97
|
-
def to_dict(self):
|
|
98
|
-
return pydantic_model_dump(self)
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
class RecceStateMetadata(BaseModel):
|
|
102
|
-
schema_version: str = "v0"
|
|
103
|
-
recce_version: str = Field(default_factory=lambda: get_version())
|
|
104
|
-
generated_at: str = Field(default_factory=lambda: datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"))
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
class ArtifactsRoot(BaseModel):
|
|
108
|
-
"""
|
|
109
|
-
Root of the artifacts.
|
|
110
|
-
|
|
111
|
-
base: artifacts of the base env. key is file name, value is dict
|
|
112
|
-
current: artifacts of the current env. key is file name, value is dict
|
|
113
|
-
"""
|
|
114
|
-
|
|
115
|
-
base: Dict[str, Optional[dict]] = {}
|
|
116
|
-
current: Dict[str, Optional[dict]] = {}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
class RecceState(BaseModel):
|
|
120
|
-
metadata: Optional[RecceStateMetadata] = None
|
|
121
|
-
runs: Optional[List[Run]] = Field(default_factory=list)
|
|
122
|
-
checks: Optional[List[Check]] = Field(default_factory=list)
|
|
123
|
-
artifacts: ArtifactsRoot = ArtifactsRoot(base={}, current={})
|
|
124
|
-
git: Optional[GitRepoInfo] = None
|
|
125
|
-
pull_request: Optional[PullRequestInfo] = None
|
|
126
|
-
|
|
127
|
-
@staticmethod
|
|
128
|
-
def from_json(json_content: str):
|
|
129
|
-
dict_data = json.loads(json_content)
|
|
130
|
-
state = RecceState(**dict_data)
|
|
131
|
-
metadata = state.metadata
|
|
132
|
-
if metadata:
|
|
133
|
-
if metadata.schema_version is None:
|
|
134
|
-
pass
|
|
135
|
-
if metadata.schema_version == "v0":
|
|
136
|
-
pass
|
|
137
|
-
else:
|
|
138
|
-
raise Exception(f"Unsupported state file version: {metadata.schema_version}")
|
|
139
|
-
return state
|
|
140
|
-
|
|
141
|
-
@staticmethod
|
|
142
|
-
def from_file(file_path: str, file_type: SupportedFileTypes = SupportedFileTypes.FILE):
|
|
143
|
-
"""
|
|
144
|
-
Load the state from a recce state file.
|
|
145
|
-
"""
|
|
146
|
-
from pathlib import Path
|
|
147
|
-
|
|
148
|
-
logger.debug(f"Load state file from: '{file_path}'")
|
|
149
|
-
if not Path(file_path).is_file():
|
|
150
|
-
return None
|
|
151
|
-
|
|
152
|
-
io = file_io_factory(file_type)
|
|
153
|
-
json_content = io.read(file_path)
|
|
154
|
-
return RecceState.from_json(json_content)
|
|
155
|
-
|
|
156
|
-
def to_json(self):
|
|
157
|
-
return pydantic_model_json_dump(self)
|
|
158
|
-
|
|
159
|
-
def to_file(self, file_path: str, file_type: SupportedFileTypes = SupportedFileTypes.FILE):
|
|
160
|
-
|
|
161
|
-
json_data = self.to_json()
|
|
162
|
-
io = file_io_factory(file_type)
|
|
163
|
-
|
|
164
|
-
io.write(file_path, json_data)
|
|
165
|
-
return f"The state file is stored at '{file_path}'"
|
|
166
|
-
|
|
167
|
-
def _merge_run(self, run: Run):
|
|
168
|
-
for r in self.runs:
|
|
169
|
-
if r.run_id == run.run_id:
|
|
170
|
-
break
|
|
171
|
-
else:
|
|
172
|
-
self.runs.append(run)
|
|
173
|
-
|
|
174
|
-
def _merge_check(self, check: Check):
|
|
175
|
-
for c in self.checks:
|
|
176
|
-
if c.check_id == check.check_id:
|
|
177
|
-
c.merge(check)
|
|
178
|
-
break
|
|
179
|
-
else:
|
|
180
|
-
self.checks.append(check)
|
|
181
|
-
|
|
182
|
-
def _merge_artifacts(self, artifacts: ArtifactsRoot):
|
|
183
|
-
self.artifacts.merge(artifacts)
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
class RecceStateLoader:
|
|
187
|
-
def __init__(
|
|
188
|
-
self,
|
|
189
|
-
review_mode: bool = False,
|
|
190
|
-
cloud_mode: bool = False,
|
|
191
|
-
state_file: Optional[str] = None,
|
|
192
|
-
cloud_options: Optional[Dict[str, str]] = None,
|
|
193
|
-
initial_state: Optional[RecceState] = None,
|
|
194
|
-
):
|
|
195
|
-
self.review_mode = review_mode
|
|
196
|
-
self.cloud_mode = cloud_mode
|
|
197
|
-
self.state_file = state_file
|
|
198
|
-
self.cloud_options = cloud_options or {}
|
|
199
|
-
self.error_message = None
|
|
200
|
-
self.hint_message = None
|
|
201
|
-
self.state: RecceState | None = initial_state
|
|
202
|
-
self.state_lock = threading.Lock()
|
|
203
|
-
self.state_etag = None
|
|
204
|
-
self.pr_info = None
|
|
205
|
-
|
|
206
|
-
if self.cloud_mode:
|
|
207
|
-
if not self.cloud_options.get("token"):
|
|
208
|
-
raise Exception(RECCE_CLOUD_TOKEN_MISSING.error_message)
|
|
209
|
-
self.pr_info = fetch_pr_metadata(cloud=self.cloud_mode, github_token=self.cloud_options.get("token"))
|
|
210
|
-
if self.pr_info.id is None:
|
|
211
|
-
raise Exception("Cannot get the pull request information from GitHub.")
|
|
212
|
-
|
|
213
|
-
# Load the state
|
|
214
|
-
self.load()
|
|
215
|
-
|
|
216
|
-
def verify(self) -> bool:
|
|
217
|
-
if self.cloud_mode:
|
|
218
|
-
if self.cloud_options.get("token") is None:
|
|
219
|
-
self.error_message = RECCE_CLOUD_TOKEN_MISSING.error_message
|
|
220
|
-
self.hint_message = RECCE_CLOUD_TOKEN_MISSING.hint_message
|
|
221
|
-
return False
|
|
222
|
-
if not self.cloud_options.get("host"):
|
|
223
|
-
if self.cloud_options.get("password") is None:
|
|
224
|
-
self.error_message = RECCE_CLOUD_PASSWORD_MISSING.error_message
|
|
225
|
-
self.hint_message = RECCE_CLOUD_PASSWORD_MISSING.hint_message
|
|
226
|
-
return False
|
|
227
|
-
else:
|
|
228
|
-
if self.review_mode is True and self.state_file is None:
|
|
229
|
-
self.error_message = "Recce can not launch without a state file."
|
|
230
|
-
self.hint_message = "Please provide a state file in the command argument."
|
|
231
|
-
return False
|
|
232
|
-
pass
|
|
233
|
-
return True
|
|
234
|
-
|
|
235
|
-
@property
|
|
236
|
-
def error_and_hint(self) -> (Union[str, None], Union[str, None]):
|
|
237
|
-
return self.error_message, self.hint_message
|
|
238
|
-
|
|
239
|
-
def update(self, state: RecceState):
|
|
240
|
-
self.state = state
|
|
241
|
-
|
|
242
|
-
def load(self, refresh=False) -> RecceState:
|
|
243
|
-
if self.state is not None and refresh is False:
|
|
244
|
-
return self.state
|
|
245
|
-
self.state_lock.acquire()
|
|
246
|
-
try:
|
|
247
|
-
if self.cloud_mode:
|
|
248
|
-
self.state, self.state_etag = self._load_state_from_cloud()
|
|
249
|
-
elif self.state_file:
|
|
250
|
-
self.state = self._load_state_from_file()
|
|
251
|
-
finally:
|
|
252
|
-
self.state_lock.release()
|
|
253
|
-
return self.state
|
|
254
|
-
|
|
255
|
-
def save_as(self, state_file: str, state: RecceState = None):
|
|
256
|
-
if self.cloud_mode:
|
|
257
|
-
raise Exception("Cannot save the state to Recce Cloud.")
|
|
258
|
-
|
|
259
|
-
self.state_file = state_file
|
|
260
|
-
self.export(state)
|
|
261
|
-
|
|
262
|
-
def export(self, state: RecceState = None) -> Union[str, None]:
|
|
263
|
-
if state is not None:
|
|
264
|
-
self.update(state)
|
|
265
|
-
# TODO: Export the current Recce state to file or cloud storage
|
|
266
|
-
start_time = time.time()
|
|
267
|
-
self.state_lock.acquire()
|
|
268
|
-
try:
|
|
269
|
-
if self.cloud_mode:
|
|
270
|
-
message, state_etag = self._export_state_to_cloud()
|
|
271
|
-
self.state_etag = state_etag
|
|
272
|
-
else:
|
|
273
|
-
if self.state_file is None:
|
|
274
|
-
return "No state file is provided. Skip storing the state."
|
|
275
|
-
logger.info(f"Store recce state to '{self.state_file}'")
|
|
276
|
-
message = self._export_state_to_file()
|
|
277
|
-
end_time = time.time()
|
|
278
|
-
elapsed_time = end_time - start_time
|
|
279
|
-
finally:
|
|
280
|
-
self.state_lock.release()
|
|
281
|
-
logger.info(f"Store state completed in {elapsed_time:.2f} seconds")
|
|
282
|
-
return message
|
|
283
|
-
|
|
284
|
-
def refresh(self):
|
|
285
|
-
new_state = self.load(refresh=True)
|
|
286
|
-
return new_state
|
|
287
|
-
|
|
288
|
-
def check_conflict(self) -> bool:
|
|
289
|
-
if not self.cloud_mode:
|
|
290
|
-
return False
|
|
291
|
-
|
|
292
|
-
if self.cloud_options.get("host", "").startswith("s3://"):
|
|
293
|
-
return False
|
|
294
|
-
|
|
295
|
-
metadata = self._get_metadata_from_recce_cloud()
|
|
296
|
-
if not metadata:
|
|
297
|
-
return False
|
|
298
|
-
|
|
299
|
-
state_etag = metadata.get("etag")
|
|
300
|
-
return state_etag != self.state_etag
|
|
301
|
-
|
|
302
|
-
def info(self):
|
|
303
|
-
if self.state is None:
|
|
304
|
-
self.error_message = "No state is loaded."
|
|
305
|
-
return None
|
|
306
|
-
|
|
307
|
-
state_info = {
|
|
308
|
-
"mode": "cloud" if self.cloud_mode else "local",
|
|
309
|
-
"source": None,
|
|
310
|
-
}
|
|
311
|
-
if self.cloud_mode:
|
|
312
|
-
if self.cloud_options.get("host", "").startswith("s3://"):
|
|
313
|
-
state_info["source"] = self.cloud_options.get("host")
|
|
314
|
-
else:
|
|
315
|
-
state_info["source"] = "Recce Cloud"
|
|
316
|
-
state_info["pull_request"] = self.pr_info
|
|
317
|
-
else:
|
|
318
|
-
state_info["source"] = self.state_file
|
|
319
|
-
return state_info
|
|
320
|
-
|
|
321
|
-
def purge(self) -> bool:
|
|
322
|
-
if self.cloud_mode is True:
|
|
323
|
-
rc, err_msg = RecceCloudStateManager(self.cloud_options).purge_cloud_state()
|
|
324
|
-
if err_msg:
|
|
325
|
-
self.error_message = err_msg
|
|
326
|
-
return rc
|
|
327
|
-
else:
|
|
328
|
-
if self.state_file is not None:
|
|
329
|
-
try:
|
|
330
|
-
os.remove(self.state_file)
|
|
331
|
-
except Exception as e:
|
|
332
|
-
self.error_message = f"Failed to remove the state file: {e}"
|
|
333
|
-
return False
|
|
334
|
-
else:
|
|
335
|
-
self.error_message = "No state file is provided. Skip removing the state file."
|
|
336
|
-
return False
|
|
337
|
-
|
|
338
|
-
def _load_state_from_file(self, file_path: Optional[str] = None) -> RecceState:
|
|
339
|
-
file_path = file_path or self.state_file
|
|
340
|
-
return RecceState.from_file(file_path) if file_path else None
|
|
341
|
-
|
|
342
|
-
def _load_state_from_cloud(self) -> Tuple[RecceState, str]:
|
|
343
|
-
"""
|
|
344
|
-
Load the state from Recce Cloud.
|
|
345
|
-
|
|
346
|
-
Returns:
|
|
347
|
-
RecceState: The state object.
|
|
348
|
-
str: The etag of the state file.
|
|
349
|
-
"""
|
|
350
|
-
if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
|
|
351
|
-
raise Exception("Cannot get the pull request information from GitHub.")
|
|
352
|
-
|
|
353
|
-
if self.cloud_options.get("host", "").startswith("s3://"):
|
|
354
|
-
logger.debug("Fetching state from AWS S3 bucket...")
|
|
355
|
-
return self._load_state_from_s3_bucket(), None
|
|
356
|
-
else:
|
|
357
|
-
logger.debug("Fetching state from Recce Cloud...")
|
|
358
|
-
metadata = self._get_metadata_from_recce_cloud()
|
|
359
|
-
if metadata is None:
|
|
360
|
-
return None, None
|
|
361
|
-
state_etag = metadata.get("etag")
|
|
362
|
-
if self.state_etag and state_etag == self.state_etag:
|
|
363
|
-
return self.state, self.state_etag
|
|
364
|
-
|
|
365
|
-
return self._load_state_from_recce_cloud(), state_etag
|
|
366
|
-
|
|
367
|
-
def _get_metadata_from_recce_cloud(self) -> Union[dict, None]:
|
|
368
|
-
recce_cloud = RecceCloud(token=self.cloud_options.get("token"))
|
|
369
|
-
return recce_cloud.get_artifact_metadata(pr_info=self.pr_info)
|
|
370
|
-
|
|
371
|
-
def _load_state_from_recce_cloud(self) -> Union[RecceState, None]:
|
|
372
|
-
import tempfile
|
|
373
|
-
|
|
374
|
-
import requests
|
|
375
|
-
|
|
376
|
-
recce_cloud = RecceCloud(token=self.cloud_options.get("token"))
|
|
377
|
-
presigned_url = recce_cloud.get_presigned_url(
|
|
378
|
-
method=PresignedUrlMethod.DOWNLOAD,
|
|
379
|
-
pr_id=self.pr_info.id,
|
|
380
|
-
repository=self.pr_info.repository,
|
|
381
|
-
artifact_name=RECCE_STATE_COMPRESSED_FILE,
|
|
382
|
-
)
|
|
383
|
-
|
|
384
|
-
password = self.cloud_options.get("password")
|
|
385
|
-
if password is None:
|
|
386
|
-
raise Exception(RECCE_CLOUD_PASSWORD_MISSING.error_message)
|
|
387
|
-
|
|
388
|
-
with tempfile.NamedTemporaryFile() as tmp:
|
|
389
|
-
headers = s3_sse_c_headers(password)
|
|
390
|
-
response = requests.get(presigned_url, headers=headers)
|
|
391
|
-
if response.status_code == 404:
|
|
392
|
-
self.error_message = "The state file is not found in Recce Cloud."
|
|
393
|
-
return None
|
|
394
|
-
elif response.status_code != 200:
|
|
395
|
-
self.error_message = response.text
|
|
396
|
-
raise Exception(
|
|
397
|
-
f"{response.status_code} Failed to download the state file from Recce Cloud. The password could be wrong."
|
|
398
|
-
)
|
|
399
|
-
with open(tmp.name, "wb") as f:
|
|
400
|
-
f.write(response.content)
|
|
401
|
-
return RecceState.from_file(tmp.name, file_type=SupportedFileTypes.GZIP)
|
|
402
|
-
|
|
403
|
-
def _load_state_from_s3_bucket(self) -> Union[RecceState, None]:
|
|
404
|
-
import tempfile
|
|
405
|
-
|
|
406
|
-
import boto3
|
|
407
|
-
|
|
408
|
-
s3_client = boto3.client("s3")
|
|
409
|
-
s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
|
|
410
|
-
s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
|
|
411
|
-
|
|
412
|
-
rc, error_message = check_s3_bucket(s3_bucket_name)
|
|
413
|
-
if rc is False:
|
|
414
|
-
raise Exception(error_message)
|
|
415
|
-
|
|
416
|
-
with tempfile.NamedTemporaryFile() as tmp:
|
|
417
|
-
try:
|
|
418
|
-
s3_client.download_file(s3_bucket_name, s3_bucket_key, tmp.name)
|
|
419
|
-
except botocore.exceptions.ClientError as e:
|
|
420
|
-
error_code = e.response.get("Error", {}).get("Code")
|
|
421
|
-
if error_code == "404":
|
|
422
|
-
self.error_message = "The state file is not found in the S3 bucket."
|
|
423
|
-
return None
|
|
424
|
-
else:
|
|
425
|
-
raise e
|
|
426
|
-
return RecceState.from_file(tmp.name, file_type=SupportedFileTypes.GZIP)
|
|
427
|
-
|
|
428
|
-
def _export_state_to_cloud(self) -> Tuple[Union[str, None], str]:
|
|
429
|
-
if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
|
|
430
|
-
raise Exception("Cannot get the pull request information from GitHub.")
|
|
431
|
-
|
|
432
|
-
check_status = CheckDAO().status()
|
|
433
|
-
metadata = {
|
|
434
|
-
"total_checks": check_status.get("total", 0),
|
|
435
|
-
"approved_checks": check_status.get("approved", 0),
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
if self.cloud_options.get("host", "").startswith("s3://"):
|
|
439
|
-
logger.info("Store recce state to AWS S3 bucket")
|
|
440
|
-
return self._export_state_to_s3_bucket(metadata=metadata), None
|
|
441
|
-
else:
|
|
442
|
-
logger.info("Store recce state to Recce Cloud")
|
|
443
|
-
message = self._export_state_to_recce_cloud(metadata=metadata)
|
|
444
|
-
metadata = self._get_metadata_from_recce_cloud()
|
|
445
|
-
if metadata is None:
|
|
446
|
-
return None
|
|
447
|
-
state_etag = metadata.get("etag")
|
|
448
|
-
return message, state_etag
|
|
449
|
-
|
|
450
|
-
def _export_state_to_recce_cloud(self, metadata: dict = None) -> Union[str, None]:
|
|
451
|
-
import tempfile
|
|
452
|
-
|
|
453
|
-
import requests
|
|
454
|
-
|
|
455
|
-
presigned_url = RecceCloud(token=self.cloud_options.get("token")).get_presigned_url(
|
|
456
|
-
method=PresignedUrlMethod.UPLOAD,
|
|
457
|
-
repository=self.pr_info.repository,
|
|
458
|
-
artifact_name=RECCE_STATE_COMPRESSED_FILE,
|
|
459
|
-
pr_id=self.pr_info.id,
|
|
460
|
-
metadata=metadata,
|
|
461
|
-
)
|
|
462
|
-
compress_passwd = self.cloud_options.get("password")
|
|
463
|
-
headers = s3_sse_c_headers(compress_passwd)
|
|
464
|
-
if metadata:
|
|
465
|
-
headers["x-amz-tagging"] = urlencode(metadata)
|
|
466
|
-
with tempfile.NamedTemporaryFile() as tmp:
|
|
467
|
-
self._export_state_to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
|
|
468
|
-
response = requests.put(presigned_url, data=open(tmp.name, "rb").read(), headers=headers)
|
|
469
|
-
if response.status_code != 200:
|
|
470
|
-
self.error_message = response.text
|
|
471
|
-
return "Failed to upload the state file to Recce Cloud. Reason: " + response.text
|
|
472
|
-
return "The state file is uploaded to Recce Cloud."
|
|
473
|
-
|
|
474
|
-
def _export_state_to_s3_bucket(self, metadata: dict = None) -> Union[str, None]:
|
|
475
|
-
import tempfile
|
|
476
|
-
|
|
477
|
-
import boto3
|
|
478
|
-
|
|
479
|
-
s3_client = boto3.client("s3")
|
|
480
|
-
s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
|
|
481
|
-
s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
|
|
482
|
-
|
|
483
|
-
rc, error_message = check_s3_bucket(s3_bucket_name)
|
|
484
|
-
if rc is False:
|
|
485
|
-
raise Exception(error_message)
|
|
486
|
-
|
|
487
|
-
with tempfile.NamedTemporaryFile() as tmp:
|
|
488
|
-
self._export_state_to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
|
|
489
|
-
|
|
490
|
-
s3_client.upload_file(
|
|
491
|
-
tmp.name,
|
|
492
|
-
s3_bucket_name,
|
|
493
|
-
s3_bucket_key,
|
|
494
|
-
# Casting all the values under metadata to string
|
|
495
|
-
ExtraArgs={"Metadata": {k: str(v) for k, v in metadata.items()}},
|
|
496
|
-
)
|
|
497
|
-
RecceCloud(token=self.cloud_options.get("token")).update_github_pull_request_check(self.pr_info, metadata)
|
|
498
|
-
return f"The state file is uploaded to ' s3://{s3_bucket_name}/{s3_bucket_key}'"
|
|
499
|
-
|
|
500
|
-
def _get_artifact_metadata_from_s3_bucket(self, artifact_name: str) -> Union[dict, None]:
|
|
501
|
-
import boto3
|
|
502
|
-
|
|
503
|
-
s3_client = boto3.client("s3")
|
|
504
|
-
s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
|
|
505
|
-
s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{artifact_name}"
|
|
506
|
-
try:
|
|
507
|
-
response = s3_client.head_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
|
|
508
|
-
metadata = response["Metadata"]
|
|
509
|
-
return metadata
|
|
510
|
-
except botocore.exceptions.ClientError as e:
|
|
511
|
-
self.error_message = e.response.get("Error", {}).get("Message")
|
|
512
|
-
raise Exception("Failed to get artifact metadata from Recce Cloud.")
|
|
513
|
-
|
|
514
|
-
def _export_state_to_file(
|
|
515
|
-
self, file_path: Optional[str] = None, file_type: SupportedFileTypes = SupportedFileTypes.FILE
|
|
516
|
-
) -> str:
|
|
517
|
-
"""
|
|
518
|
-
Store the state to a file. Store happens when terminating the server or run instance.
|
|
519
|
-
"""
|
|
520
|
-
|
|
521
|
-
file_path = file_path or self.state_file
|
|
522
|
-
json_data = self.state.to_json()
|
|
523
|
-
io = file_io_factory(file_type)
|
|
524
|
-
|
|
525
|
-
io.write(file_path, json_data)
|
|
526
|
-
return f"The state file is stored at '{file_path}'"
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
class RecceCloudStateManager:
|
|
530
|
-
error_message: str
|
|
531
|
-
hint_message: str
|
|
532
|
-
|
|
533
|
-
# It is a class to upload, download and purge the state file on Recce Cloud.
|
|
534
|
-
|
|
535
|
-
def __init__(self, cloud_options: Optional[Dict[str, str]] = None):
|
|
536
|
-
self.cloud_options = cloud_options or {}
|
|
537
|
-
self.pr_info = None
|
|
538
|
-
self.error_message = None
|
|
539
|
-
self.hint_message = None
|
|
540
|
-
|
|
541
|
-
if not self.cloud_options.get("token"):
|
|
542
|
-
raise Exception(RECCE_CLOUD_TOKEN_MISSING.error_message)
|
|
543
|
-
self.pr_info = fetch_pr_metadata(cloud=True, github_token=self.cloud_options.get("token"))
|
|
544
|
-
if self.pr_info.id is None:
|
|
545
|
-
raise Exception("Cannot get the pull request information from GitHub.")
|
|
546
|
-
|
|
547
|
-
def verify(self) -> bool:
|
|
548
|
-
if self.cloud_options.get("token") is None:
|
|
549
|
-
self.error_message = RECCE_CLOUD_TOKEN_MISSING.error_message
|
|
550
|
-
self.hint_message = RECCE_CLOUD_TOKEN_MISSING.hint_message
|
|
551
|
-
return False
|
|
552
|
-
if self.cloud_options.get("password") is None:
|
|
553
|
-
self.error_message = RECCE_CLOUD_PASSWORD_MISSING.error_message
|
|
554
|
-
self.hint_message = RECCE_CLOUD_PASSWORD_MISSING.hint_message
|
|
555
|
-
return False
|
|
556
|
-
return True
|
|
557
|
-
|
|
558
|
-
@property
|
|
559
|
-
def error_and_hint(self) -> (Union[str, None], Union[str, None]):
|
|
560
|
-
return self.error_message, self.hint_message
|
|
561
|
-
|
|
562
|
-
def _check_state_in_recce_cloud(self) -> bool:
|
|
563
|
-
return RecceCloud(token=self.cloud_options.get("token")).check_artifacts_exists(self.pr_info)
|
|
564
|
-
|
|
565
|
-
def _check_state_in_s3_bucket(self) -> bool:
|
|
566
|
-
import boto3
|
|
567
|
-
|
|
568
|
-
s3_client = boto3.client("s3")
|
|
569
|
-
s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
|
|
570
|
-
s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
|
|
571
|
-
try:
|
|
572
|
-
s3_client.head_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
|
|
573
|
-
except botocore.exceptions.ClientError as e:
|
|
574
|
-
error_code = e.response.get("Error", {}).get("Code")
|
|
575
|
-
if error_code == "404":
|
|
576
|
-
return False
|
|
577
|
-
return True
|
|
578
|
-
|
|
579
|
-
def check_cloud_state_exists(self) -> bool:
|
|
580
|
-
if self.cloud_options.get("host", "").startswith("s3://"):
|
|
581
|
-
return self._check_state_in_s3_bucket()
|
|
582
|
-
else:
|
|
583
|
-
return self._check_state_in_recce_cloud()
|
|
584
|
-
|
|
585
|
-
def _upload_state_to_recce_cloud(self, state: RecceState, metadata: dict = None) -> Union[str, None]:
|
|
586
|
-
import tempfile
|
|
587
|
-
|
|
588
|
-
import requests
|
|
589
|
-
|
|
590
|
-
presigned_url = RecceCloud(token=self.cloud_options.get("token")).get_presigned_url(
|
|
591
|
-
method=PresignedUrlMethod.UPLOAD,
|
|
592
|
-
repository=self.pr_info.repository,
|
|
593
|
-
artifact_name=RECCE_STATE_COMPRESSED_FILE,
|
|
594
|
-
pr_id=self.pr_info.id,
|
|
595
|
-
metadata=metadata,
|
|
596
|
-
)
|
|
597
|
-
|
|
598
|
-
compress_passwd = self.cloud_options.get("password")
|
|
599
|
-
headers = s3_sse_c_headers(compress_passwd)
|
|
600
|
-
with tempfile.NamedTemporaryFile() as tmp:
|
|
601
|
-
state.to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
|
|
602
|
-
response = requests.put(presigned_url, data=open(tmp.name, "rb").read(), headers=headers)
|
|
603
|
-
if response.status_code != 200:
|
|
604
|
-
return f"Failed to upload the state file to Recce Cloud. Reason: {response.text}"
|
|
605
|
-
return "The state file is uploaded to Recce Cloud."
|
|
606
|
-
|
|
607
|
-
def _upload_state_to_s3_bucket(self, state: RecceState, metadata: dict = None) -> Union[str, None]:
|
|
608
|
-
import tempfile
|
|
609
|
-
|
|
610
|
-
import boto3
|
|
611
|
-
|
|
612
|
-
s3_client = boto3.client("s3")
|
|
613
|
-
s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
|
|
614
|
-
s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
|
|
615
|
-
|
|
616
|
-
rc, error_message = check_s3_bucket(s3_bucket_name)
|
|
617
|
-
if rc is False:
|
|
618
|
-
raise Exception(error_message)
|
|
619
|
-
|
|
620
|
-
with tempfile.NamedTemporaryFile() as tmp:
|
|
621
|
-
state.to_file(tmp.name, file_type=SupportedFileTypes.GZIP)
|
|
622
|
-
|
|
623
|
-
s3_client.upload_file(
|
|
624
|
-
tmp.name,
|
|
625
|
-
s3_bucket_name,
|
|
626
|
-
s3_bucket_key,
|
|
627
|
-
# Casting all the values under metadata to string
|
|
628
|
-
ExtraArgs={"Metadata": {k: str(v) for k, v in metadata.items()}},
|
|
629
|
-
)
|
|
630
|
-
RecceCloud(token=self.cloud_options.get("token")).update_github_pull_request_check(self.pr_info, metadata)
|
|
631
|
-
return f"The state file is uploaded to ' s3://{s3_bucket_name}/{s3_bucket_key}'"
|
|
632
|
-
|
|
633
|
-
def upload_state_to_cloud(self, state: RecceState) -> Union[str, None]:
|
|
634
|
-
if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
|
|
635
|
-
raise Exception("Cannot get the pull request information from GitHub.")
|
|
636
|
-
|
|
637
|
-
checks = state.checks
|
|
638
|
-
|
|
639
|
-
metadata = {
|
|
640
|
-
"total_checks": len(checks),
|
|
641
|
-
"approved_checks": len([c for c in checks if c.is_checked]),
|
|
642
|
-
}
|
|
643
|
-
|
|
644
|
-
if self.cloud_options.get("host", "").startswith("s3://"):
|
|
645
|
-
return self._upload_state_to_s3_bucket(state, metadata)
|
|
646
|
-
else:
|
|
647
|
-
return self._upload_state_to_recce_cloud(state, metadata)
|
|
648
|
-
|
|
649
|
-
def _download_state_from_s3_bucket(self, filepath):
|
|
650
|
-
import io
|
|
651
|
-
|
|
652
|
-
import boto3
|
|
653
|
-
|
|
654
|
-
s3_client = boto3.client("s3")
|
|
655
|
-
s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
|
|
656
|
-
s3_bucket_key = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/{RECCE_STATE_COMPRESSED_FILE}"
|
|
657
|
-
|
|
658
|
-
rc, error_message = check_s3_bucket(s3_bucket_name)
|
|
659
|
-
if rc is False:
|
|
660
|
-
raise Exception(error_message)
|
|
661
|
-
|
|
662
|
-
response = s3_client.get_object(Bucket=s3_bucket_name, Key=s3_bucket_key)
|
|
663
|
-
byte_stream = io.BytesIO(response["Body"].read())
|
|
664
|
-
gzip_io = file_io_factory(SupportedFileTypes.GZIP)
|
|
665
|
-
decompressed_content = gzip_io.read_fileobj(byte_stream)
|
|
666
|
-
|
|
667
|
-
dirs = os.path.dirname(filepath)
|
|
668
|
-
if dirs:
|
|
669
|
-
os.makedirs(dirs, exist_ok=True)
|
|
670
|
-
with open(filepath, "wb") as f:
|
|
671
|
-
f.write(decompressed_content)
|
|
672
|
-
|
|
673
|
-
def _download_state_from_recce_cloud(self, filepath):
|
|
674
|
-
import io
|
|
675
|
-
|
|
676
|
-
import requests
|
|
677
|
-
|
|
678
|
-
presigned_url = RecceCloud(token=self.cloud_options.get("token")).get_presigned_url(
|
|
679
|
-
method=PresignedUrlMethod.DOWNLOAD,
|
|
680
|
-
repository=self.pr_info.repository,
|
|
681
|
-
artifact_name=RECCE_STATE_COMPRESSED_FILE,
|
|
682
|
-
pr_id=self.pr_info.id,
|
|
683
|
-
)
|
|
684
|
-
|
|
685
|
-
password = self.cloud_options.get("password")
|
|
686
|
-
if password is None:
|
|
687
|
-
raise Exception(RECCE_CLOUD_PASSWORD_MISSING.error_message)
|
|
688
|
-
|
|
689
|
-
headers = s3_sse_c_headers(password)
|
|
690
|
-
response = requests.get(presigned_url, headers=headers)
|
|
691
|
-
|
|
692
|
-
if response.status_code != 200:
|
|
693
|
-
raise Exception(
|
|
694
|
-
f"{response.status_code} Failed to download the state file from Recce Cloud. The password could be wrong."
|
|
695
|
-
)
|
|
696
|
-
|
|
697
|
-
byte_stream = io.BytesIO(response.content)
|
|
698
|
-
gzip_io = file_io_factory(SupportedFileTypes.GZIP)
|
|
699
|
-
decompressed_content = gzip_io.read_fileobj(byte_stream)
|
|
700
|
-
|
|
701
|
-
dirs = os.path.dirname(filepath)
|
|
702
|
-
if dirs:
|
|
703
|
-
os.makedirs(dirs, exist_ok=True)
|
|
704
|
-
with open(filepath, "wb") as f:
|
|
705
|
-
f.write(decompressed_content)
|
|
706
|
-
|
|
707
|
-
def download_state_from_cloud(self, filepath: str) -> Union[str, None]:
|
|
708
|
-
if (self.pr_info is None) or (self.pr_info.id is None) or (self.pr_info.repository is None):
|
|
709
|
-
raise Exception("Cannot get the pull request information from GitHub.")
|
|
710
|
-
|
|
711
|
-
if self.cloud_options.get("host", "").startswith("s3://"):
|
|
712
|
-
logger.debug("Download state file from AWS S3 bucket...")
|
|
713
|
-
return self._download_state_from_s3_bucket(filepath)
|
|
714
|
-
else:
|
|
715
|
-
logger.debug("Download state file from Recce Cloud...")
|
|
716
|
-
return self._download_state_from_recce_cloud(filepath)
|
|
717
|
-
|
|
718
|
-
def _purge_state_from_s3_bucket(self) -> (bool, str):
|
|
719
|
-
import boto3
|
|
720
|
-
from rich.console import Console
|
|
721
|
-
|
|
722
|
-
console = Console()
|
|
723
|
-
delete_objects = []
|
|
724
|
-
logger.debug("Purging the state from AWS S3 bucket...")
|
|
725
|
-
s3_client = boto3.client("s3")
|
|
726
|
-
s3_bucket_name = self.cloud_options.get("host").replace("s3://", "")
|
|
727
|
-
s3_key_prefix = f"github/{self.pr_info.repository}/pulls/{self.pr_info.id}/"
|
|
728
|
-
list_response = s3_client.list_objects_v2(Bucket=s3_bucket_name, Prefix=s3_key_prefix)
|
|
729
|
-
if "Contents" in list_response:
|
|
730
|
-
for obj in list_response["Contents"]:
|
|
731
|
-
key = obj["Key"]
|
|
732
|
-
delete_objects.append({"Key": key})
|
|
733
|
-
console.print(f"[green]Deleted[/green]: {key}")
|
|
734
|
-
else:
|
|
735
|
-
return False, "No state file found in the S3 bucket."
|
|
736
|
-
|
|
737
|
-
delete_response = s3_client.delete_objects(Bucket=s3_bucket_name, Delete={"Objects": delete_objects})
|
|
738
|
-
if "Deleted" not in delete_response:
|
|
739
|
-
return False, "Failed to delete the state file from the S3 bucket."
|
|
740
|
-
RecceCloud(token=self.cloud_options.get("token")).update_github_pull_request_check(self.pr_info)
|
|
741
|
-
return True, None
|
|
742
|
-
|
|
743
|
-
def _purge_state_from_recce_cloud(self) -> (bool, str):
|
|
744
|
-
try:
|
|
745
|
-
RecceCloud(token=self.cloud_options.get("token")).purge_artifacts(self.pr_info)
|
|
746
|
-
except RecceCloudException as e:
|
|
747
|
-
return False, e.reason
|
|
748
|
-
return True, None
|
|
749
|
-
|
|
750
|
-
def purge_cloud_state(self) -> (bool, str):
|
|
751
|
-
if self.cloud_options.get("host", "").startswith("s3://"):
|
|
752
|
-
return self._purge_state_from_s3_bucket()
|
|
753
|
-
else:
|
|
754
|
-
return self._purge_state_from_recce_cloud()
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
class RecceShareStateManager:
|
|
758
|
-
error_message: str
|
|
759
|
-
hint_message: str
|
|
760
|
-
|
|
761
|
-
# It is a class to share state file on Recce Cloud.
|
|
762
|
-
|
|
763
|
-
def __init__(self, auth_options: Optional[Dict[str, str]] = None):
|
|
764
|
-
self.auth_options = auth_options or {}
|
|
765
|
-
self.error_message = None
|
|
766
|
-
self.hint_message = None
|
|
767
|
-
|
|
768
|
-
def verify(self) -> bool:
|
|
769
|
-
if self.auth_options.get("api_token") is None:
|
|
770
|
-
self.error_message = RECCE_API_TOKEN_MISSING.error_message
|
|
771
|
-
self.hint_message = RECCE_API_TOKEN_MISSING.hint_message
|
|
772
|
-
return False
|
|
773
|
-
return True
|
|
774
|
-
|
|
775
|
-
@property
|
|
776
|
-
def error_and_hint(self) -> (Union[str, None], Union[str, None]):
|
|
777
|
-
return self.error_message, self.hint_message
|
|
778
|
-
|
|
779
|
-
def share_state(self, file_name: str, state: RecceState) -> Dict:
|
|
780
|
-
import tempfile
|
|
781
|
-
|
|
782
|
-
with tempfile.NamedTemporaryFile() as tmp:
|
|
783
|
-
state.to_file(tmp.name, file_type=SupportedFileTypes.FILE)
|
|
784
|
-
response = RecceCloud(token=self.auth_options.get("api_token")).share_state(file_name, open(tmp.name, "rb"))
|
|
785
|
-
return response
|