recce-cloud-nightly 1.26.0.20251119__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of recce-cloud-nightly might be problematic. Click here for more details.
- recce_cloud/VERSION +1 -0
- recce_cloud/__init__.py +24 -0
- recce_cloud/api/__init__.py +17 -0
- recce_cloud/api/base.py +111 -0
- recce_cloud/api/client.py +150 -0
- recce_cloud/api/exceptions.py +26 -0
- recce_cloud/api/factory.py +63 -0
- recce_cloud/api/github.py +76 -0
- recce_cloud/api/gitlab.py +82 -0
- recce_cloud/artifact.py +57 -0
- recce_cloud/ci_providers/__init__.py +9 -0
- recce_cloud/ci_providers/base.py +82 -0
- recce_cloud/ci_providers/detector.py +147 -0
- recce_cloud/ci_providers/github_actions.py +136 -0
- recce_cloud/ci_providers/gitlab_ci.py +130 -0
- recce_cloud/cli.py +245 -0
- recce_cloud/upload.py +214 -0
- recce_cloud_nightly-1.26.0.20251119.dist-info/METADATA +163 -0
- recce_cloud_nightly-1.26.0.20251119.dist-info/RECORD +23 -0
- recce_cloud_nightly-1.26.0.20251119.dist-info/WHEEL +5 -0
- recce_cloud_nightly-1.26.0.20251119.dist-info/entry_points.txt +2 -0
- recce_cloud_nightly-1.26.0.20251119.dist-info/licenses/LICENSE +201 -0
- recce_cloud_nightly-1.26.0.20251119.dist-info/top_level.txt +1 -0
recce_cloud/cli.py
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
"""
|
|
3
|
+
Recce Cloud CLI - Lightweight command for managing Recce Cloud operations.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import logging
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
import click
|
|
11
|
+
from rich.console import Console
|
|
12
|
+
from rich.logging import RichHandler
|
|
13
|
+
|
|
14
|
+
from recce_cloud import __version__
|
|
15
|
+
from recce_cloud.artifact import get_adapter_type, verify_artifacts_path
|
|
16
|
+
from recce_cloud.ci_providers import CIDetector
|
|
17
|
+
from recce_cloud.upload import upload_to_existing_session, upload_with_platform_apis
|
|
18
|
+
|
|
19
|
+
# Configure logging
|
|
20
|
+
logging.basicConfig(
|
|
21
|
+
level=logging.INFO,
|
|
22
|
+
format="%(message)s",
|
|
23
|
+
handlers=[RichHandler(console=Console(stderr=True), show_time=False, show_path=False)],
|
|
24
|
+
)
|
|
25
|
+
logger = logging.getLogger(__name__)
|
|
26
|
+
|
|
27
|
+
# Suppress CI detector logs since we display formatted output in the CLI
|
|
28
|
+
logging.getLogger("recce_cloud.ci_providers.detector").setLevel(logging.WARNING)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@click.group()
|
|
32
|
+
def cloud_cli():
|
|
33
|
+
"""
|
|
34
|
+
Recce Cloud CLI - Manage Recce Cloud sessions and state files.
|
|
35
|
+
|
|
36
|
+
A lightweight tool for CI/CD environments to interact with Recce Cloud
|
|
37
|
+
without the heavy dependencies of the full recce package.
|
|
38
|
+
"""
|
|
39
|
+
pass
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@cloud_cli.command()
|
|
43
|
+
def version():
|
|
44
|
+
"""Show the version of recce-cloud."""
|
|
45
|
+
click.echo(__version__)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@cloud_cli.command()
|
|
49
|
+
@click.option(
|
|
50
|
+
"--target-path",
|
|
51
|
+
type=click.Path(exists=True),
|
|
52
|
+
default="target",
|
|
53
|
+
help="Path to dbt target directory containing manifest.json and catalog.json",
|
|
54
|
+
)
|
|
55
|
+
@click.option(
|
|
56
|
+
"--session-id",
|
|
57
|
+
envvar="RECCE_SESSION_ID",
|
|
58
|
+
help="Recce Cloud session ID to upload artifacts to (or use RECCE_SESSION_ID env var). "
|
|
59
|
+
"If not provided, session will be created automatically using platform-specific APIs (GitHub/GitLab).",
|
|
60
|
+
)
|
|
61
|
+
@click.option(
|
|
62
|
+
"--cr",
|
|
63
|
+
type=int,
|
|
64
|
+
help="Change request number (PR/MR) (overrides auto-detection)",
|
|
65
|
+
)
|
|
66
|
+
@click.option(
|
|
67
|
+
"--type",
|
|
68
|
+
"session_type",
|
|
69
|
+
type=click.Choice(["cr", "prod", "dev"]),
|
|
70
|
+
help="Session type (overrides auto-detection)",
|
|
71
|
+
)
|
|
72
|
+
@click.option(
|
|
73
|
+
"--dry-run",
|
|
74
|
+
is_flag=True,
|
|
75
|
+
help="Show what would be uploaded without actually uploading",
|
|
76
|
+
)
|
|
77
|
+
def upload(target_path, session_id, cr, session_type, dry_run):
|
|
78
|
+
"""
|
|
79
|
+
Upload dbt artifacts (manifest.json, catalog.json) to Recce Cloud.
|
|
80
|
+
|
|
81
|
+
\b
|
|
82
|
+
Authentication (auto-detected):
|
|
83
|
+
- RECCE_API_TOKEN (for --session-id workflow)
|
|
84
|
+
- GITHUB_TOKEN (GitHub Actions)
|
|
85
|
+
- CI_JOB_TOKEN (GitLab CI)
|
|
86
|
+
|
|
87
|
+
\b
|
|
88
|
+
Common Examples:
|
|
89
|
+
# Auto-create session in PR/MR
|
|
90
|
+
recce-cloud upload
|
|
91
|
+
|
|
92
|
+
# Upload production metadata from main branch
|
|
93
|
+
recce-cloud upload --type prod
|
|
94
|
+
|
|
95
|
+
# Upload to specific session
|
|
96
|
+
recce-cloud upload --session-id abc123
|
|
97
|
+
|
|
98
|
+
# Custom target path
|
|
99
|
+
recce-cloud upload --target-path custom-target
|
|
100
|
+
"""
|
|
101
|
+
console = Console()
|
|
102
|
+
|
|
103
|
+
# 1. Auto-detect CI environment information
|
|
104
|
+
console.rule("CI Environment Detection", style="blue")
|
|
105
|
+
try:
|
|
106
|
+
ci_info = CIDetector.detect()
|
|
107
|
+
ci_info = CIDetector.apply_overrides(ci_info, cr=cr, session_type=session_type)
|
|
108
|
+
|
|
109
|
+
# Display detected CI information immediately
|
|
110
|
+
if ci_info:
|
|
111
|
+
info_table = []
|
|
112
|
+
if ci_info.platform:
|
|
113
|
+
info_table.append(f"[cyan]Platform:[/cyan] {ci_info.platform}")
|
|
114
|
+
|
|
115
|
+
# Display CR number as PR or MR based on platform
|
|
116
|
+
if ci_info.cr_number is not None:
|
|
117
|
+
if ci_info.platform == "github-actions":
|
|
118
|
+
info_table.append(f"[cyan]PR Number:[/cyan] {ci_info.cr_number}")
|
|
119
|
+
elif ci_info.platform == "gitlab-ci":
|
|
120
|
+
info_table.append(f"[cyan]MR Number:[/cyan] {ci_info.cr_number}")
|
|
121
|
+
else:
|
|
122
|
+
info_table.append(f"[cyan]CR Number:[/cyan] {ci_info.cr_number}")
|
|
123
|
+
|
|
124
|
+
# Display CR URL as PR URL or MR URL based on platform
|
|
125
|
+
if ci_info.cr_url:
|
|
126
|
+
if ci_info.platform == "github-actions":
|
|
127
|
+
info_table.append(f"[cyan]PR URL:[/cyan] {ci_info.cr_url}")
|
|
128
|
+
elif ci_info.platform == "gitlab-ci":
|
|
129
|
+
info_table.append(f"[cyan]MR URL:[/cyan] {ci_info.cr_url}")
|
|
130
|
+
else:
|
|
131
|
+
info_table.append(f"[cyan]CR URL:[/cyan] {ci_info.cr_url}")
|
|
132
|
+
|
|
133
|
+
if ci_info.session_type:
|
|
134
|
+
info_table.append(f"[cyan]Session Type:[/cyan] {ci_info.session_type}")
|
|
135
|
+
if ci_info.commit_sha:
|
|
136
|
+
info_table.append(f"[cyan]Commit SHA:[/cyan] {ci_info.commit_sha[:8]}...")
|
|
137
|
+
if ci_info.base_branch:
|
|
138
|
+
info_table.append(f"[cyan]Base Branch:[/cyan] {ci_info.base_branch}")
|
|
139
|
+
if ci_info.source_branch:
|
|
140
|
+
info_table.append(f"[cyan]Source Branch:[/cyan] {ci_info.source_branch}")
|
|
141
|
+
if ci_info.repository:
|
|
142
|
+
info_table.append(f"[cyan]Repository:[/cyan] {ci_info.repository}")
|
|
143
|
+
|
|
144
|
+
for line in info_table:
|
|
145
|
+
console.print(line)
|
|
146
|
+
else:
|
|
147
|
+
console.print("[yellow]No CI environment detected[/yellow]")
|
|
148
|
+
except Exception as e:
|
|
149
|
+
console.print(f"[yellow]Warning:[/yellow] Failed to detect CI environment: {e}")
|
|
150
|
+
console.print("Continuing without CI metadata...")
|
|
151
|
+
ci_info = None
|
|
152
|
+
|
|
153
|
+
# 2. Validate artifacts exist
|
|
154
|
+
if not verify_artifacts_path(target_path):
|
|
155
|
+
console.print(f"[red]Error:[/red] Invalid target path: {target_path}")
|
|
156
|
+
console.print("Please provide a valid target path containing manifest.json and catalog.json.")
|
|
157
|
+
sys.exit(3)
|
|
158
|
+
|
|
159
|
+
manifest_path = os.path.join(target_path, "manifest.json")
|
|
160
|
+
catalog_path = os.path.join(target_path, "catalog.json")
|
|
161
|
+
|
|
162
|
+
# 3. Extract adapter type from manifest
|
|
163
|
+
try:
|
|
164
|
+
adapter_type = get_adapter_type(manifest_path)
|
|
165
|
+
except Exception as e:
|
|
166
|
+
console.print("[red]Error:[/red] Failed to parse adapter type from manifest.json")
|
|
167
|
+
console.print(f"Reason: {e}")
|
|
168
|
+
sys.exit(3)
|
|
169
|
+
|
|
170
|
+
# 4. Handle dry-run mode (before authentication or API calls)
|
|
171
|
+
if dry_run:
|
|
172
|
+
console.rule("Dry Run Summary", style="yellow")
|
|
173
|
+
console.print("[yellow]Dry run mode enabled - no actual upload will be performed[/yellow]")
|
|
174
|
+
console.print()
|
|
175
|
+
|
|
176
|
+
# Display platform information if detected
|
|
177
|
+
if ci_info and ci_info.platform:
|
|
178
|
+
console.print("[cyan]Platform Information:[/cyan]")
|
|
179
|
+
console.print(f" • Platform: {ci_info.platform}")
|
|
180
|
+
if ci_info.repository:
|
|
181
|
+
console.print(f" • Repository: {ci_info.repository}")
|
|
182
|
+
if ci_info.cr_number is not None:
|
|
183
|
+
console.print(f" • CR Number: {ci_info.cr_number}")
|
|
184
|
+
if ci_info.commit_sha:
|
|
185
|
+
console.print(f" • Commit SHA: {ci_info.commit_sha[:8]}")
|
|
186
|
+
if ci_info.source_branch:
|
|
187
|
+
console.print(f" • Source Branch: {ci_info.source_branch}")
|
|
188
|
+
if ci_info.base_branch:
|
|
189
|
+
console.print(f" • Base Branch: {ci_info.base_branch}")
|
|
190
|
+
if ci_info.session_type:
|
|
191
|
+
console.print(f" • Session Type: {ci_info.session_type}")
|
|
192
|
+
console.print()
|
|
193
|
+
|
|
194
|
+
# Display upload summary
|
|
195
|
+
console.print("[cyan]Upload Workflow:[/cyan]")
|
|
196
|
+
if session_id:
|
|
197
|
+
console.print(" • Upload to existing session")
|
|
198
|
+
console.print(f" • Session ID: {session_id}")
|
|
199
|
+
else:
|
|
200
|
+
console.print(" • Auto-create session and upload")
|
|
201
|
+
if ci_info and ci_info.platform in ["github-actions", "gitlab-ci"]:
|
|
202
|
+
console.print(" • Platform-specific APIs will be used")
|
|
203
|
+
else:
|
|
204
|
+
console.print(" • [yellow]Warning: Platform not supported for auto-session creation[/yellow]")
|
|
205
|
+
|
|
206
|
+
console.print()
|
|
207
|
+
console.print("[cyan]Files to upload:[/cyan]")
|
|
208
|
+
console.print(f" • manifest.json: {os.path.abspath(manifest_path)}")
|
|
209
|
+
console.print(f" • catalog.json: {os.path.abspath(catalog_path)}")
|
|
210
|
+
console.print(f" • Adapter type: {adapter_type}")
|
|
211
|
+
|
|
212
|
+
console.print()
|
|
213
|
+
console.print("[green]✓[/green] Dry run completed successfully")
|
|
214
|
+
sys.exit(0)
|
|
215
|
+
|
|
216
|
+
# 5. Choose upload workflow based on whether session_id is provided
|
|
217
|
+
if session_id:
|
|
218
|
+
# Generic workflow: Upload to existing session using session ID
|
|
219
|
+
# This workflow requires RECCE_API_TOKEN
|
|
220
|
+
token = os.getenv("RECCE_API_TOKEN")
|
|
221
|
+
if not token:
|
|
222
|
+
console.print("[red]Error:[/red] No RECCE_API_TOKEN provided")
|
|
223
|
+
console.print("Set RECCE_API_TOKEN environment variable for session-based upload")
|
|
224
|
+
sys.exit(2)
|
|
225
|
+
|
|
226
|
+
upload_to_existing_session(console, token, session_id, manifest_path, catalog_path, adapter_type, target_path)
|
|
227
|
+
else:
|
|
228
|
+
# Platform-specific workflow: Use platform APIs to create session and upload
|
|
229
|
+
# This workflow MUST use CI job tokens (CI_JOB_TOKEN or GITHUB_TOKEN)
|
|
230
|
+
if not ci_info or not ci_info.access_token:
|
|
231
|
+
console.print("[red]Error:[/red] Platform-specific upload requires CI environment")
|
|
232
|
+
console.print("Either run in GitHub Actions/GitLab CI or provide --session-id for generic upload")
|
|
233
|
+
sys.exit(2)
|
|
234
|
+
|
|
235
|
+
token = ci_info.access_token
|
|
236
|
+
if ci_info.platform == "github-actions":
|
|
237
|
+
console.print("[cyan]Info:[/cyan] Using GITHUB_TOKEN for platform-specific authentication")
|
|
238
|
+
elif ci_info.platform == "gitlab-ci":
|
|
239
|
+
console.print("[cyan]Info:[/cyan] Using CI_JOB_TOKEN for platform-specific authentication")
|
|
240
|
+
|
|
241
|
+
upload_with_platform_apis(console, token, ci_info, manifest_path, catalog_path, adapter_type, target_path)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
if __name__ == "__main__":
|
|
245
|
+
cloud_cli()
|
recce_cloud/upload.py
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Upload helper functions for recce-cloud CLI.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
import requests
|
|
9
|
+
|
|
10
|
+
from recce_cloud.api.client import RecceCloudClient
|
|
11
|
+
from recce_cloud.api.exceptions import RecceCloudException
|
|
12
|
+
from recce_cloud.api.factory import create_platform_client
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def upload_to_existing_session(
|
|
16
|
+
console, token: str, session_id: str, manifest_path: str, catalog_path: str, adapter_type: str, target_path: str
|
|
17
|
+
):
|
|
18
|
+
"""
|
|
19
|
+
Upload artifacts to an existing Recce Cloud session using session ID.
|
|
20
|
+
|
|
21
|
+
This is the generic workflow that requires a pre-existing session ID.
|
|
22
|
+
"""
|
|
23
|
+
try:
|
|
24
|
+
client = RecceCloudClient(token)
|
|
25
|
+
except Exception as e:
|
|
26
|
+
console.print("[red]Error:[/red] Failed to initialize API client")
|
|
27
|
+
console.print(f"Reason: {e}")
|
|
28
|
+
sys.exit(2)
|
|
29
|
+
|
|
30
|
+
# Get session info (org_id, project_id)
|
|
31
|
+
console.print(f'Uploading artifacts for session ID "{session_id}"')
|
|
32
|
+
try:
|
|
33
|
+
session = client.get_session(session_id)
|
|
34
|
+
if session.get("status") == "error":
|
|
35
|
+
console.print(f"[red]Error:[/red] {session.get('message')}")
|
|
36
|
+
sys.exit(2)
|
|
37
|
+
|
|
38
|
+
org_id = session.get("org_id")
|
|
39
|
+
if org_id is None:
|
|
40
|
+
console.print(f"[red]Error:[/red] Session ID {session_id} does not belong to any organization.")
|
|
41
|
+
sys.exit(2)
|
|
42
|
+
|
|
43
|
+
project_id = session.get("project_id")
|
|
44
|
+
if project_id is None:
|
|
45
|
+
console.print(f"[red]Error:[/red] Session ID {session_id} does not belong to any project.")
|
|
46
|
+
sys.exit(2)
|
|
47
|
+
|
|
48
|
+
except RecceCloudException as e:
|
|
49
|
+
console.print("[red]Error:[/red] Failed to get session info")
|
|
50
|
+
console.print(f"Reason: {e.reason}")
|
|
51
|
+
sys.exit(2)
|
|
52
|
+
except Exception as e:
|
|
53
|
+
console.print("[red]Error:[/red] Failed to get session info")
|
|
54
|
+
console.print(f"Reason: {e}")
|
|
55
|
+
sys.exit(2)
|
|
56
|
+
|
|
57
|
+
# Get presigned URLs
|
|
58
|
+
try:
|
|
59
|
+
presigned_urls = client.get_upload_urls_by_session_id(org_id, project_id, session_id)
|
|
60
|
+
except RecceCloudException as e:
|
|
61
|
+
console.print("[red]Error:[/red] Failed to get upload URLs")
|
|
62
|
+
console.print(f"Reason: {e.reason}")
|
|
63
|
+
sys.exit(4)
|
|
64
|
+
except Exception as e:
|
|
65
|
+
console.print("[red]Error:[/red] Failed to get upload URLs")
|
|
66
|
+
console.print(f"Reason: {e}")
|
|
67
|
+
sys.exit(4)
|
|
68
|
+
|
|
69
|
+
# Upload manifest.json
|
|
70
|
+
console.print(f'Uploading manifest from path "{manifest_path}"')
|
|
71
|
+
try:
|
|
72
|
+
with open(manifest_path, "rb") as f:
|
|
73
|
+
response = requests.put(presigned_urls["manifest_url"], data=f.read())
|
|
74
|
+
if response.status_code not in [200, 204]:
|
|
75
|
+
raise Exception(f"Upload failed with status {response.status_code}: {response.text}")
|
|
76
|
+
except Exception as e:
|
|
77
|
+
console.print("[red]Error:[/red] Failed to upload manifest.json")
|
|
78
|
+
console.print(f"Reason: {e}")
|
|
79
|
+
sys.exit(4)
|
|
80
|
+
|
|
81
|
+
# Upload catalog.json
|
|
82
|
+
console.print(f'Uploading catalog from path "{catalog_path}"')
|
|
83
|
+
try:
|
|
84
|
+
with open(catalog_path, "rb") as f:
|
|
85
|
+
response = requests.put(presigned_urls["catalog_url"], data=f.read())
|
|
86
|
+
if response.status_code not in [200, 204]:
|
|
87
|
+
raise Exception(f"Upload failed with status {response.status_code}: {response.text}")
|
|
88
|
+
except Exception as e:
|
|
89
|
+
console.print("[red]Error:[/red] Failed to upload catalog.json")
|
|
90
|
+
console.print(f"Reason: {e}")
|
|
91
|
+
sys.exit(4)
|
|
92
|
+
|
|
93
|
+
# Update session metadata
|
|
94
|
+
try:
|
|
95
|
+
client.update_session(org_id, project_id, session_id, adapter_type)
|
|
96
|
+
except RecceCloudException as e:
|
|
97
|
+
console.print("[red]Error:[/red] Failed to update session metadata")
|
|
98
|
+
console.print(f"Reason: {e.reason}")
|
|
99
|
+
sys.exit(4)
|
|
100
|
+
except Exception as e:
|
|
101
|
+
console.print("[red]Error:[/red] Failed to update session metadata")
|
|
102
|
+
console.print(f"Reason: {e}")
|
|
103
|
+
sys.exit(4)
|
|
104
|
+
|
|
105
|
+
# Success!
|
|
106
|
+
console.rule("Uploaded Successfully", style="green")
|
|
107
|
+
console.print(
|
|
108
|
+
f'Uploaded dbt artifacts to Recce Cloud for session ID "{session_id}" from "{os.path.abspath(target_path)}"'
|
|
109
|
+
)
|
|
110
|
+
sys.exit(0)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def upload_with_platform_apis(
|
|
114
|
+
console, token: str, ci_info, manifest_path: str, catalog_path: str, adapter_type: str, target_path: str
|
|
115
|
+
):
|
|
116
|
+
"""
|
|
117
|
+
Upload artifacts using platform-specific APIs (GitHub Actions or GitLab CI).
|
|
118
|
+
|
|
119
|
+
This workflow uses touch-recce-session to create a session automatically.
|
|
120
|
+
"""
|
|
121
|
+
# Validate platform support
|
|
122
|
+
if ci_info.platform not in ["github-actions", "gitlab-ci"]:
|
|
123
|
+
console.print("[red]Error:[/red] Platform-specific upload requires GitHub Actions or GitLab CI environment")
|
|
124
|
+
console.print(f"Detected platform: {ci_info.platform or 'unknown'}")
|
|
125
|
+
console.print(
|
|
126
|
+
"Either run this command in a supported CI environment or provide --session-id for generic upload"
|
|
127
|
+
)
|
|
128
|
+
sys.exit(1)
|
|
129
|
+
|
|
130
|
+
# Create platform-specific client
|
|
131
|
+
try:
|
|
132
|
+
client = create_platform_client(token, ci_info)
|
|
133
|
+
except ValueError as e:
|
|
134
|
+
console.print("[red]Error:[/red] Failed to create platform client")
|
|
135
|
+
console.print(f"Reason: {e}")
|
|
136
|
+
sys.exit(2)
|
|
137
|
+
|
|
138
|
+
# Touch session to create or get session ID
|
|
139
|
+
console.rule("Creating/touching session", style="blue")
|
|
140
|
+
try:
|
|
141
|
+
session_response = client.touch_recce_session(
|
|
142
|
+
branch=ci_info.source_branch or ci_info.base_branch or "main",
|
|
143
|
+
adapter_type=adapter_type,
|
|
144
|
+
cr_number=ci_info.cr_number,
|
|
145
|
+
commit_sha=ci_info.commit_sha,
|
|
146
|
+
session_type=ci_info.session_type,
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
session_id = session_response.get("session_id")
|
|
150
|
+
manifest_upload_url = session_response.get("manifest_upload_url")
|
|
151
|
+
catalog_upload_url = session_response.get("catalog_upload_url")
|
|
152
|
+
|
|
153
|
+
if not session_id or not manifest_upload_url or not catalog_upload_url:
|
|
154
|
+
console.print("[red]Error:[/red] Incomplete response from touch-recce-session API")
|
|
155
|
+
console.print(f"Response: {session_response}")
|
|
156
|
+
sys.exit(4)
|
|
157
|
+
|
|
158
|
+
console.print(f"[green]Session ID:[/green] {session_id}")
|
|
159
|
+
|
|
160
|
+
except RecceCloudException as e:
|
|
161
|
+
console.print("[red]Error:[/red] Failed to create/touch session")
|
|
162
|
+
console.print(f"Reason: {e.reason}")
|
|
163
|
+
sys.exit(4)
|
|
164
|
+
except Exception as e:
|
|
165
|
+
console.print("[red]Error:[/red] Failed to create/touch session")
|
|
166
|
+
console.print(f"Reason: {e}")
|
|
167
|
+
sys.exit(4)
|
|
168
|
+
|
|
169
|
+
# Upload manifest.json
|
|
170
|
+
console.print(f'Uploading manifest from path "{manifest_path}"')
|
|
171
|
+
try:
|
|
172
|
+
with open(manifest_path, "rb") as f:
|
|
173
|
+
response = requests.put(manifest_upload_url, data=f.read())
|
|
174
|
+
if response.status_code not in [200, 204]:
|
|
175
|
+
raise Exception(f"Upload failed with status {response.status_code}: {response.text}")
|
|
176
|
+
except Exception as e:
|
|
177
|
+
console.print("[red]Error:[/red] Failed to upload manifest.json")
|
|
178
|
+
console.print(f"Reason: {e}")
|
|
179
|
+
sys.exit(4)
|
|
180
|
+
|
|
181
|
+
# Upload catalog.json
|
|
182
|
+
console.print(f'Uploading catalog from path "{catalog_path}"')
|
|
183
|
+
try:
|
|
184
|
+
with open(catalog_path, "rb") as f:
|
|
185
|
+
response = requests.put(catalog_upload_url, data=f.read())
|
|
186
|
+
if response.status_code not in [200, 204]:
|
|
187
|
+
raise Exception(f"Upload failed with status {response.status_code}: {response.text}")
|
|
188
|
+
except Exception as e:
|
|
189
|
+
console.print("[red]Error:[/red] Failed to upload catalog.json")
|
|
190
|
+
console.print(f"Reason: {e}")
|
|
191
|
+
sys.exit(4)
|
|
192
|
+
|
|
193
|
+
# Notify upload completion
|
|
194
|
+
console.print("Notifying upload completion...")
|
|
195
|
+
try:
|
|
196
|
+
client.upload_completed(session_id=session_id, commit_sha=ci_info.commit_sha)
|
|
197
|
+
except RecceCloudException as e:
|
|
198
|
+
console.print("[yellow]Warning:[/yellow] Failed to notify upload completion")
|
|
199
|
+
console.print(f"Reason: {e.reason}")
|
|
200
|
+
# Non-fatal, continue
|
|
201
|
+
except Exception as e:
|
|
202
|
+
console.print("[yellow]Warning:[/yellow] Failed to notify upload completion")
|
|
203
|
+
console.print(f"Reason: {e}")
|
|
204
|
+
# Non-fatal, continue
|
|
205
|
+
|
|
206
|
+
# Success!
|
|
207
|
+
console.rule("Uploaded Successfully", style="green")
|
|
208
|
+
console.print(f'Uploaded dbt artifacts to Recce Cloud for session ID "{session_id}"')
|
|
209
|
+
console.print(f'Artifacts from: "{os.path.abspath(target_path)}"')
|
|
210
|
+
|
|
211
|
+
if ci_info.cr_url:
|
|
212
|
+
console.print(f"Change request: {ci_info.cr_url}")
|
|
213
|
+
|
|
214
|
+
sys.exit(0)
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: recce-cloud-nightly
|
|
3
|
+
Version: 1.26.0.20251119
|
|
4
|
+
Summary: Lightweight CLI for Recce Cloud operations
|
|
5
|
+
Home-page: https://github.com/InfuseAI/recce
|
|
6
|
+
Author: InfuseAI Dev Team
|
|
7
|
+
Author-email: dev@infuseai.io
|
|
8
|
+
Project-URL: Bug Tracker, https://github.com/InfuseAI/recce/issues
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Development Status :: 4 - Beta
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: click>=7.1
|
|
21
|
+
Requires-Dist: requests>=2.28.1
|
|
22
|
+
Requires-Dist: rich>=12.0.0
|
|
23
|
+
Dynamic: author
|
|
24
|
+
Dynamic: author-email
|
|
25
|
+
Dynamic: classifier
|
|
26
|
+
Dynamic: description
|
|
27
|
+
Dynamic: description-content-type
|
|
28
|
+
Dynamic: home-page
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
Dynamic: project-url
|
|
31
|
+
Dynamic: requires-dist
|
|
32
|
+
Dynamic: requires-python
|
|
33
|
+
Dynamic: summary
|
|
34
|
+
|
|
35
|
+
<p align="center">
|
|
36
|
+
<a href="https://reccehq.com">
|
|
37
|
+
<picture>
|
|
38
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://reccehq.com/assets/images/recce-logo-stacked.avif">
|
|
39
|
+
<source media="(prefers-color-scheme: light)" srcset="https://reccehq.com/assets/images/recce-logo-stacked.avif">
|
|
40
|
+
<img alt="Recce: DataRecce.io" src="https://reccehq.com/assets/images/recce-logo-stacked.avif" width="200" style="display: block; margin: 0 auto 20px;">
|
|
41
|
+
</picture>
|
|
42
|
+
</a>
|
|
43
|
+
</p>
|
|
44
|
+
|
|
45
|
+
<h3 align="center">Helping data teams preview, validate, and ship data changes with confidence.</h3>
|
|
46
|
+
|
|
47
|
+
<p align="center">
|
|
48
|
+
<a href="https://pypi.org/project/recce/"><img src="https://img.shields.io/badge/pip_install-recce-006DAD?style=flat-square" alt="install"></a>
|
|
49
|
+
<a href="https://pypi.org/project/recce/"><img src="https://img.shields.io/pypi/v/recce?style=flat-square" alt="pipy"></a>
|
|
50
|
+
<a href="https://pypi.org/project/recce/"><img src="https://img.shields.io/pypi/pyversions/recce?style=flat-square" alt="Python"></a>
|
|
51
|
+
<a href="https://pypi.org/project/recce/#files"><img src="https://img.shields.io/pypi/dw/recce?style=flat-square" alt="downloads"></a>
|
|
52
|
+
<a href="https://github.com/DataRecce/recce/blob/main/LICENSE"><img src="https://img.shields.io/github/license/DataRecce/recce?style=flat-square" alt="license"></a>
|
|
53
|
+
<a href="https://getdbt.slack.com/archives/C05C28V7CPP"><img src="https://img.shields.io/badge/Slack-4A154B?style=flat-square&logo=slack&logoColor=white" alt="Slack"></a>
|
|
54
|
+
<a href="https://discord.com/invite/5zb2aK9KBV"><img src="https://img.shields.io/discord/664381609771925514?color=%237289DA&label=chat&logo=discord&logoColor=white&style=flat-square" alt="InfuseAI Discord Invite"></a>
|
|
55
|
+
</p>
|
|
56
|
+
|
|
57
|
+
<p align="center">
|
|
58
|
+
<a href="https://cal.com/team/recce/chat?utm_source=banner&utm_campaign=oss">
|
|
59
|
+
<img alt="Book us with Cal.com" src="https://cal.com/book-with-cal-light.svg" />
|
|
60
|
+
</a>
|
|
61
|
+
</p>
|
|
62
|
+
|
|
63
|
+
## Trust, Verify, Ship
|
|
64
|
+
Cut dbt review time by 90% and ship accurate data fast
|
|
65
|
+
|
|
66
|
+
Recce gives data teams a faster, more reliable way to understand, review, and ship changes without all the guesswork or manual overhead.
|
|
67
|
+
|
|
68
|
+
## Quick Start
|
|
69
|
+
|
|
70
|
+
### Installation
|
|
71
|
+
|
|
72
|
+
Recce offers two packages to fit different use cases:
|
|
73
|
+
|
|
74
|
+
**For full local development and Recce Cloud features:**
|
|
75
|
+
```bash
|
|
76
|
+
pip install -U recce
|
|
77
|
+
recce server
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
**For CI/CD artifact uploads only (lightweight):**
|
|
81
|
+
```bash
|
|
82
|
+
pip install -U recce-cloud
|
|
83
|
+
recce-cloud upload
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
The `recce-cloud` package is a lightweight CLI tool designed specifically for CI/CD environments where you only need to upload dbt artifacts to Recce Cloud. It has minimal dependencies and installs faster than the full `recce` package.
|
|
87
|
+
|
|
88
|
+
### Getting Started
|
|
89
|
+
|
|
90
|
+
You can launch Recce in any dbt project in just two commands:
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
# cd into your dbt project
|
|
94
|
+
pip install -U recce
|
|
95
|
+
recce server
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
This starts Recce locally, where you can explore lineage and run queries. To unlock the full set of diffing tools, such as data comparisons and impact checks, you'll need to prepare two environments to compare against. You can follow our [Getting Started](https://docs.reccehq.com/get-started/) and [5-minute Jaffle Shop tutorial](https://docs.reccehq.com/get-started-jaffle-shop/) to try it out step-by-step.
|
|
99
|
+
|
|
100
|
+
## What You Get
|
|
101
|
+
|
|
102
|
+
Recce gives you a clear, fast way to understand what your data changes are doing and why they matter. It helps you catch problems early, verify metrics, and share your findings with others, all as part of your normal workflow.
|
|
103
|
+
|
|
104
|
+
<a href="https://pr46.demo.reccehq.com/"><img width="1347" alt="readme" src="https://github.com/user-attachments/assets/773e4c3a-0a15-49e0-8d1b-38a55af17cb0" /></a>
|
|
105
|
+
|
|
106
|
+
<a href="https://reccehq.com"><img src="https://docs.reccehq.com/assets/images/1-whats-recce/diff-readme2.png" style="width: 100%; max-width: 600px; display: block; margin: 0 auto 20px;" alt="Model and column level diff"/></a>
|
|
107
|
+
|
|
108
|
+
<a href="https://reccehq.com"><img src="https://docs.reccehq.com/assets/images/1-whats-recce/checklist-readme3.png" style="width: 100%; max-width: 600px; display: block; margin: 0 auto 20px;" alt="Checklist for collaboration"/></a>
|
|
109
|
+
|
|
110
|
+
### Using Recce for Impact Assessment in dbt PR Review
|
|
111
|
+
|
|
112
|
+
- Select nodes in the lineage to perform Checks (diffs) as part of your impact assessment during development or PR
|
|
113
|
+
review.
|
|
114
|
+
- Add Checks to your Checklist to note observed impacts.
|
|
115
|
+
- Share your Checklist with the PR reviewer.
|
|
116
|
+
- (`Recce Cloud`) Automatically sync Check status between Recce Instances
|
|
117
|
+
- (`Recce Cloud`) Block PR merging until all Recce Checks have been approved
|
|
118
|
+
|
|
119
|
+
Read more about using Recce on our [blog](https://blog.reccehq.com).
|
|
120
|
+
|
|
121
|
+
### What’s Included
|
|
122
|
+
|
|
123
|
+
- [Lineage and impact mapping](https://docs.reccehq.com/features/lineage/): Quickly see which models and columns are affected by a change. Navigate lineage down to the column level, and spot breaking changes with clear visual cues.
|
|
124
|
+
- Metric and data comparisons: Use [Profile, Value, Top-K, and Histogram Diffs](https://docs.reccehq.com/features/lineage/#node-details) to compare results before and after changes. Validate things like row counts, category distributions, and numeric ranges without writing extra SQL.
|
|
125
|
+
- [Query diff](https://docs.reccehq.com/features/query/): Write and compare any two queries side by side. This is helpful when validating fixes or reviewing changes with teammates.
|
|
126
|
+
- [Checklist for reviews and approvals](https://docs.reccehq.com/features/checklist/): Turn your validation steps into a checklist. Add notes, rerun checks, and share the results with reviewers or stakeholders. In Recce Cloud, checklists can sync automatically and even block PRs until checks are approved.
|
|
127
|
+
- Secure by design: Recce is [SOC 2 compliant](https://trust.reccehq.com/) to meet enterprise security standards. It runs locally or in your private environment, and your data stays in your warehouse.
|
|
128
|
+
|
|
129
|
+
👉 Want to dive deeper? Check out the [full documentation](https://docs.reccehq.com/).
|
|
130
|
+
|
|
131
|
+
## Recce Cloud
|
|
132
|
+
|
|
133
|
+
Ready to collaborate and move faster as a team? Recce Cloud adds real-time collaboration, automatic checklist sync, and PR gating, so nothing gets merged without a full review.
|
|
134
|
+
|
|
135
|
+
- Share checklists across environments
|
|
136
|
+
- Invite stakeholders to review data changes
|
|
137
|
+
- Block merges until all Checks are approved
|
|
138
|
+
- Launch demo links from your CI with full context
|
|
139
|
+
|
|
140
|
+
Recce Cloud is a hosted version of Recce that standardizes your workflow, keeps teams aligned, and reduces errors—so you can ship data changes with confidence.
|
|
141
|
+
👉 [View Pricing and Plans](https://reccehq.com/pricing)
|
|
142
|
+
|
|
143
|
+
## Community & Support
|
|
144
|
+
|
|
145
|
+
Here's where you can get in touch with the Recce team and find support, add a subscribe to our newsletter option as well:
|
|
146
|
+
|
|
147
|
+
- [chat on our website](https://reccehq.com/). We welcome you to start a chat or drop us a note. Our team monitors the chat box and will follow up soon.
|
|
148
|
+
- [Our discord](https://discord.com/invite/VpwXRC34jz)
|
|
149
|
+
- [dbt Slack](https://www.getdbt.com/community/join-the-community) in the [#tools-recce](https://getdbt.slack.com/archives/C05C28V7CPP) channel
|
|
150
|
+
- Email us [help@reccehq.com](mailto:help@reccehq.com)
|
|
151
|
+
|
|
152
|
+
If you believe you have found a bug on our open source, or there is some missing functionality in Recce, please open a [GitHub Issue](https://github.com/DataRecce/recce/issues).
|
|
153
|
+
|
|
154
|
+
## Recce on the web
|
|
155
|
+
|
|
156
|
+
You can follow along with news about Recce and blogs from our team in the following places:
|
|
157
|
+
|
|
158
|
+
- [RecceHQ.com](https://reccehq.com/)
|
|
159
|
+
- [LinkedIn](https://www.linkedin.com/company/datarecce)
|
|
160
|
+
- [Blog](https://blog.reccehq.com/)
|
|
161
|
+
- [@datarecce](https://x.com/DataRecce) on Twitter/X
|
|
162
|
+
- [@DataRecce@mastodon.social](https://mastodon.social/@DataRecce) on Mastodon
|
|
163
|
+
- [@datarecce.bsky.social](https://bsky.app/profile/datarecce.bsky.social) on BlueSky
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
recce_cloud/VERSION,sha256=QLJSrCl7IX6pmUUQBFunGagcpfnfIZ40A1S6zkMT6lQ,16
|
|
2
|
+
recce_cloud/__init__.py,sha256=SYFgYA-KJmqVzZRLq65rtvt9JZfTDFJTf9qye7q_SJE,721
|
|
3
|
+
recce_cloud/artifact.py,sha256=C6q1i8d0JdTIVLcJY1ZEmJJ0uBz-0g3zVZD8ss0K1iI,1458
|
|
4
|
+
recce_cloud/cli.py,sha256=7vpRbH2FusbFBxferRJAonQROMmekv5Wf48CxfOF3_w,9617
|
|
5
|
+
recce_cloud/upload.py,sha256=GLFG3q9KIcYedPcLZM5x1dWSEF9OdZsX1z-SUET1M0E,8347
|
|
6
|
+
recce_cloud/api/__init__.py,sha256=HrGOh7wp_U3G3gOXyGjJo1WoDXrMvhwjw8VGO2aKNTw,562
|
|
7
|
+
recce_cloud/api/base.py,sha256=ceD9cTgqctEu8aPqSCX9NNlC10-w4ocwgIcqjLnOpUM,3678
|
|
8
|
+
recce_cloud/api/client.py,sha256=LByXcj2yr9jqpYQIt3zXM6So1dq46j_0FitssBSelHA,5422
|
|
9
|
+
recce_cloud/api/exceptions.py,sha256=BSfuh7dGZsx1qDkr2ZhRIAoPo_eXkAIhyho5PrUX2yo,634
|
|
10
|
+
recce_cloud/api/factory.py,sha256=0hdo2jTdTlvd8cU7YTZFJJ2q3ybSssA2JKLr0bO1QmQ,2081
|
|
11
|
+
recce_cloud/api/github.py,sha256=y4Og6uAN_sBEi7UJqnKXe3maf8KpupOisdr-eGBtf6g,2494
|
|
12
|
+
recce_cloud/api/gitlab.py,sha256=DHBhnx9ICo1iP17jvKGVN8flKPhfzVMjUAiJLyxtjHE,2779
|
|
13
|
+
recce_cloud/ci_providers/__init__.py,sha256=cB2bydn4QOMurJVfWhIN9uTD9gIyiBcYOloN-0TvMW4,328
|
|
14
|
+
recce_cloud/ci_providers/base.py,sha256=v8P_eZdMwkgo3vCIHoyu1vBfetJICUNKz3z8ezOgwl0,2577
|
|
15
|
+
recce_cloud/ci_providers/detector.py,sha256=GvZuPo7Gq8eKCxsgYiYjQ3CYu3icPbi0moAYflnLDOk,5056
|
|
16
|
+
recce_cloud/ci_providers/github_actions.py,sha256=Vs4_YuqBsccjUeiT1a-wQBz-7sq5TnoGFwPJXniua_w,4311
|
|
17
|
+
recce_cloud/ci_providers/gitlab_ci.py,sha256=nQa2gZClXzLQ2soSqIhY5vul8RNQF-j_P-0bheD5U4s,4175
|
|
18
|
+
recce_cloud_nightly-1.26.0.20251119.dist-info/licenses/LICENSE,sha256=CQjjMy9aYPhfe8xG_bcpIfKtNkdxLZ5IOb8oPygtUhY,11343
|
|
19
|
+
recce_cloud_nightly-1.26.0.20251119.dist-info/METADATA,sha256=A3Yt8-7hXvLPEIBFh9FWLzfErY2c-tPgRvhhOeH_HaQ,8975
|
|
20
|
+
recce_cloud_nightly-1.26.0.20251119.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
|
|
21
|
+
recce_cloud_nightly-1.26.0.20251119.dist-info/entry_points.txt,sha256=MVzgsEDWaQ4fWf33zlyPF83wg5Rgg5Axq2hlLEf0Yxg,58
|
|
22
|
+
recce_cloud_nightly-1.26.0.20251119.dist-info/top_level.txt,sha256=X6xEIhYi_nQ4mqRBEYBcu-iySBoOPDGTFCTPyFTDYiA,12
|
|
23
|
+
recce_cloud_nightly-1.26.0.20251119.dist-info/RECORD,,
|