recce-nightly 1.3.0.20250507__py3-none-any.whl → 1.4.0.20250515__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of recce-nightly might be problematic. Click here for more details.
- recce/VERSION +1 -1
- recce/__init__.py +22 -22
- recce/adapter/base.py +11 -14
- recce/adapter/dbt_adapter/__init__.py +355 -316
- recce/adapter/dbt_adapter/dbt_version.py +3 -0
- recce/adapter/sqlmesh_adapter.py +24 -35
- recce/apis/check_api.py +39 -28
- recce/apis/check_func.py +33 -27
- recce/apis/run_api.py +25 -19
- recce/apis/run_func.py +29 -23
- recce/artifact.py +44 -49
- recce/cli.py +484 -285
- recce/config.py +42 -33
- recce/core.py +52 -44
- recce/data/404.html +1 -1
- recce/data/_next/static/chunks/{368-7587b306577df275.js → 778-aef312bffb4c0312.js} +15 -15
- recce/data/_next/static/chunks/8d700b6a.ed11a130057c7a47.js +1 -0
- recce/data/_next/static/chunks/app/layout-c713a2829d3279e4.js +1 -0
- recce/data/_next/static/chunks/app/page-7086764277331fcb.js +1 -0
- recce/data/_next/static/chunks/{cd9f8d63-cf0d5a7b0f7a92e8.js → cd9f8d63-e020f408095ed77c.js} +3 -3
- recce/data/_next/static/chunks/webpack-b787cb1a4f2293de.js +1 -0
- recce/data/_next/static/css/88b8abc134cfd59a.css +3 -0
- recce/data/index.html +2 -2
- recce/data/index.txt +2 -2
- recce/diff.py +6 -12
- recce/event/__init__.py +74 -72
- recce/event/collector.py +27 -20
- recce/event/track.py +39 -27
- recce/exceptions.py +1 -1
- recce/git.py +7 -7
- recce/github.py +57 -53
- recce/models/__init__.py +1 -1
- recce/models/check.py +6 -7
- recce/models/run.py +1 -0
- recce/models/types.py +27 -27
- recce/pull_request.py +26 -24
- recce/run.py +148 -111
- recce/server.py +103 -89
- recce/state.py +209 -177
- recce/summary.py +168 -143
- recce/tasks/__init__.py +3 -3
- recce/tasks/core.py +11 -13
- recce/tasks/dataframe.py +19 -17
- recce/tasks/histogram.py +69 -34
- recce/tasks/lineage.py +2 -2
- recce/tasks/profile.py +147 -86
- recce/tasks/query.py +139 -87
- recce/tasks/rowcount.py +33 -30
- recce/tasks/schema.py +14 -14
- recce/tasks/top_k.py +35 -35
- recce/tasks/valuediff.py +216 -152
- recce/util/breaking.py +77 -84
- recce/util/cll.py +55 -51
- recce/util/io.py +19 -17
- recce/util/logger.py +1 -1
- recce/util/recce_cloud.py +70 -72
- recce/util/singleton.py +4 -4
- recce/yaml/__init__.py +7 -10
- {recce_nightly-1.3.0.20250507.dist-info → recce_nightly-1.4.0.20250515.dist-info}/METADATA +5 -2
- recce_nightly-1.4.0.20250515.dist-info/RECORD +143 -0
- {recce_nightly-1.3.0.20250507.dist-info → recce_nightly-1.4.0.20250515.dist-info}/WHEEL +1 -1
- tests/adapter/dbt_adapter/conftest.py +1 -0
- tests/adapter/dbt_adapter/dbt_test_helper.py +28 -18
- tests/adapter/dbt_adapter/test_dbt_adapter.py +0 -15
- tests/adapter/dbt_adapter/test_dbt_cll.py +39 -32
- tests/adapter/dbt_adapter/test_selector.py +22 -21
- tests/tasks/test_histogram.py +58 -66
- tests/tasks/test_lineage.py +36 -23
- tests/tasks/test_preset_checks.py +45 -31
- tests/tasks/test_profile.py +340 -15
- tests/tasks/test_query.py +40 -40
- tests/tasks/test_row_count.py +65 -46
- tests/tasks/test_schema.py +65 -42
- tests/tasks/test_top_k.py +22 -18
- tests/tasks/test_valuediff.py +43 -32
- tests/test_cli.py +71 -58
- tests/test_config.py +7 -9
- tests/test_core.py +5 -3
- tests/test_dbt.py +7 -7
- tests/test_pull_request.py +1 -1
- tests/test_server.py +19 -13
- tests/test_state.py +40 -27
- tests/test_summary.py +18 -14
- recce/data/_next/static/chunks/8d700b6a-f0b1f6b9e0d97ce2.js +0 -1
- recce/data/_next/static/chunks/app/layout-9102e22cb73f74d6.js +0 -1
- recce/data/_next/static/chunks/app/page-92f13c8fad9fae3d.js +0 -1
- recce/data/_next/static/chunks/webpack-567d72f0bc0820d5.js +0 -1
- recce_nightly-1.3.0.20250507.dist-info/RECORD +0 -142
- /recce/data/_next/static/{K5iKlCYhdcpq8Ea6ck9J_ → q0Xsc9Sd6PDuo1lshYpLu}/_buildManifest.js +0 -0
- /recce/data/_next/static/{K5iKlCYhdcpq8Ea6ck9J_ → q0Xsc9Sd6PDuo1lshYpLu}/_ssgManifest.js +0 -0
- {recce_nightly-1.3.0.20250507.dist-info → recce_nightly-1.4.0.20250515.dist-info}/entry_points.txt +0 -0
- {recce_nightly-1.3.0.20250507.dist-info → recce_nightly-1.4.0.20250515.dist-info}/licenses/LICENSE +0 -0
- {recce_nightly-1.3.0.20250507.dist-info → recce_nightly-1.4.0.20250515.dist-info}/top_level.txt +0 -0
recce/data/index.txt
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
2:I[86822,[],"ClientPageRoot"]
|
|
2
|
-
3:I[
|
|
2
|
+
3:I[1996,["266","static/chunks/e24bf851-0f8cbc99656833e7.js","517","static/chunks/c1ceaa8b-a1e442154d23515e.js","376","static/chunks/3a92ee20-3b5d922d4157af5e.js","455","static/chunks/6ef81909-694dc38134099299.js","678","static/chunks/3998a672-eaad84bdd88cc73e.js","509","static/chunks/9746af58-d74bef4d03eea6ab.js","648","static/chunks/ce84277d-f42c2c58049cea2d.js","989","static/chunks/47d8844f-79a1b53c66a7d7ec.js","147","static/chunks/a30376cd-7d806e1602f2dc3a.js","995","static/chunks/fee69bc6-f17d36c080742e74.js","739","static/chunks/7a8a3e83-d7fa409d97b38b2b.js","283","static/chunks/450c323b-1bb5db526e54435a.js","303","static/chunks/36e1c10d-bb0210cbd6573a8d.js","22","static/chunks/29e3cc0d-8c150e37dff9631b.js","25","static/chunks/b63b1b3f-7395c74e11a14e95.js","355","static/chunks/7f27ae6c-413f6b869a04183a.js","495","static/chunks/6dc81886-c94b9b91bc2c3caf.js","599","static/chunks/c132bf7d-8102037f9ccf372a.js","971","static/chunks/cd9f8d63-e020f408095ed77c.js","778","static/chunks/778-aef312bffb4c0312.js","931","static/chunks/app/page-7086764277331fcb.js"],"default",1]
|
|
3
3
|
4:I[79137,[],""]
|
|
4
4
|
5:I[63846,[],""]
|
|
5
|
-
0:["
|
|
5
|
+
0:["q0Xsc9Sd6PDuo1lshYpLu",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/c9ecb46a4b21c126.css","precedence":"next","crossOrigin":"$undefined"}]]],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/88b8abc134cfd59a.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"suppressHydrationWarning":true,"children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
|
6
6
|
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"recce"}],["$","meta","3",{"name":"description","content":"Recce: Data validation toolkit for comprehensive PR review"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"32x32"}]]
|
|
7
7
|
1:null
|
recce/diff.py
CHANGED
|
@@ -5,28 +5,22 @@ import pandas as pd
|
|
|
5
5
|
|
|
6
6
|
def diff_text(before: str, after: str):
|
|
7
7
|
if before is None and after is None:
|
|
8
|
-
print(
|
|
8
|
+
print("not found in both states")
|
|
9
9
|
return
|
|
10
10
|
elif before == after:
|
|
11
|
-
print(
|
|
11
|
+
print("no changes")
|
|
12
12
|
return
|
|
13
13
|
|
|
14
|
-
diff_output = difflib.unified_diff(
|
|
15
|
-
before.splitlines(),
|
|
16
|
-
after.splitlines(),
|
|
17
|
-
"base",
|
|
18
|
-
"current",
|
|
19
|
-
lineterm=""
|
|
20
|
-
)
|
|
14
|
+
diff_output = difflib.unified_diff(before.splitlines(), after.splitlines(), "base", "current", lineterm="")
|
|
21
15
|
for line in diff_output:
|
|
22
16
|
print(line)
|
|
23
17
|
|
|
24
18
|
|
|
25
19
|
def diff_dataframe(before: pd.DataFrame, after: pd.DataFrame):
|
|
26
20
|
if before is None and after is None:
|
|
27
|
-
print(
|
|
21
|
+
print("not found in both states")
|
|
28
22
|
return
|
|
29
23
|
|
|
30
24
|
before_aligned, after_aligned = before.align(after)
|
|
31
|
-
diff = before_aligned.compare(after_aligned, result_names=(
|
|
32
|
-
print(diff.to_string(na_rep=
|
|
25
|
+
diff = before_aligned.compare(after_aligned, result_names=("base", "current"))
|
|
26
|
+
print(diff.to_string(na_rep="-"))
|
recce/event/__init__.py
CHANGED
|
@@ -10,17 +10,21 @@ from typing import Dict
|
|
|
10
10
|
|
|
11
11
|
import sentry_sdk
|
|
12
12
|
|
|
13
|
-
from recce import
|
|
13
|
+
from recce import get_runner, get_version, is_ci_env
|
|
14
14
|
from recce import yaml as pyml
|
|
15
15
|
from recce.event.collector import Collector
|
|
16
16
|
from recce.git import current_branch, hosting_repo
|
|
17
|
-
from recce.github import
|
|
18
|
-
get_github_codespace_available_at
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
17
|
+
from recce.github import (
|
|
18
|
+
get_github_codespace_available_at,
|
|
19
|
+
get_github_codespace_info,
|
|
20
|
+
get_github_codespace_name,
|
|
21
|
+
is_github_codespace,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
USER_HOME = os.path.expanduser("~")
|
|
25
|
+
RECCE_USER_HOME = os.path.join(USER_HOME, ".recce")
|
|
26
|
+
RECCE_USER_PROFILE = os.path.join(RECCE_USER_HOME, "profile.yml")
|
|
27
|
+
RECCE_USER_EVENT_PATH = os.path.join(RECCE_USER_HOME, ".unsend_events.json")
|
|
24
28
|
|
|
25
29
|
__version__ = get_version()
|
|
26
30
|
_collector = Collector()
|
|
@@ -33,13 +37,13 @@ def init():
|
|
|
33
37
|
|
|
34
38
|
# Amplitude init
|
|
35
39
|
_collector.set_api_key(api_key)
|
|
36
|
-
_collector.set_user_id(user_profile.get(
|
|
40
|
+
_collector.set_user_id(user_profile.get("user_id"))
|
|
37
41
|
_collector.set_unsend_events_file(RECCE_USER_EVENT_PATH)
|
|
38
42
|
|
|
39
43
|
# Sentry init
|
|
40
44
|
sentry_env = _get_sentry_env()
|
|
41
45
|
sentry_dns = _get_sentry_dns()
|
|
42
|
-
release_version = __version__ if sentry_env !=
|
|
46
|
+
release_version = __version__ if sentry_env != "development" else None
|
|
43
47
|
sentry_sdk.init(
|
|
44
48
|
dsn=sentry_dns,
|
|
45
49
|
environment=sentry_env,
|
|
@@ -47,58 +51,58 @@ def init():
|
|
|
47
51
|
# Set traces_sample_rate to 1.0 to capture 100%
|
|
48
52
|
# of transactions for performance monitoring.
|
|
49
53
|
# We recommend adjusting this value in production.
|
|
50
|
-
traces_sample_rate=1.0
|
|
54
|
+
traces_sample_rate=1.0,
|
|
51
55
|
)
|
|
52
|
-
sentry_sdk.set_tag(
|
|
53
|
-
sentry_sdk.set_tag(
|
|
54
|
-
sentry_sdk.set_tag(
|
|
55
|
-
sentry_sdk.set_tag(
|
|
56
|
-
sentry_sdk.set_tag(
|
|
56
|
+
sentry_sdk.set_tag("recce.version", __version__)
|
|
57
|
+
sentry_sdk.set_tag("platform", sys.platform)
|
|
58
|
+
sentry_sdk.set_tag("is_ci_env", is_ci_env())
|
|
59
|
+
sentry_sdk.set_tag("is_github_codespace", is_github_codespace())
|
|
60
|
+
sentry_sdk.set_tag("system_timezone", get_system_timezone())
|
|
57
61
|
|
|
58
62
|
|
|
59
63
|
def get_user_id():
|
|
60
|
-
return load_user_profile().get(
|
|
64
|
+
return load_user_profile().get("user_id")
|
|
61
65
|
|
|
62
66
|
|
|
63
67
|
def get_recce_api_token():
|
|
64
|
-
return load_user_profile().get(
|
|
68
|
+
return load_user_profile().get("api_token")
|
|
65
69
|
|
|
66
70
|
|
|
67
71
|
def is_anonymous_tracking():
|
|
68
|
-
return load_user_profile().get(
|
|
72
|
+
return load_user_profile().get("anonymous_tracking", False)
|
|
69
73
|
|
|
70
74
|
|
|
71
75
|
def _get_sentry_dns():
|
|
72
|
-
dns_file = os.path.normpath(os.path.join(os.path.dirname(__file__),
|
|
76
|
+
dns_file = os.path.normpath(os.path.join(os.path.dirname(__file__), "SENTRY_DNS"))
|
|
73
77
|
with open(dns_file) as f:
|
|
74
78
|
dns = f.read().strip()
|
|
75
79
|
return dns
|
|
76
80
|
|
|
77
81
|
|
|
78
82
|
def _get_sentry_env():
|
|
79
|
-
if
|
|
80
|
-
return
|
|
81
|
-
elif re.match(r
|
|
82
|
-
return
|
|
83
|
-
elif
|
|
84
|
-
return
|
|
85
|
-
elif
|
|
86
|
-
return
|
|
87
|
-
elif
|
|
88
|
-
return
|
|
89
|
-
return
|
|
83
|
+
if ".dev" in __version__:
|
|
84
|
+
return "development"
|
|
85
|
+
elif re.match(r"^\d+\.\d+\.\d+\.\d{8}[a|b|rc]?.*$", __version__):
|
|
86
|
+
return "nightly"
|
|
87
|
+
elif "a" in __version__:
|
|
88
|
+
return "alpha"
|
|
89
|
+
elif "b" in __version__:
|
|
90
|
+
return "beta"
|
|
91
|
+
elif "rc" in __version__:
|
|
92
|
+
return "release-candidate"
|
|
93
|
+
return "production"
|
|
90
94
|
|
|
91
95
|
|
|
92
96
|
def _get_api_key():
|
|
93
|
-
if os.getenv(
|
|
97
|
+
if os.getenv("RECCE_EVENT_API_KEY"):
|
|
94
98
|
# For local testing purpose
|
|
95
|
-
return os.getenv(
|
|
99
|
+
return os.getenv("RECCE_EVENT_API_KEY")
|
|
96
100
|
|
|
97
|
-
config_file = os.path.abspath(os.path.join(os.path.dirname(__file__),
|
|
101
|
+
config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), "CONFIG"))
|
|
98
102
|
try:
|
|
99
103
|
with open(config_file) as fh:
|
|
100
104
|
config = pyml.load(fh)
|
|
101
|
-
return config.get(
|
|
105
|
+
return config.get("event_api_key")
|
|
102
106
|
except Exception:
|
|
103
107
|
return None
|
|
104
108
|
|
|
@@ -108,15 +112,15 @@ def _generate_user_profile():
|
|
|
108
112
|
os.makedirs(RECCE_USER_HOME, exist_ok=True)
|
|
109
113
|
except Exception:
|
|
110
114
|
# TODO: should show warning message but not raise exception
|
|
111
|
-
print(
|
|
115
|
+
print("Please disable command tracking to continue.")
|
|
112
116
|
exit(1)
|
|
113
117
|
if is_github_codespace() is True:
|
|
114
|
-
salted_name = f
|
|
118
|
+
salted_name = f"codespace-{get_github_codespace_name()}"
|
|
115
119
|
user_id = hashlib.sha256(salted_name.encode()).hexdigest()
|
|
116
120
|
else:
|
|
117
121
|
user_id = uuid.uuid4().hex
|
|
118
|
-
with open(RECCE_USER_PROFILE,
|
|
119
|
-
pyml.dump({
|
|
122
|
+
with open(RECCE_USER_PROFILE, "w+") as f:
|
|
123
|
+
pyml.dump({"user_id": user_id, "anonymous_tracking": True}, f)
|
|
120
124
|
return dict(user_id=user_id, anonymous_tracking=True)
|
|
121
125
|
|
|
122
126
|
|
|
@@ -125,9 +129,9 @@ def load_user_profile():
|
|
|
125
129
|
if not os.path.exists(RECCE_USER_PROFILE):
|
|
126
130
|
user_profile = _generate_user_profile()
|
|
127
131
|
else:
|
|
128
|
-
with open(RECCE_USER_PROFILE,
|
|
132
|
+
with open(RECCE_USER_PROFILE, "r") as f:
|
|
129
133
|
user_profile = pyml.load(f)
|
|
130
|
-
if user_profile.get(
|
|
134
|
+
if user_profile.get("user_id") is None:
|
|
131
135
|
user_profile = _generate_user_profile()
|
|
132
136
|
|
|
133
137
|
return user_profile
|
|
@@ -136,7 +140,7 @@ def load_user_profile():
|
|
|
136
140
|
def update_user_profile(update_values):
|
|
137
141
|
original = load_user_profile()
|
|
138
142
|
original.update(update_values)
|
|
139
|
-
with open(RECCE_USER_PROFILE,
|
|
143
|
+
with open(RECCE_USER_PROFILE, "w+") as f:
|
|
140
144
|
pyml.dump(original, f)
|
|
141
145
|
return original
|
|
142
146
|
|
|
@@ -146,10 +150,10 @@ def flush_events(command=None):
|
|
|
146
150
|
|
|
147
151
|
|
|
148
152
|
def should_log_event():
|
|
149
|
-
with open(RECCE_USER_PROFILE,
|
|
153
|
+
with open(RECCE_USER_PROFILE, "r") as f:
|
|
150
154
|
user_profile = pyml.load(f)
|
|
151
155
|
# TODO: default anonymous_tracking to false if field is not present
|
|
152
|
-
tracking = user_profile.get(
|
|
156
|
+
tracking = user_profile.get("anonymous_tracking", False)
|
|
153
157
|
tracking = tracking and isinstance(tracking, bool)
|
|
154
158
|
if not tracking:
|
|
155
159
|
return False
|
|
@@ -166,18 +170,18 @@ def log_event(prop, event_type, **kwargs):
|
|
|
166
170
|
|
|
167
171
|
repo = hosting_repo()
|
|
168
172
|
if repo is not None:
|
|
169
|
-
prop[
|
|
173
|
+
prop["repository"] = sha256(repo.encode()).hexdigest()
|
|
170
174
|
|
|
171
175
|
branch = current_branch()
|
|
172
176
|
if branch is not None:
|
|
173
|
-
prop[
|
|
177
|
+
prop["branch"] = sha256(branch.encode()).hexdigest()
|
|
174
178
|
|
|
175
179
|
runner = get_runner()
|
|
176
180
|
if runner is not None:
|
|
177
|
-
prop[
|
|
181
|
+
prop["runner_type"] = runner
|
|
178
182
|
|
|
179
|
-
if runner ==
|
|
180
|
-
prop[
|
|
183
|
+
if runner == "github codespaces":
|
|
184
|
+
prop["codespaces_name"] = get_github_codespace_name()
|
|
181
185
|
|
|
182
186
|
payload = dict(
|
|
183
187
|
**prop,
|
|
@@ -191,11 +195,11 @@ def log_api_event(endpoint_name, prop):
|
|
|
191
195
|
**prop,
|
|
192
196
|
endpoint_name=endpoint_name,
|
|
193
197
|
)
|
|
194
|
-
log_event(prop,
|
|
198
|
+
log_event(prop, "api_event")
|
|
195
199
|
_collector.schedule_flush()
|
|
196
200
|
|
|
197
201
|
|
|
198
|
-
def log_load_state(command=
|
|
202
|
+
def log_load_state(command="server", single_env=False):
|
|
199
203
|
from recce.models import CheckDAO
|
|
200
204
|
|
|
201
205
|
checks = 0
|
|
@@ -213,10 +217,10 @@ def log_load_state(command='server', single_env=False):
|
|
|
213
217
|
)
|
|
214
218
|
|
|
215
219
|
if command == "server":
|
|
216
|
-
prop[
|
|
220
|
+
prop["single_env"] = single_env
|
|
217
221
|
|
|
218
|
-
log_event(prop,
|
|
219
|
-
if command ==
|
|
222
|
+
log_event(prop, "load_state")
|
|
223
|
+
if command == "server":
|
|
220
224
|
_collector.schedule_flush()
|
|
221
225
|
|
|
222
226
|
|
|
@@ -227,31 +231,29 @@ def log_codespaces_events(command):
|
|
|
227
231
|
return
|
|
228
232
|
|
|
229
233
|
user_prop = dict(
|
|
230
|
-
location=codespace.get(
|
|
231
|
-
is_prebuild=codespace.get(
|
|
234
|
+
location=codespace.get("location"),
|
|
235
|
+
is_prebuild=codespace.get("prebuild", False),
|
|
232
236
|
)
|
|
233
237
|
|
|
234
238
|
prop = dict(
|
|
235
|
-
machine=codespace.get(
|
|
239
|
+
machine=codespace.get("machine", {}).get("display_name"),
|
|
236
240
|
codespaces_name=get_github_codespace_name(),
|
|
237
241
|
)
|
|
238
242
|
|
|
239
243
|
# Codespace created event, send once
|
|
240
|
-
codespace_created_at = load_user_profile().get(
|
|
244
|
+
codespace_created_at = load_user_profile().get("codespace_created_at")
|
|
241
245
|
if codespace_created_at is None:
|
|
242
|
-
created_at = datetime.fromisoformat(codespace.get(
|
|
243
|
-
prop[
|
|
244
|
-
_collector.log_event(prop,
|
|
245
|
-
|
|
246
|
-
update_user_profile({'codespace_created_at': codespace.get('created_at')})
|
|
246
|
+
created_at = datetime.fromisoformat(codespace.get("created_at"))
|
|
247
|
+
prop["state"] = "created"
|
|
248
|
+
_collector.log_event(prop, "codespace_instance", event_triggered_at=created_at, user_properties=user_prop)
|
|
249
|
+
update_user_profile({"codespace_created_at": codespace.get("created_at")})
|
|
247
250
|
|
|
248
251
|
# Codespace available event, send multiple times as start/stop it
|
|
249
252
|
available_at = get_github_codespace_available_at(codespace)
|
|
250
|
-
if available_at and available_at.isoformat() != load_user_profile().get(
|
|
251
|
-
prop[
|
|
252
|
-
_collector.log_event(prop,
|
|
253
|
-
|
|
254
|
-
update_user_profile({'codespace_available_at': available_at.isoformat()})
|
|
253
|
+
if available_at and available_at.isoformat() != load_user_profile().get("codespace_available_at"):
|
|
254
|
+
prop["state"] = "available"
|
|
255
|
+
_collector.log_event(prop, "codespace_instance", event_triggered_at=available_at, user_properties=user_prop)
|
|
256
|
+
update_user_profile({"codespace_available_at": available_at.isoformat()})
|
|
255
257
|
|
|
256
258
|
# Codespace instance event should be flushed immediately
|
|
257
259
|
_collector.send_events()
|
|
@@ -259,20 +261,20 @@ def log_codespaces_events(command):
|
|
|
259
261
|
|
|
260
262
|
def log_single_env_event():
|
|
261
263
|
prop = dict(
|
|
262
|
-
action=
|
|
264
|
+
action="launch_server",
|
|
263
265
|
)
|
|
264
|
-
log_event(prop,
|
|
266
|
+
log_event(prop, "[Experiment] single_environment")
|
|
265
267
|
_collector.schedule_flush()
|
|
266
268
|
|
|
267
269
|
|
|
268
270
|
def log_performance(feature_name: str, metrics: Dict):
|
|
269
271
|
prop = metrics
|
|
270
|
-
log_event(prop, f
|
|
272
|
+
log_event(prop, f"[Performance] {feature_name}")
|
|
271
273
|
_collector.schedule_flush()
|
|
272
274
|
|
|
273
275
|
|
|
274
276
|
def capture_exception(e):
|
|
275
|
-
user_id = load_user_profile().get(
|
|
277
|
+
user_id = load_user_profile().get("user_id")
|
|
276
278
|
if is_ci_env() is True:
|
|
277
279
|
user_id = f"{user_id}_CI"
|
|
278
280
|
|
recce/event/collector.py
CHANGED
|
@@ -4,9 +4,8 @@ import platform
|
|
|
4
4
|
import sys
|
|
5
5
|
import time
|
|
6
6
|
from contextlib import contextmanager
|
|
7
|
-
from datetime import datetime
|
|
7
|
+
from datetime import datetime, timezone
|
|
8
8
|
from json import JSONDecodeError
|
|
9
|
-
from datetime import timezone
|
|
10
9
|
|
|
11
10
|
import portalocker
|
|
12
11
|
import requests
|
|
@@ -17,7 +16,7 @@ from recce.github import is_github_codespace
|
|
|
17
16
|
|
|
18
17
|
class Collector:
|
|
19
18
|
def __init__(self):
|
|
20
|
-
self._api_endpoint =
|
|
19
|
+
self._api_endpoint = "https://api.amplitude.com/2/httpapi"
|
|
21
20
|
self._api_key = None
|
|
22
21
|
self._user_id = None
|
|
23
22
|
|
|
@@ -35,6 +34,7 @@ class Collector:
|
|
|
35
34
|
|
|
36
35
|
# send async thread
|
|
37
36
|
import threading
|
|
37
|
+
|
|
38
38
|
if self._flush_timer:
|
|
39
39
|
try:
|
|
40
40
|
self._flush_timer.cancel()
|
|
@@ -59,11 +59,18 @@ class Collector:
|
|
|
59
59
|
self._unsend_events_file = unsend_events_file
|
|
60
60
|
self._check_required_files()
|
|
61
61
|
|
|
62
|
-
def _log_event(
|
|
62
|
+
def _log_event(
|
|
63
|
+
self,
|
|
64
|
+
user_id,
|
|
65
|
+
event_type,
|
|
66
|
+
created_at,
|
|
67
|
+
user_properties,
|
|
68
|
+
event_properties,
|
|
69
|
+
):
|
|
63
70
|
event = dict(
|
|
64
71
|
user_id=user_id,
|
|
65
72
|
event_type=event_type,
|
|
66
|
-
ip=
|
|
73
|
+
ip="$remote",
|
|
67
74
|
time=int(time.mktime(created_at.timetuple())),
|
|
68
75
|
user_properties=user_properties,
|
|
69
76
|
event_properties=event_properties,
|
|
@@ -91,7 +98,7 @@ class Collector:
|
|
|
91
98
|
else:
|
|
92
99
|
# Convert to UTC timezone
|
|
93
100
|
created_at = event_triggered_at.astimezone(timezone.utc)
|
|
94
|
-
python_version = f
|
|
101
|
+
python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
|
|
95
102
|
|
|
96
103
|
# when the recce is running in automation use cases
|
|
97
104
|
# replace the user id with project_id to avoid so many unique user id
|
|
@@ -120,17 +127,17 @@ class Collector:
|
|
|
120
127
|
if not os.path.exists(user_home):
|
|
121
128
|
os.makedirs(user_home, exist_ok=True)
|
|
122
129
|
if not os.path.exists(self._unsend_events_file):
|
|
123
|
-
with portalocker.Lock(self._unsend_events_file,
|
|
124
|
-
f.write(json.dumps({
|
|
130
|
+
with portalocker.Lock(self._unsend_events_file, "w+", timeout=5) as f:
|
|
131
|
+
f.write(json.dumps({"unsend_events": []}))
|
|
125
132
|
|
|
126
133
|
def _is_full(self):
|
|
127
|
-
with portalocker.Lock(self._unsend_events_file,
|
|
134
|
+
with portalocker.Lock(self._unsend_events_file, "r+", timeout=5) as f:
|
|
128
135
|
o = json.loads(f.read())
|
|
129
|
-
return len(o.get(
|
|
136
|
+
return len(o.get("unsend_events", [])) >= self._upload_threshold
|
|
130
137
|
|
|
131
138
|
@contextmanager
|
|
132
139
|
def load_json(self):
|
|
133
|
-
with portalocker.Lock(self._unsend_events_file,
|
|
140
|
+
with portalocker.Lock(self._unsend_events_file, "r+", timeout=5) as f:
|
|
134
141
|
try:
|
|
135
142
|
o = json.loads(f.read())
|
|
136
143
|
yield o
|
|
@@ -146,9 +153,9 @@ class Collector:
|
|
|
146
153
|
with self.load_json() as o:
|
|
147
154
|
payload = dict(
|
|
148
155
|
api_key=self._api_key,
|
|
149
|
-
events=o[
|
|
156
|
+
events=o["unsend_events"],
|
|
150
157
|
)
|
|
151
|
-
o[
|
|
158
|
+
o["unsend_events"] = []
|
|
152
159
|
try:
|
|
153
160
|
requests.post(self._api_endpoint, json=payload)
|
|
154
161
|
except Exception:
|
|
@@ -157,17 +164,17 @@ class Collector:
|
|
|
157
164
|
|
|
158
165
|
def _store_to_file(self, event):
|
|
159
166
|
with self.load_json() as o:
|
|
160
|
-
events = o.get(
|
|
167
|
+
events = o.get("unsend_events", None)
|
|
161
168
|
if events is None:
|
|
162
|
-
o[
|
|
169
|
+
o["unsend_events"] = []
|
|
163
170
|
|
|
164
|
-
o[
|
|
171
|
+
o["unsend_events"].append(event)
|
|
165
172
|
|
|
166
173
|
def _cleanup_unsend_events(self):
|
|
167
174
|
with self.load_json() as o:
|
|
168
|
-
events = o.get(
|
|
175
|
+
events = o.get("unsend_events", None)
|
|
169
176
|
if events is None:
|
|
170
|
-
o[
|
|
177
|
+
o["unsend_events"] = []
|
|
171
178
|
|
|
172
|
-
while len(o[
|
|
173
|
-
o[
|
|
179
|
+
while len(o["unsend_events"]) > self._delete_threshold:
|
|
180
|
+
o["unsend_events"].pop(0)
|
recce/event/track.py
CHANGED
|
@@ -18,8 +18,8 @@ from recce.git import current_branch, hosting_repo
|
|
|
18
18
|
|
|
19
19
|
console = Console()
|
|
20
20
|
|
|
21
|
-
_enable_traceback: bool = os.environ.get(
|
|
22
|
-
logger = logging.getLogger(
|
|
21
|
+
_enable_traceback: bool = os.environ.get("RECCE_PRINT_TRACEBACK") == "1"
|
|
22
|
+
logger = logging.getLogger("uvicorn")
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
class TrackCommand(Command):
|
|
@@ -39,11 +39,23 @@ class TrackCommand(Command):
|
|
|
39
39
|
deprecated: bool = False,
|
|
40
40
|
beta: bool = False,
|
|
41
41
|
) -> None:
|
|
42
|
-
super(TrackCommand, self).__init__(
|
|
43
|
-
|
|
42
|
+
super(TrackCommand, self).__init__(
|
|
43
|
+
name,
|
|
44
|
+
context_settings,
|
|
45
|
+
callback,
|
|
46
|
+
params,
|
|
47
|
+
help,
|
|
48
|
+
epilog,
|
|
49
|
+
short_help,
|
|
50
|
+
options_metavar,
|
|
51
|
+
add_help_option,
|
|
52
|
+
no_args_is_help,
|
|
53
|
+
hidden,
|
|
54
|
+
deprecated,
|
|
55
|
+
)
|
|
44
56
|
|
|
45
57
|
def _show_error_message(self, msg, params):
|
|
46
|
-
if params.get(
|
|
58
|
+
if params.get("debug"):
|
|
47
59
|
console.print_exception(show_locals=True)
|
|
48
60
|
else:
|
|
49
61
|
print(traceback.format_exc())
|
|
@@ -51,39 +63,39 @@ class TrackCommand(Command):
|
|
|
51
63
|
# console.out(msg, highlight=False)
|
|
52
64
|
|
|
53
65
|
def _show_hint_message(self, hint):
|
|
54
|
-
console.print(f
|
|
66
|
+
console.print(f"[bold yellow]Hint[/bold yellow]:\n {escape(hint)}")
|
|
55
67
|
|
|
56
68
|
def invoke(self, ctx: Context) -> t.Any:
|
|
57
69
|
status = False
|
|
58
70
|
start_time = time.time()
|
|
59
|
-
reason =
|
|
60
|
-
event.set_exception_tag(
|
|
71
|
+
reason = "error"
|
|
72
|
+
event.set_exception_tag("command", ctx.command.name)
|
|
61
73
|
event.log_codespaces_events(ctx.command.name)
|
|
62
74
|
|
|
63
75
|
try:
|
|
64
76
|
ret = super(TrackCommand, self).invoke(ctx)
|
|
65
77
|
if ret is None or ret == 0:
|
|
66
78
|
status = True
|
|
67
|
-
reason =
|
|
79
|
+
reason = "ok"
|
|
68
80
|
else:
|
|
69
|
-
reason =
|
|
81
|
+
reason = "error"
|
|
70
82
|
sys.exit(ret)
|
|
71
83
|
return ret
|
|
72
84
|
except RecceException as e:
|
|
73
85
|
logger.debug(traceback.format_exc())
|
|
74
86
|
console.log("[Error] " + str(e))
|
|
75
|
-
reason =
|
|
87
|
+
reason = "error"
|
|
76
88
|
sys.exit(1)
|
|
77
89
|
except SystemExit as e:
|
|
78
|
-
reason =
|
|
90
|
+
reason = "error"
|
|
79
91
|
raise e
|
|
80
92
|
except KeyboardInterrupt as e:
|
|
81
|
-
reason =
|
|
93
|
+
reason = "aborted"
|
|
82
94
|
raise e
|
|
83
95
|
except Exception as e:
|
|
84
96
|
self._show_error_message(str(e), ctx.params)
|
|
85
97
|
event.capture_exception(e)
|
|
86
|
-
reason =
|
|
98
|
+
reason = "fatal"
|
|
87
99
|
event.flush_exceptions()
|
|
88
100
|
sys.exit(1)
|
|
89
101
|
finally:
|
|
@@ -93,32 +105,32 @@ class TrackCommand(Command):
|
|
|
93
105
|
branch = current_branch()
|
|
94
106
|
command = ctx.command.name
|
|
95
107
|
duration = end_time - start_time
|
|
96
|
-
target_path = ctx.params.get(
|
|
97
|
-
target_base_path = ctx.params.get(
|
|
108
|
+
target_path = ctx.params.get("target_path", None)
|
|
109
|
+
target_base_path = ctx.params.get("target_base_path", None)
|
|
98
110
|
props = dict(
|
|
99
111
|
command=command,
|
|
100
112
|
status=status,
|
|
101
113
|
reason=reason,
|
|
102
114
|
duration=duration,
|
|
103
|
-
cloud=ctx.params.get(
|
|
104
|
-
review=ctx.params.get(
|
|
105
|
-
debug=ctx.params.get(
|
|
115
|
+
cloud=ctx.params.get("cloud", False),
|
|
116
|
+
review=ctx.params.get("review", False),
|
|
117
|
+
debug=ctx.params.get("debug", False),
|
|
106
118
|
)
|
|
107
119
|
|
|
108
120
|
if runner is not None:
|
|
109
|
-
props[
|
|
121
|
+
props["runner_type"] = runner
|
|
110
122
|
|
|
111
123
|
if repo is not None:
|
|
112
|
-
props[
|
|
124
|
+
props["repository"] = sha256(repo.encode()).hexdigest()
|
|
113
125
|
|
|
114
126
|
if branch is not None:
|
|
115
|
-
props[
|
|
127
|
+
props["branch"] = sha256(branch.encode()).hexdigest()
|
|
116
128
|
|
|
117
129
|
if target_path is not None:
|
|
118
|
-
props[
|
|
130
|
+
props["target_path"] = sha256(target_path.encode()).hexdigest()
|
|
119
131
|
|
|
120
132
|
if target_base_path is not None:
|
|
121
|
-
props[
|
|
133
|
+
props["target_base_path"] = sha256(target_base_path.encode()).hexdigest()
|
|
122
134
|
|
|
123
135
|
try:
|
|
124
136
|
recce_context = load_context()
|
|
@@ -128,9 +140,9 @@ class TrackCommand(Command):
|
|
|
128
140
|
|
|
129
141
|
if recce_context is not None:
|
|
130
142
|
if recce_context.adapter_type == "dbt":
|
|
131
|
-
props[
|
|
143
|
+
props["adapter_type"] = "DBT"
|
|
132
144
|
elif recce_context.adapter_type == "sqlmesh":
|
|
133
|
-
props[
|
|
145
|
+
props["adapter_type"] = "SQLMesh"
|
|
134
146
|
|
|
135
|
-
event.log_event(props,
|
|
147
|
+
event.log_event(props, "command", params=ctx.params)
|
|
136
148
|
event.flush_events()
|