tinybird 0.0.1.dev291__py3-none-any.whl → 1.0.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tinybird/ch_utils/constants.py +5 -0
- tinybird/connectors.py +1 -7
- tinybird/context.py +3 -3
- tinybird/datafile/common.py +10 -8
- tinybird/datafile/parse_pipe.py +2 -2
- tinybird/feedback_manager.py +3 -0
- tinybird/prompts.py +1 -0
- tinybird/service_datasources.py +223 -0
- tinybird/sql_template.py +26 -11
- tinybird/sql_template_fmt.py +14 -4
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/cli.py +1 -0
- tinybird/tb/client.py +104 -26
- tinybird/tb/config.py +24 -0
- tinybird/tb/modules/agent/agent.py +103 -67
- tinybird/tb/modules/agent/banner.py +15 -15
- tinybird/tb/modules/agent/explore_agent.py +5 -0
- tinybird/tb/modules/agent/mock_agent.py +5 -1
- tinybird/tb/modules/agent/models.py +6 -2
- tinybird/tb/modules/agent/prompts.py +49 -2
- tinybird/tb/modules/agent/tools/deploy.py +1 -1
- tinybird/tb/modules/agent/tools/execute_query.py +15 -18
- tinybird/tb/modules/agent/tools/request_endpoint.py +1 -1
- tinybird/tb/modules/agent/tools/run_command.py +9 -0
- tinybird/tb/modules/agent/utils.py +38 -48
- tinybird/tb/modules/branch.py +150 -0
- tinybird/tb/modules/build.py +58 -13
- tinybird/tb/modules/build_common.py +209 -25
- tinybird/tb/modules/cli.py +129 -16
- tinybird/tb/modules/common.py +172 -146
- tinybird/tb/modules/connection.py +125 -194
- tinybird/tb/modules/connection_kafka.py +382 -0
- tinybird/tb/modules/copy.py +3 -1
- tinybird/tb/modules/create.py +83 -150
- tinybird/tb/modules/datafile/build.py +27 -38
- tinybird/tb/modules/datafile/build_datasource.py +21 -25
- tinybird/tb/modules/datafile/diff.py +1 -1
- tinybird/tb/modules/datafile/format_pipe.py +46 -7
- tinybird/tb/modules/datafile/playground.py +59 -68
- tinybird/tb/modules/datafile/pull.py +2 -3
- tinybird/tb/modules/datasource.py +477 -308
- tinybird/tb/modules/deployment.py +2 -0
- tinybird/tb/modules/deployment_common.py +84 -44
- tinybird/tb/modules/deprecations.py +4 -4
- tinybird/tb/modules/dev_server.py +33 -12
- tinybird/tb/modules/exceptions.py +14 -0
- tinybird/tb/modules/feedback_manager.py +1 -1
- tinybird/tb/modules/info.py +69 -12
- tinybird/tb/modules/infra.py +4 -5
- tinybird/tb/modules/job_common.py +15 -0
- tinybird/tb/modules/local.py +143 -23
- tinybird/tb/modules/local_common.py +347 -19
- tinybird/tb/modules/local_logs.py +209 -0
- tinybird/tb/modules/login.py +21 -2
- tinybird/tb/modules/login_common.py +254 -12
- tinybird/tb/modules/mock.py +5 -54
- tinybird/tb/modules/mock_common.py +0 -54
- tinybird/tb/modules/open.py +10 -5
- tinybird/tb/modules/project.py +14 -5
- tinybird/tb/modules/shell.py +15 -7
- tinybird/tb/modules/sink.py +3 -1
- tinybird/tb/modules/telemetry.py +11 -3
- tinybird/tb/modules/test.py +13 -9
- tinybird/tb/modules/test_common.py +13 -87
- tinybird/tb/modules/tinyunit/tinyunit.py +0 -14
- tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -6
- tinybird/tb/modules/watch.py +5 -3
- tinybird/tb_cli_modules/common.py +2 -2
- tinybird/tb_cli_modules/telemetry.py +1 -1
- tinybird/tornado_template.py +6 -7
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/METADATA +32 -6
- tinybird-1.0.5.dist-info/RECORD +132 -0
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/WHEEL +1 -1
- tinybird-0.0.1.dev291.dist-info/RECORD +0 -128
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import platform
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
from docker.client import DockerClient
|
|
6
|
+
from docker.models.containers import Container
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def clickhouse_is_ready(container: Container) -> bool:
|
|
10
|
+
try:
|
|
11
|
+
result = container.exec_run("clickhouse 'SELECT 1 AS healthcheck'")
|
|
12
|
+
return result.output.decode("utf-8").strip() == "1"
|
|
13
|
+
except Exception:
|
|
14
|
+
return False
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def redis_is_ready(container: Container) -> bool:
|
|
18
|
+
try:
|
|
19
|
+
result = container.exec_run("redis-cli PING")
|
|
20
|
+
return result.output.decode("utf-8").strip() == "PONG"
|
|
21
|
+
except Exception:
|
|
22
|
+
return False
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def local_authentication_is_ready(container: Container) -> bool:
|
|
26
|
+
try:
|
|
27
|
+
result = container.exec_run("curl -s http://localhost:8000/tokens")
|
|
28
|
+
data = json.loads(result.output.decode("utf-8").strip())
|
|
29
|
+
token_keys = ["admin_token", "user_token", "workspace_admin_token"]
|
|
30
|
+
return all(key in data for key in token_keys)
|
|
31
|
+
except Exception:
|
|
32
|
+
return False
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def server_is_ready(container: Container) -> bool:
|
|
36
|
+
try:
|
|
37
|
+
result = container.exec_run("curl -s http://localhost:8001/health/liveness")
|
|
38
|
+
is_live = result.output.decode("utf-8").strip() == "alive"
|
|
39
|
+
if not is_live:
|
|
40
|
+
return False
|
|
41
|
+
result = container.exec_run("curl -s http://localhost:8001/health/readiness")
|
|
42
|
+
return result.output.decode("utf-8").strip() == "ready"
|
|
43
|
+
except Exception:
|
|
44
|
+
return False
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def events_is_ready(container: Container) -> bool:
|
|
48
|
+
try:
|
|
49
|
+
result = container.exec_run("curl -s http://localhost:8042/health/liveness")
|
|
50
|
+
is_live = result.output.decode("utf-8").strip() == "alive"
|
|
51
|
+
if not is_live:
|
|
52
|
+
return False
|
|
53
|
+
result = container.exec_run("curl -s http://localhost:8042/health/readiness")
|
|
54
|
+
return result.output.decode("utf-8").strip() == "ready"
|
|
55
|
+
except Exception:
|
|
56
|
+
return False
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def container_is_ready(container: Container) -> bool:
|
|
60
|
+
health = container.attrs.get("State", {}).get("Health", {}).get("Status")
|
|
61
|
+
status = container.status
|
|
62
|
+
return health == "healthy" and status == "running"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def container_is_starting(container: Container) -> bool:
|
|
66
|
+
status = container.status
|
|
67
|
+
health = container.attrs.get("State", {}).get("Health", {}).get("Status")
|
|
68
|
+
return status == "restarting" or (status == "running" and health == "starting")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def container_is_stopping(container: Container) -> bool:
|
|
72
|
+
status = container.status
|
|
73
|
+
return status == "stopping"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def container_is_unhealthy(container: Container) -> bool:
|
|
77
|
+
health = container.attrs.get("State", {}).get("Health", {}).get("Status")
|
|
78
|
+
return health == "unhealthy"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def bytes_to_gb(b):
|
|
82
|
+
return round(b / (1024**3), 2) # two decimal places (e.g., 1.75 GB)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def get_container(client, name_or_id):
|
|
86
|
+
return client.containers.get(name_or_id)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def get_image_arch(client, image_ref):
|
|
90
|
+
try:
|
|
91
|
+
image = client.images.get(image_ref)
|
|
92
|
+
return (image.attrs.get("Architecture") or "").lower()
|
|
93
|
+
except Exception:
|
|
94
|
+
return ""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def is_emulated(host_arch, image_arch):
|
|
98
|
+
# Architecture equivalents - same arch with different names
|
|
99
|
+
arch_equivalents = [
|
|
100
|
+
{"x86_64", "amd64"},
|
|
101
|
+
{"aarch64", "arm64"},
|
|
102
|
+
]
|
|
103
|
+
|
|
104
|
+
if not host_arch or not image_arch:
|
|
105
|
+
return False
|
|
106
|
+
|
|
107
|
+
if host_arch == image_arch:
|
|
108
|
+
return False
|
|
109
|
+
|
|
110
|
+
# Check if architectures are equivalent
|
|
111
|
+
return all(not (host_arch in equiv_set and image_arch in equiv_set) for equiv_set in arch_equivalents)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def mem_usage_percent(container):
|
|
115
|
+
st = container.stats(stream=False)
|
|
116
|
+
mem = st.get("memory_stats", {}) or {}
|
|
117
|
+
limit = float(mem.get("limit") or 0.0)
|
|
118
|
+
usage = float(mem.get("usage") or 0.0)
|
|
119
|
+
stats = mem.get("stats", {}) or {}
|
|
120
|
+
inactive = float(stats.get("total_inactive_file") or stats.get("inactive_file") or 0.0)
|
|
121
|
+
used = max(usage - inactive, 0.0)
|
|
122
|
+
pct = (used / limit * 100.0) if limit > 0 else None
|
|
123
|
+
return used, limit, pct
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def container_stats(container: Container, client: DockerClient):
|
|
127
|
+
host_arch = platform.machine().lower()
|
|
128
|
+
image_arch = get_image_arch(client, container.attrs.get("Config", {}).get("Image", ""))
|
|
129
|
+
emu = is_emulated(host_arch, image_arch)
|
|
130
|
+
used_b, limit_b, pct = mem_usage_percent(container)
|
|
131
|
+
pct = round(pct, 1) if pct is not None else None
|
|
132
|
+
used_gb = bytes_to_gb(used_b)
|
|
133
|
+
limit_gb = bytes_to_gb(limit_b) if limit_b > 0 else None
|
|
134
|
+
lim_str = f"{limit_gb} GB" if limit_gb else "no-limit"
|
|
135
|
+
arch_str = f"arch={host_arch} img={image_arch or 'unknown'} emulated={str(emu).lower()}"
|
|
136
|
+
cpu_usage_pct = cpu_usage_stats(container)
|
|
137
|
+
return f"memory {used_gb}/{lim_str} cpu {cpu_usage_pct} {arch_str}"
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def cpu_usage_stats(container: Container) -> str:
|
|
141
|
+
st = container.stats(stream=False)
|
|
142
|
+
cpu = st.get("cpu_stats", {}) or {}
|
|
143
|
+
cpu_usage = cpu.get("cpu_usage", {}) or {}
|
|
144
|
+
total_usage = cpu_usage.get("total_usage", 0)
|
|
145
|
+
system_cpu_usage = cpu.get("system_cpu_usage", 0)
|
|
146
|
+
pct = (total_usage / system_cpu_usage * 100.0) if system_cpu_usage > 0 else None
|
|
147
|
+
return f"{round(pct, 1) if pct is not None else 'N/A'}%"
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def check_memory_sufficient(container: Container, client: DockerClient) -> tuple[bool, Optional[str]]:
|
|
151
|
+
"""
|
|
152
|
+
Check if container has sufficient memory.
|
|
153
|
+
|
|
154
|
+
Returns:
|
|
155
|
+
tuple[bool, str | None]: (is_sufficient, warning_message)
|
|
156
|
+
- is_sufficient: True if memory is sufficient, False otherwise
|
|
157
|
+
- warning_message: None if sufficient, otherwise a warning message
|
|
158
|
+
"""
|
|
159
|
+
host_arch = platform.machine().lower()
|
|
160
|
+
image_arch = get_image_arch(client, container.attrs.get("Config", {}).get("Image", ""))
|
|
161
|
+
is_emu = is_emulated(host_arch, image_arch)
|
|
162
|
+
used_b, limit_b, pct = mem_usage_percent(container)
|
|
163
|
+
|
|
164
|
+
if limit_b <= 0:
|
|
165
|
+
# No memory limit set
|
|
166
|
+
return True, None
|
|
167
|
+
|
|
168
|
+
limit_gb = bytes_to_gb(limit_b)
|
|
169
|
+
used_gb = bytes_to_gb(used_b)
|
|
170
|
+
|
|
171
|
+
# Memory thresholds
|
|
172
|
+
# For emulated containers, we need more memory and lower threshold
|
|
173
|
+
HIGH_MEMORY_THRESHOLD_EMULATED = 70.0 # 70% for emulated
|
|
174
|
+
HIGH_MEMORY_THRESHOLD_NATIVE = 85.0 # 85% for native
|
|
175
|
+
MINIMUM_MEMORY_GB_EMULATED = 6.0 # Minimum 6GB for emulated
|
|
176
|
+
MINIMUM_MEMORY_GB_NATIVE = 4.0 # Minimum 4GB for native
|
|
177
|
+
|
|
178
|
+
warnings = []
|
|
179
|
+
|
|
180
|
+
# Check memory percentage
|
|
181
|
+
if pct is not None:
|
|
182
|
+
threshold = HIGH_MEMORY_THRESHOLD_EMULATED if is_emu else HIGH_MEMORY_THRESHOLD_NATIVE
|
|
183
|
+
if pct >= threshold:
|
|
184
|
+
warnings.append(
|
|
185
|
+
f"Memory usage is at {pct:.1f}% ({used_gb}/{limit_gb:.2f} GB), "
|
|
186
|
+
f"which exceeds the recommended threshold of {threshold:.0f}%."
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
# Check absolute memory limit
|
|
190
|
+
min_memory = MINIMUM_MEMORY_GB_EMULATED if is_emu else MINIMUM_MEMORY_GB_NATIVE
|
|
191
|
+
if limit_gb < min_memory:
|
|
192
|
+
arch_msg = f" (running emulated {image_arch} on {host_arch})" if is_emu else ""
|
|
193
|
+
warnings.append(
|
|
194
|
+
f"Memory limit is {limit_gb:.2f} GB{arch_msg}, but at least {min_memory:.1f} GB is recommended."
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
if warnings:
|
|
198
|
+
warning_msg = " ".join(warnings)
|
|
199
|
+
if is_emu:
|
|
200
|
+
warning_msg += (
|
|
201
|
+
"\n"
|
|
202
|
+
f"You're running an emulated container ({image_arch} on {host_arch}), which requires more resources.\n"
|
|
203
|
+
"Consider increasing Docker's memory allocation."
|
|
204
|
+
)
|
|
205
|
+
else:
|
|
206
|
+
warning_msg += "Consider increasing Docker's memory allocation."
|
|
207
|
+
return False, warning_msg
|
|
208
|
+
|
|
209
|
+
return True, None
|
tinybird/tb/modules/login.py
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
|
+
import platform
|
|
2
|
+
import sys
|
|
1
3
|
from typing import Optional
|
|
2
4
|
|
|
3
5
|
import click
|
|
4
6
|
|
|
7
|
+
from tinybird.tb.config import CURRENT_VERSION
|
|
5
8
|
from tinybird.tb.modules.cli import cli
|
|
6
9
|
from tinybird.tb.modules.login_common import login
|
|
10
|
+
from tinybird.tb.modules.telemetry import add_telemetry_event
|
|
7
11
|
|
|
8
12
|
|
|
9
13
|
@cli.command("login", help="Authenticate using the browser.")
|
|
@@ -11,12 +15,13 @@ from tinybird.tb.modules.login_common import login
|
|
|
11
15
|
"--host",
|
|
12
16
|
type=str,
|
|
13
17
|
default=None,
|
|
14
|
-
help="Set
|
|
18
|
+
help="Set the API host to authenticate to. See https://www.tinybird.co/docs/api-reference#regions-and-endpoints for the available list of regions.",
|
|
15
19
|
)
|
|
16
20
|
@click.option(
|
|
17
21
|
"--auth-host",
|
|
18
22
|
default="https://cloud.tinybird.co",
|
|
19
|
-
help="Set the host to authenticate to. If unset, the default host will be used.",
|
|
23
|
+
help="Set the auth host to authenticate to. If unset, the default host will be used.",
|
|
24
|
+
hidden=True,
|
|
20
25
|
)
|
|
21
26
|
@click.option(
|
|
22
27
|
"--workspace",
|
|
@@ -37,3 +42,17 @@ from tinybird.tb.modules.login_common import login
|
|
|
37
42
|
)
|
|
38
43
|
def login_cmd(host: Optional[str], auth_host: str, workspace: str, interactive: bool, method: str):
|
|
39
44
|
login(host, auth_host, workspace, interactive, method)
|
|
45
|
+
# we send a telemetry event manually so we have user and workspace info available
|
|
46
|
+
add_telemetry_event(
|
|
47
|
+
"system_info",
|
|
48
|
+
platform=platform.platform(),
|
|
49
|
+
system=platform.system(),
|
|
50
|
+
arch=platform.machine(),
|
|
51
|
+
processor=platform.processor(),
|
|
52
|
+
python_runtime=platform.python_implementation(),
|
|
53
|
+
python_version=platform.python_version(),
|
|
54
|
+
is_ci=False,
|
|
55
|
+
ci_product=None,
|
|
56
|
+
cli_version=CURRENT_VERSION,
|
|
57
|
+
cli_args=sys.argv[1:] if len(sys.argv) > 1 else [],
|
|
58
|
+
)
|
|
@@ -11,17 +11,21 @@ import threading
|
|
|
11
11
|
import time
|
|
12
12
|
import urllib.parse
|
|
13
13
|
import webbrowser
|
|
14
|
+
from datetime import datetime
|
|
15
|
+
from pathlib import Path
|
|
14
16
|
from typing import Any, Dict, Optional
|
|
15
17
|
from urllib.parse import urlencode
|
|
16
18
|
|
|
17
19
|
import click
|
|
18
20
|
import requests
|
|
21
|
+
from click import Context
|
|
19
22
|
|
|
20
23
|
from tinybird.tb.config import DEFAULT_API_HOST
|
|
21
|
-
from tinybird.tb.modules.common import ask_for_region_interactively, get_regions
|
|
24
|
+
from tinybird.tb.modules.common import ask_for_region_interactively, get_region_from_host, get_regions
|
|
22
25
|
from tinybird.tb.modules.config import CLIConfig
|
|
23
26
|
from tinybird.tb.modules.exceptions import CLILoginException
|
|
24
27
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
28
|
+
from tinybird.tb.modules.telemetry import is_ci_environment
|
|
25
29
|
|
|
26
30
|
SERVER_MAX_WAIT_TIME = 180
|
|
27
31
|
|
|
@@ -163,14 +167,13 @@ def login(
|
|
|
163
167
|
"method": "code",
|
|
164
168
|
}
|
|
165
169
|
response = requests.get(f"{auth_host}/api/cli-login?{urlencode(params)}")
|
|
166
|
-
|
|
167
170
|
try:
|
|
168
171
|
if response.status_code == 200:
|
|
169
172
|
data = response.json()
|
|
170
173
|
user_token = data.get("user_token", "")
|
|
171
174
|
workspace_token = data.get("workspace_token", "")
|
|
172
175
|
if user_token and workspace_token:
|
|
173
|
-
authenticate_with_tokens(data,
|
|
176
|
+
authenticate_with_tokens(data, cli_config)
|
|
174
177
|
break
|
|
175
178
|
except Exception:
|
|
176
179
|
pass
|
|
@@ -216,7 +219,7 @@ def login(
|
|
|
216
219
|
)
|
|
217
220
|
|
|
218
221
|
data = response.json()
|
|
219
|
-
authenticate_with_tokens(data,
|
|
222
|
+
authenticate_with_tokens(data, cli_config)
|
|
220
223
|
else:
|
|
221
224
|
raise Exception("Authentication failed or timed out.")
|
|
222
225
|
except Exception as e:
|
|
@@ -249,9 +252,8 @@ def open_url(url: str, *, new_tab: bool = False) -> bool:
|
|
|
249
252
|
if new_tab:
|
|
250
253
|
if wb.open_new_tab(url):
|
|
251
254
|
return True
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
return True
|
|
255
|
+
elif wb.open(url):
|
|
256
|
+
return True
|
|
255
257
|
except webbrowser.Error:
|
|
256
258
|
pass # keep going
|
|
257
259
|
|
|
@@ -295,9 +297,246 @@ def create_one_time_code():
|
|
|
295
297
|
return seperator.join(parts), full_code
|
|
296
298
|
|
|
297
299
|
|
|
298
|
-
def
|
|
300
|
+
def check_current_folder_in_sessions(ctx: Context) -> None:
|
|
301
|
+
"""
|
|
302
|
+
Check if the current folder is tracked in the sessions file before running a command.
|
|
303
|
+
|
|
304
|
+
If the current folder is not in sessions.txt:
|
|
305
|
+
- If no sessions exist or file doesn't exist: silently add it (first usage is fine)
|
|
306
|
+
- If other folders exist: warn the user they're running from an untracked folder
|
|
307
|
+
|
|
308
|
+
Args:
|
|
309
|
+
provider: The provider name (e.g., 'gcp', 'aws')
|
|
310
|
+
region: The region name
|
|
311
|
+
workspace_name: The workspace name
|
|
312
|
+
current_folder: The current working directory
|
|
313
|
+
"""
|
|
314
|
+
try:
|
|
315
|
+
env = ctx.ensure_object(dict)["env"]
|
|
316
|
+
if env != "cloud" or ctx.invoked_subcommand == "login" or is_ci_environment():
|
|
317
|
+
return
|
|
318
|
+
|
|
319
|
+
current_folder = os.getcwd()
|
|
320
|
+
cli_config = CLIConfig.get_project_config()
|
|
321
|
+
regions = get_regions(cli_config)
|
|
322
|
+
host = cli_config.get_host()
|
|
323
|
+
if not host:
|
|
324
|
+
return
|
|
325
|
+
|
|
326
|
+
region = get_region_from_host(host, regions)
|
|
327
|
+
|
|
328
|
+
if not region:
|
|
329
|
+
return
|
|
330
|
+
|
|
331
|
+
provider = region.get("provider", "unknown")
|
|
332
|
+
region_name = region.get("name", "unknown")
|
|
333
|
+
current_folder = os.getcwd()
|
|
334
|
+
home_dir = Path.home()
|
|
335
|
+
workspace_name = cli_config.get("name", "unknown")
|
|
336
|
+
sessions_dir = home_dir / ".tinybird" / provider / region_name / workspace_name
|
|
337
|
+
sessions_file = sessions_dir / "sessions.txt"
|
|
338
|
+
|
|
339
|
+
# Normalize the current folder path
|
|
340
|
+
current_folder = os.path.abspath(current_folder)
|
|
341
|
+
|
|
342
|
+
# Read existing sessions
|
|
343
|
+
existing_sessions: dict[str, str] = {} # folder -> timestamp
|
|
344
|
+
if sessions_file.exists():
|
|
345
|
+
try:
|
|
346
|
+
with open(sessions_file, "r") as f:
|
|
347
|
+
for line in f:
|
|
348
|
+
line = line.strip()
|
|
349
|
+
if line:
|
|
350
|
+
# Format: <folder_path>\t<timestamp>
|
|
351
|
+
parts = line.split("\t")
|
|
352
|
+
if len(parts) >= 2 and parts[0]:
|
|
353
|
+
existing_sessions[parts[0]] = parts[1]
|
|
354
|
+
except Exception:
|
|
355
|
+
# If we can't read the file, just continue silently
|
|
356
|
+
return
|
|
357
|
+
|
|
358
|
+
# Check if current folder is already tracked
|
|
359
|
+
if current_folder in existing_sessions:
|
|
360
|
+
# Already tracked, update the timestamp
|
|
361
|
+
try:
|
|
362
|
+
existing_sessions[current_folder] = datetime.now().isoformat()
|
|
363
|
+
sessions_dir.mkdir(parents=True, exist_ok=True)
|
|
364
|
+
with open(sessions_file, "w") as f:
|
|
365
|
+
f.writelines(f"{folder}\t{timestamp}\n" for folder, timestamp in existing_sessions.items())
|
|
366
|
+
except Exception:
|
|
367
|
+
# Silently fail, don't block the command
|
|
368
|
+
pass
|
|
369
|
+
return
|
|
370
|
+
|
|
371
|
+
# Current folder is not tracked
|
|
372
|
+
if not existing_sessions:
|
|
373
|
+
# No previous sessions, this is the first time - silently add it
|
|
374
|
+
try:
|
|
375
|
+
sessions_dir.mkdir(parents=True, exist_ok=True)
|
|
376
|
+
timestamp = datetime.now().isoformat()
|
|
377
|
+
with open(sessions_file, "a") as f:
|
|
378
|
+
f.write(f"{current_folder}\t{timestamp}\n")
|
|
379
|
+
except Exception:
|
|
380
|
+
# Silently fail, don't block the command
|
|
381
|
+
pass
|
|
382
|
+
else:
|
|
383
|
+
# Other folders exist, warn the user
|
|
384
|
+
click.echo("")
|
|
385
|
+
click.echo(FeedbackManager.warning(message="Running command from an untracked folder"))
|
|
386
|
+
click.echo(FeedbackManager.gray(message="Current folder: ") + FeedbackManager.info(message=current_folder))
|
|
387
|
+
tracked_folders = ", ".join(existing_sessions.keys())
|
|
388
|
+
click.echo(
|
|
389
|
+
FeedbackManager.gray(message="Tracked folders: ") + FeedbackManager.info(message=tracked_folders)
|
|
390
|
+
)
|
|
391
|
+
confirmed = click.confirm(
|
|
392
|
+
FeedbackManager.highlight(message="» Are you sure you want to continue?"),
|
|
393
|
+
default=True,
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
if not confirmed:
|
|
397
|
+
raise TrackFolderCancelled("Command cancelled by user.")
|
|
398
|
+
|
|
399
|
+
# Add current folder to the tracked list
|
|
400
|
+
try:
|
|
401
|
+
timestamp = datetime.now().isoformat()
|
|
402
|
+
with open(sessions_file, "a") as f:
|
|
403
|
+
f.write(f"{current_folder}\t{timestamp}\n")
|
|
404
|
+
except Exception:
|
|
405
|
+
# Silently fail, don't block the command
|
|
406
|
+
pass
|
|
407
|
+
|
|
408
|
+
except TrackFolderCancelled:
|
|
409
|
+
raise
|
|
410
|
+
except Exception:
|
|
411
|
+
# Don't block execution if folder check fails
|
|
412
|
+
pass
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
class TrackFolderCancelled(Exception):
|
|
416
|
+
"""Exception raised when the user cancels the folder tracking"""
|
|
417
|
+
|
|
418
|
+
pass
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def check_and_warn_folder_change(cli_config: CLIConfig) -> None:
|
|
422
|
+
"""
|
|
423
|
+
Check if the user is logging in from a folder that hasn't been tracked before.
|
|
424
|
+
|
|
425
|
+
Reads from ~/.tinybird/<provider>/<region>/<workspace_name>/sessions.txt to track
|
|
426
|
+
folder usage history. If the current folder is not in the list of tracked folders
|
|
427
|
+
and there are existing sessions, prompts the user to confirm.
|
|
428
|
+
|
|
429
|
+
Behavior:
|
|
430
|
+
- If no sessions exist: Silently add current folder
|
|
431
|
+
- If current folder is already tracked: Update timestamp and continue
|
|
432
|
+
- If current folder is not tracked but other folders are: Show warning and ask for confirmation
|
|
433
|
+
|
|
434
|
+
Raises:
|
|
435
|
+
TrackFolderCancelled: If user declines the folder change confirmation
|
|
436
|
+
"""
|
|
437
|
+
if is_ci_environment():
|
|
438
|
+
return
|
|
439
|
+
|
|
440
|
+
host = cli_config.get_host()
|
|
441
|
+
workspace_name = cli_config.get("name", None)
|
|
442
|
+
if not host or not workspace_name:
|
|
443
|
+
return
|
|
444
|
+
|
|
445
|
+
regions = get_regions(cli_config)
|
|
446
|
+
region = get_region_from_host(host, regions)
|
|
447
|
+
|
|
448
|
+
if not region:
|
|
449
|
+
return
|
|
450
|
+
|
|
451
|
+
provider = region.get("provider", "unknown")
|
|
452
|
+
region_name = region.get("name", "unknown")
|
|
453
|
+
|
|
454
|
+
current_folder = os.getcwd()
|
|
455
|
+
|
|
456
|
+
home_dir = Path.home()
|
|
457
|
+
sessions_dir = home_dir / ".tinybird" / provider / region_name / workspace_name
|
|
458
|
+
sessions_file = sessions_dir / "sessions.txt"
|
|
459
|
+
|
|
460
|
+
# Normalize the current folder path
|
|
461
|
+
current_folder = os.path.abspath(current_folder)
|
|
462
|
+
|
|
463
|
+
# Read existing sessions if the file exists
|
|
464
|
+
existing_sessions: dict[str, str] = {} # folder -> timestamp
|
|
465
|
+
if sessions_file.exists():
|
|
466
|
+
try:
|
|
467
|
+
with open(sessions_file, "r") as f:
|
|
468
|
+
for line in f:
|
|
469
|
+
line = line.strip()
|
|
470
|
+
if line:
|
|
471
|
+
# Format: <folder_path>\t<timestamp>
|
|
472
|
+
parts = line.split("\t")
|
|
473
|
+
if len(parts) >= 2 and parts[0]:
|
|
474
|
+
existing_sessions[parts[0]] = parts[1]
|
|
475
|
+
except Exception as e:
|
|
476
|
+
# If we can't read the file, just continue without warning
|
|
477
|
+
click.echo(FeedbackManager.warning(message=f"Warning: Could not read sessions file: {e}"))
|
|
478
|
+
|
|
479
|
+
# If current folder is already tracked, update timestamp and return
|
|
480
|
+
if current_folder in existing_sessions:
|
|
481
|
+
try:
|
|
482
|
+
existing_sessions[current_folder] = datetime.now().isoformat()
|
|
483
|
+
sessions_dir.mkdir(parents=True, exist_ok=True)
|
|
484
|
+
with open(sessions_file, "w") as f:
|
|
485
|
+
f.writelines(f"{folder}\t{timestamp}\n" for folder, timestamp in existing_sessions.items())
|
|
486
|
+
except Exception as e:
|
|
487
|
+
click.echo(FeedbackManager.warning(message=f"Warning: Could not update sessions file: {e}"))
|
|
488
|
+
return
|
|
489
|
+
|
|
490
|
+
# If there are existing tracked folders but current folder is not tracked, warn the user
|
|
491
|
+
if existing_sessions:
|
|
492
|
+
click.echo("")
|
|
493
|
+
click.echo(
|
|
494
|
+
FeedbackManager.warning(message="Login from a different folder than previous sessions has been detected.")
|
|
495
|
+
)
|
|
496
|
+
click.echo(FeedbackManager.gray(message="Current folder: ") + FeedbackManager.info(message=current_folder))
|
|
497
|
+
tracked_folders = ", ".join(existing_sessions.keys())
|
|
498
|
+
click.echo(FeedbackManager.gray(message="Tracked folders: ") + FeedbackManager.info(message=tracked_folders))
|
|
499
|
+
|
|
500
|
+
# Ask for confirmation
|
|
501
|
+
confirmed = click.confirm(
|
|
502
|
+
FeedbackManager.highlight(message="» Are you sure you want to continue?"),
|
|
503
|
+
default=True,
|
|
504
|
+
)
|
|
505
|
+
|
|
506
|
+
if not confirmed:
|
|
507
|
+
raise TrackFolderCancelled("Login cancelled by user.")
|
|
508
|
+
|
|
509
|
+
# User accepted, show pull suggestion
|
|
510
|
+
click.echo(
|
|
511
|
+
FeedbackManager.warning(
|
|
512
|
+
message="Remember to run `tb --cloud pull` to have your latest resources available."
|
|
513
|
+
)
|
|
514
|
+
)
|
|
515
|
+
|
|
516
|
+
# Record the current session
|
|
517
|
+
try:
|
|
518
|
+
# Create the directory if it doesn't exist
|
|
519
|
+
sessions_dir.mkdir(parents=True, exist_ok=True)
|
|
520
|
+
|
|
521
|
+
# Add current folder to sessions and update timestamp
|
|
522
|
+
timestamp = datetime.now().isoformat()
|
|
523
|
+
existing_sessions[current_folder] = timestamp
|
|
524
|
+
|
|
525
|
+
# Write all sessions to file
|
|
526
|
+
with open(sessions_file, "w") as f:
|
|
527
|
+
f.writelines(f"{folder}\t{ts}\n" for folder, ts in existing_sessions.items())
|
|
528
|
+
except Exception as e:
|
|
529
|
+
# If we can't write the file, just warn but don't block the login
|
|
530
|
+
click.echo(FeedbackManager.warning(message=f"Warning: Could not update sessions file: {e}"))
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def authenticate_with_tokens(data: Dict[str, Any], cli_config: CLIConfig):
|
|
299
534
|
cli_config.set_token(data.get("workspace_token", ""))
|
|
300
|
-
host =
|
|
535
|
+
host = data.get("api_host", "")
|
|
536
|
+
|
|
537
|
+
if not host:
|
|
538
|
+
raise Exception("API host not found in the authentication response")
|
|
539
|
+
|
|
301
540
|
cli_config.set_token_for_host(data.get("workspace_token", ""), host)
|
|
302
541
|
cli_config.set_user_token(data.get("user_token", ""))
|
|
303
542
|
cli_config.set_host(host)
|
|
@@ -306,9 +545,6 @@ def authenticate_with_tokens(data: Dict[str, Any], host: Optional[str], cli_conf
|
|
|
306
545
|
if k in ws:
|
|
307
546
|
cli_config[k] = ws[k]
|
|
308
547
|
|
|
309
|
-
path = os.path.join(os.getcwd(), ".tinyb")
|
|
310
|
-
cli_config.persist_to_file(override_with_path=path)
|
|
311
|
-
|
|
312
548
|
auth_info: Dict[str, Any] = cli_config.get_user_client().check_auth_login()
|
|
313
549
|
if not auth_info.get("is_valid", False):
|
|
314
550
|
raise Exception(FeedbackManager.error_auth_login_not_valid(host=cli_config.get_host()))
|
|
@@ -316,6 +552,12 @@ def authenticate_with_tokens(data: Dict[str, Any], host: Optional[str], cli_conf
|
|
|
316
552
|
if not auth_info.get("is_user", False):
|
|
317
553
|
raise Exception(FeedbackManager.error_auth_login_not_user(host=cli_config.get_host()))
|
|
318
554
|
|
|
555
|
+
# Check for folder change before persisting config
|
|
556
|
+
check_and_warn_folder_change(cli_config)
|
|
557
|
+
|
|
558
|
+
path = os.path.join(os.getcwd(), ".tinyb")
|
|
559
|
+
cli_config.persist_to_file(override_with_path=path)
|
|
560
|
+
|
|
319
561
|
click.echo(FeedbackManager.gray(message="\nWorkspace: ") + FeedbackManager.info(message=ws["name"]))
|
|
320
562
|
click.echo(FeedbackManager.gray(message="User: ") + FeedbackManager.info(message=ws["user_email"]))
|
|
321
563
|
click.echo(FeedbackManager.gray(message="Host: ") + FeedbackManager.info(message=host))
|
tinybird/tb/modules/mock.py
CHANGED
|
@@ -1,15 +1,9 @@
|
|
|
1
|
-
import glob
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
|
|
4
1
|
import click
|
|
5
2
|
|
|
6
|
-
from tinybird.tb.
|
|
3
|
+
from tinybird.tb.modules.agent import run_agent
|
|
7
4
|
from tinybird.tb.modules.cli import cli
|
|
8
|
-
from tinybird.tb.modules.config import CLIConfig
|
|
9
|
-
from tinybird.tb.modules.datafile.fixture import persist_fixture
|
|
10
5
|
from tinybird.tb.modules.exceptions import CLIMockException
|
|
11
6
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
12
|
-
from tinybird.tb.modules.mock_common import append_mock_data, create_mock_data
|
|
13
7
|
from tinybird.tb.modules.project import Project
|
|
14
8
|
|
|
15
9
|
|
|
@@ -31,60 +25,17 @@ from tinybird.tb.modules.project import Project
|
|
|
31
25
|
)
|
|
32
26
|
@click.pass_context
|
|
33
27
|
def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, format_: str) -> None:
|
|
34
|
-
"""Generate sample data for a data source.
|
|
35
|
-
|
|
36
|
-
Args:
|
|
37
|
-
datasource: Path to the datasource file to load sample data into
|
|
38
|
-
rows: Number of events to send
|
|
39
|
-
prompt: Extra context to use for data generation
|
|
40
|
-
skip: Skip following up on the generated data
|
|
41
|
-
"""
|
|
28
|
+
"""Generate sample data for a data source."""
|
|
42
29
|
|
|
43
30
|
try:
|
|
44
|
-
tb_client: TinyB = ctx.ensure_object(dict)["client"]
|
|
45
31
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
46
32
|
ctx_config = ctx.ensure_object(dict)["config"]
|
|
33
|
+
prompt = f"""Generate mock data for the following datasource: {datasource} with {rows} rows and {format_} format. Extra context: {prompt}"""
|
|
47
34
|
env = ctx.ensure_object(dict)["env"]
|
|
48
|
-
datasource_path = Path(datasource)
|
|
49
|
-
datasource_name = datasource
|
|
50
|
-
folder = project.folder
|
|
51
|
-
click.echo(FeedbackManager.highlight(message=f"\n» Creating fixture for {datasource_name}..."))
|
|
52
|
-
if datasource_path.suffix == ".datasource":
|
|
53
|
-
datasource_name = datasource_path.stem
|
|
54
|
-
else:
|
|
55
|
-
datasource_from_glob = glob.glob(f"{folder}/**/{datasource}.datasource")
|
|
56
|
-
if datasource_from_glob:
|
|
57
|
-
datasource_path = Path(datasource_from_glob[0])
|
|
58
|
-
|
|
59
|
-
if not datasource_path.exists():
|
|
60
|
-
raise Exception(f"Datasource '{datasource_path.stem}' not found")
|
|
61
|
-
|
|
62
|
-
datasource_content = datasource_path.read_text()
|
|
63
|
-
config = CLIConfig.get_project_config()
|
|
64
|
-
user_token = ctx_config.get("user_token")
|
|
65
|
-
|
|
66
|
-
if not user_token:
|
|
67
|
-
raise Exception("This action requires authentication. Run 'tb login' first.")
|
|
68
|
-
|
|
69
|
-
data = create_mock_data(
|
|
70
|
-
datasource_name,
|
|
71
|
-
datasource_content,
|
|
72
|
-
rows,
|
|
73
|
-
prompt,
|
|
74
|
-
config,
|
|
75
|
-
ctx_config,
|
|
76
|
-
user_token,
|
|
77
|
-
tb_client,
|
|
78
|
-
format_,
|
|
79
|
-
folder,
|
|
80
|
-
)
|
|
81
|
-
|
|
82
|
-
fixture_path = persist_fixture(datasource_name, data, folder, format=format_)
|
|
83
|
-
click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}.{format_} created"))
|
|
84
35
|
if env == "cloud":
|
|
85
|
-
|
|
36
|
+
prompt += "Append the fixture data to the datasource in Tinybird Cloud."
|
|
86
37
|
|
|
87
|
-
|
|
38
|
+
run_agent(ctx_config, project, True, prompt=prompt, feature="tb_mock")
|
|
88
39
|
|
|
89
40
|
except Exception as e:
|
|
90
41
|
raise CLIMockException(FeedbackManager.error(message=str(e)))
|