tinybird 0.0.1.dev5__py3-none-any.whl → 0.0.1.dev6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/__cli__.py +7 -8
- tinybird/tb/cli.py +28 -0
- tinybird/{tb_cli_modules → tb/modules}/auth.py +5 -5
- tinybird/{tb_cli_modules → tb/modules}/branch.py +6 -5
- tinybird/{tb_cli_modules → tb/modules}/build.py +8 -8
- tinybird/tb/modules/cicd.py +271 -0
- tinybird/{tb_cli_modules → tb/modules}/cli.py +23 -23
- tinybird/tb/modules/common.py +2098 -0
- tinybird/tb/modules/config.py +352 -0
- tinybird/{tb_cli_modules → tb/modules}/connection.py +4 -4
- tinybird/{tb_cli_modules → tb/modules}/create.py +11 -7
- tinybird/{datafile.py → tb/modules/datafile.py} +6 -7
- tinybird/{tb_cli_modules → tb/modules}/datasource.py +7 -6
- tinybird/tb/modules/exceptions.py +91 -0
- tinybird/{tb_cli_modules → tb/modules}/fmt.py +3 -3
- tinybird/{tb_cli_modules → tb/modules}/job.py +3 -3
- tinybird/{tb_cli_modules → tb/modules}/llm.py +1 -1
- tinybird/{tb_cli_modules → tb/modules}/local.py +9 -5
- tinybird/{tb_cli_modules → tb/modules}/mock.py +5 -5
- tinybird/{tb_cli_modules → tb/modules}/pipe.py +5 -5
- tinybird/tb/modules/regions.py +9 -0
- tinybird/{tb_cli_modules → tb/modules}/tag.py +2 -2
- tinybird/tb/modules/telemetry.py +310 -0
- tinybird/{tb_cli_modules → tb/modules}/test.py +5 -5
- tinybird/{tb_cli_modules → tb/modules}/tinyunit/tinyunit.py +1 -1
- tinybird/{tb_cli_modules → tb/modules}/token.py +3 -3
- tinybird/{tb_cli_modules → tb/modules}/workspace.py +5 -5
- tinybird/{tb_cli_modules → tb/modules}/workspace_members.py +4 -4
- tinybird/tb_cli_modules/common.py +9 -25
- tinybird/tb_cli_modules/config.py +0 -8
- {tinybird-0.0.1.dev5.dist-info → tinybird-0.0.1.dev6.dist-info}/METADATA +1 -1
- tinybird-0.0.1.dev6.dist-info/RECORD +58 -0
- tinybird-0.0.1.dev6.dist-info/entry_points.txt +2 -0
- tinybird/tb_cli.py +0 -28
- tinybird-0.0.1.dev5.dist-info/RECORD +0 -52
- tinybird-0.0.1.dev5.dist-info/entry_points.txt +0 -2
- /tinybird/{tb_cli_modules → tb/modules}/prompts.py +0 -0
- /tinybird/{tb_cli_modules → tb/modules}/table.py +0 -0
- /tinybird/{tb_cli_modules → tb/modules}/tinyunit/tinyunit_lib.py +0 -0
- {tinybird-0.0.1.dev5.dist-info → tinybird-0.0.1.dev6.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev5.dist-info → tinybird-0.0.1.dev6.dist-info}/top_level.txt +0 -0
|
@@ -4,8 +4,8 @@ import click
|
|
|
4
4
|
from click import Context
|
|
5
5
|
|
|
6
6
|
from tinybird.feedback_manager import FeedbackManager
|
|
7
|
-
from tinybird.
|
|
8
|
-
from tinybird.
|
|
7
|
+
from tinybird.tb.modules.cli import cli
|
|
8
|
+
from tinybird.tb.modules.common import coro, echo_safe_humanfriendly_tables_format_smart_table
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
@cli.group()
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import platform
|
|
5
|
+
import re
|
|
6
|
+
import sys
|
|
7
|
+
import threading
|
|
8
|
+
import uuid
|
|
9
|
+
from copy import deepcopy
|
|
10
|
+
from datetime import datetime
|
|
11
|
+
from typing import Any, Callable, Dict, List, Optional, Tuple
|
|
12
|
+
from urllib.parse import urlencode
|
|
13
|
+
|
|
14
|
+
import requests
|
|
15
|
+
|
|
16
|
+
from tinybird.config import CURRENT_VERSION
|
|
17
|
+
|
|
18
|
+
TELEMETRY_TIMEOUT: int = 1
|
|
19
|
+
TELEMETRY_DATASOURCE: str = "tb_cli_telemetry"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def get_ci_product_name() -> Optional[str]:
|
|
23
|
+
if _is_env_true("TB_DISABLE_CI_DETECTION"):
|
|
24
|
+
return None
|
|
25
|
+
|
|
26
|
+
CI_CHECKS: List[Tuple[str, Callable[[], bool]]] = [
|
|
27
|
+
("Azure pipelines", lambda: _is_env_true("TF_BUILD")),
|
|
28
|
+
("GitHub Actions", lambda: _is_env_true("GITHUB_ACTIONS")),
|
|
29
|
+
("Appveyor", lambda: _is_env_true("APPVEYOR")),
|
|
30
|
+
("Travis CI", lambda: _is_env_true("TRAVIS")),
|
|
31
|
+
("Circle CI", lambda: _is_env_true("CIRCLECI")),
|
|
32
|
+
("Amazon Web Services CodeBuild", lambda: _is_env_present(["CODEBUILD_BUILD_ID", "AWS_REGION"])),
|
|
33
|
+
("Jenkins", lambda: _is_env_present(["BUILD_ID", "BUILD_URL"])),
|
|
34
|
+
("Google Cloud Build", lambda: _is_env_present(["BUILD_ID", "PROJECT_ID"])),
|
|
35
|
+
("TeamCity", lambda: _is_env_present(["TEAMCITY_VERSION"])),
|
|
36
|
+
("JetBrains Space", lambda: _is_env_present(["JB_SPACE_API_URL"])),
|
|
37
|
+
("Generic CI", lambda: _is_env_true("CI")),
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
return next((check[0] for check in CI_CHECKS if check[1]()), None)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def is_ci_environment() -> bool:
|
|
44
|
+
ci_product: Optional[str] = get_ci_product_name()
|
|
45
|
+
return ci_product is not None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def silence_errors(f: Callable) -> Callable:
|
|
49
|
+
"""Decorator to silence all errors in the decorated
|
|
50
|
+
function.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
@functools.wraps(f)
|
|
54
|
+
def wrapper(*args, **kwargs) -> Any:
|
|
55
|
+
try:
|
|
56
|
+
return f(*args, **kwargs)
|
|
57
|
+
except Exception:
|
|
58
|
+
return None
|
|
59
|
+
|
|
60
|
+
return wrapper
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _is_env_true(env_var: str) -> bool:
|
|
64
|
+
"""Checks if `env_var` is `true` or `1`."""
|
|
65
|
+
return os.getenv(env_var, "").lower() in ("true", "1")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _is_env_present(envs: List[str]) -> bool:
|
|
69
|
+
"""Checks if all of the variables passed in `envs`
|
|
70
|
+
are defined (ie: not empty)
|
|
71
|
+
"""
|
|
72
|
+
return all(os.getenv(env_var, None) is not None for env_var in envs)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _hide_tokens(text: str) -> str:
|
|
76
|
+
"""Cuts any token in a way that they get unusable if leaked,
|
|
77
|
+
but we still can use them for debugging if needed.
|
|
78
|
+
"""
|
|
79
|
+
return re.sub(r"p\.ey[A-Za-z0-9-_\.]+", lambda s: f"{s[0][:10]}...{s[0][-10:]}", text)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class TelemetryHelper:
|
|
83
|
+
def __init__(self, tb_host: Optional[str] = None, max_enqueued_events: int = 5) -> None:
|
|
84
|
+
self.tb_host = tb_host or os.getenv("TB_CLI_TELEMETRY_HOST", "https://api.tinybird.co")
|
|
85
|
+
self.max_enqueued_events: int = max_enqueued_events
|
|
86
|
+
|
|
87
|
+
self.enabled: bool = True
|
|
88
|
+
self.events: List[Dict[str, Any]] = []
|
|
89
|
+
self.telemetry_token: Optional[str] = None
|
|
90
|
+
|
|
91
|
+
run_id = str(uuid.uuid4())
|
|
92
|
+
|
|
93
|
+
self._defaults: Dict[str, Any] = {
|
|
94
|
+
# Per-event values
|
|
95
|
+
"event": "<the event>",
|
|
96
|
+
"event_data": "<the event data>",
|
|
97
|
+
"timestamp": "<the timestamp>",
|
|
98
|
+
# Static values
|
|
99
|
+
"run_id": run_id,
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
self._threads: List[threading.Thread] = []
|
|
103
|
+
self.log(f"Telemetry initialized with run_id: {run_id}")
|
|
104
|
+
|
|
105
|
+
@silence_errors
|
|
106
|
+
def add_event(self, event: str, event_data: Dict[str, Any]) -> None:
|
|
107
|
+
if not self.enabled:
|
|
108
|
+
self.log("Helper is disabled")
|
|
109
|
+
return
|
|
110
|
+
|
|
111
|
+
if "x.y.z" in CURRENT_VERSION and not _is_env_true("TB_CLI_TELEMETRY_SEND_IN_LOCAL"):
|
|
112
|
+
self.log("Not sending events in local development mode")
|
|
113
|
+
return
|
|
114
|
+
|
|
115
|
+
# Let's save deep copies to not interfere with original objects
|
|
116
|
+
event_dict: Dict[str, Any] = deepcopy(self._defaults)
|
|
117
|
+
event_dict["event"] = event
|
|
118
|
+
event_dict["event_data"] = json.dumps(event_data)
|
|
119
|
+
event_dict["timestamp"] = datetime.utcnow().isoformat()
|
|
120
|
+
|
|
121
|
+
self.events.append(event_dict)
|
|
122
|
+
if len(self.events) >= self.max_enqueued_events:
|
|
123
|
+
self.flush()
|
|
124
|
+
|
|
125
|
+
@silence_errors
|
|
126
|
+
def flush(self, wait: bool = False) -> None:
|
|
127
|
+
if self.enabled and len(self.events) > 0:
|
|
128
|
+
# Take the ownership for the pending events.
|
|
129
|
+
#
|
|
130
|
+
# We need this because the proper flush() is done in
|
|
131
|
+
# a thread to avoid blocking the user and we could send
|
|
132
|
+
# the same event twice if we maintain the same list after
|
|
133
|
+
# during the sending.
|
|
134
|
+
|
|
135
|
+
events: List[Dict[str, Any]] = self.events
|
|
136
|
+
self.events = []
|
|
137
|
+
|
|
138
|
+
self.log(f"Flusing {len(events)} events in a new thread...")
|
|
139
|
+
thread: threading.Thread = threading.Thread(target=self._flush, args=[events])
|
|
140
|
+
self._threads.append(thread)
|
|
141
|
+
thread.start()
|
|
142
|
+
|
|
143
|
+
if wait:
|
|
144
|
+
for t in self._threads:
|
|
145
|
+
t.join()
|
|
146
|
+
if t.is_alive():
|
|
147
|
+
self.log(f"Couldn't wait for the end of the thread {t.name}")
|
|
148
|
+
self._threads.clear()
|
|
149
|
+
|
|
150
|
+
@silence_errors
|
|
151
|
+
def _flush(self, events: List[Dict[str, Any]]) -> None:
|
|
152
|
+
"""Actual flush. This is where we use HFI to ingest events."""
|
|
153
|
+
|
|
154
|
+
timeout: int
|
|
155
|
+
try:
|
|
156
|
+
timeout = int(os.getenv("TB_CLI_TELEMETRY_TIMEOUT", TELEMETRY_TIMEOUT))
|
|
157
|
+
timeout = max(TELEMETRY_TIMEOUT, timeout)
|
|
158
|
+
except ValueError:
|
|
159
|
+
timeout = TELEMETRY_TIMEOUT
|
|
160
|
+
|
|
161
|
+
if not self.telemetry_token:
|
|
162
|
+
self.telemetry_token = os.getenv("TB_CLI_TELEMETRY_TOKEN")
|
|
163
|
+
if self.telemetry_token:
|
|
164
|
+
self.log("Got telemetry token from environment TB_CLI_TELEMETRY_TOKEN")
|
|
165
|
+
|
|
166
|
+
with requests.Session() as session:
|
|
167
|
+
if not self.telemetry_token:
|
|
168
|
+
url: str = f"{self.tb_host}/v0/regions"
|
|
169
|
+
self.log(f"Requesting token from {url}...")
|
|
170
|
+
try:
|
|
171
|
+
r = session.get(url, timeout=timeout)
|
|
172
|
+
regions: List[Dict[str, Any]] = json.loads(r.content.decode())["regions"]
|
|
173
|
+
self.telemetry_token = next(
|
|
174
|
+
(r.get("telemetry_token", None) for r in regions if r["api_host"] == self.tb_host), None
|
|
175
|
+
)
|
|
176
|
+
if self.telemetry_token:
|
|
177
|
+
self.log(f"Got telemetry token from {url}")
|
|
178
|
+
except requests.exceptions.Timeout:
|
|
179
|
+
self.log(f"Disabling due to timeout after {timeout} seconds")
|
|
180
|
+
self.enabled = False
|
|
181
|
+
return
|
|
182
|
+
except Exception as ex:
|
|
183
|
+
self.log(str(ex))
|
|
184
|
+
|
|
185
|
+
if not self.telemetry_token:
|
|
186
|
+
self.log("Disabling due to lack of token")
|
|
187
|
+
self.enabled = False
|
|
188
|
+
return
|
|
189
|
+
|
|
190
|
+
self.log(f"token={self.telemetry_token}")
|
|
191
|
+
|
|
192
|
+
data: str = _hide_tokens("\n".join(json.dumps(e) for e in events))
|
|
193
|
+
|
|
194
|
+
# Note we don't use `wait` as this telemetry isn't a critical
|
|
195
|
+
# operation to support and we don't want to generate overhead
|
|
196
|
+
params: Dict[str, Any] = {"name": TELEMETRY_DATASOURCE, "token": self.telemetry_token}
|
|
197
|
+
url = f"{self.tb_host}/v0/events?{urlencode(params)}"
|
|
198
|
+
|
|
199
|
+
try:
|
|
200
|
+
self.log(f"Sending data to {url}...")
|
|
201
|
+
r = session.post(url, data=data, timeout=timeout)
|
|
202
|
+
except requests.exceptions.Timeout:
|
|
203
|
+
self.log(f"Disabling due to timeout after {timeout} seconds")
|
|
204
|
+
self.enabled = False
|
|
205
|
+
return
|
|
206
|
+
|
|
207
|
+
self.log(f"Received status {r.status_code}: {r.text}")
|
|
208
|
+
|
|
209
|
+
if r.status_code == 200 or r.status_code == 202:
|
|
210
|
+
self.log(f"Successfully sent {len(events)} events to {self.tb_host}")
|
|
211
|
+
self.events.clear()
|
|
212
|
+
return
|
|
213
|
+
|
|
214
|
+
if r.status_code in (403, 404):
|
|
215
|
+
self.log(f"Disabling due to {r.status_code} errors")
|
|
216
|
+
self.enabled = False
|
|
217
|
+
return
|
|
218
|
+
|
|
219
|
+
if r.status_code >= 500:
|
|
220
|
+
self.log(f"Disabling telemetry and discarding {len(events)} events")
|
|
221
|
+
self.enabled = False
|
|
222
|
+
|
|
223
|
+
@silence_errors
|
|
224
|
+
def log(self, msg: str) -> None:
|
|
225
|
+
"""Internal logging function to help with development and debugging."""
|
|
226
|
+
if not _is_env_true("TB_CLI_TELEMETRY_DEBUG"):
|
|
227
|
+
return
|
|
228
|
+
print(f"> Telemetry: {msg}") # noqa: T201
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
_helper_instance: Optional[TelemetryHelper] = None
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
@silence_errors
|
|
235
|
+
def init_telemetry() -> None:
|
|
236
|
+
"""Setups the telemetry helper with the config present in `config`.
|
|
237
|
+
If no config is provided, it tries to get it from the passed Click context.
|
|
238
|
+
|
|
239
|
+
We need to call this method any time we suspect the config changes any value.
|
|
240
|
+
"""
|
|
241
|
+
|
|
242
|
+
telemetry = _get_helper()
|
|
243
|
+
if telemetry:
|
|
244
|
+
telemetry.log("Initialized")
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
@silence_errors
|
|
248
|
+
def add_telemetry_event(event: str, **kw_event_data: Any) -> None:
|
|
249
|
+
"""Adds a new telemetry event."""
|
|
250
|
+
|
|
251
|
+
telemetry = _get_helper()
|
|
252
|
+
if not telemetry:
|
|
253
|
+
return
|
|
254
|
+
|
|
255
|
+
try:
|
|
256
|
+
telemetry.add_event(event, dict(**kw_event_data))
|
|
257
|
+
except Exception as ex:
|
|
258
|
+
telemetry.log(str(ex))
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
@silence_errors
|
|
262
|
+
def add_telemetry_sysinfo_event() -> None:
|
|
263
|
+
"""Collects system info and sends a `system_info` event
|
|
264
|
+
with the data.
|
|
265
|
+
"""
|
|
266
|
+
|
|
267
|
+
ci_product: Optional[str] = get_ci_product_name()
|
|
268
|
+
|
|
269
|
+
add_telemetry_event(
|
|
270
|
+
"system_info",
|
|
271
|
+
platform=platform.platform(),
|
|
272
|
+
system=platform.system(),
|
|
273
|
+
arch=platform.machine(),
|
|
274
|
+
processor=platform.processor(),
|
|
275
|
+
python_runtime=platform.python_implementation(),
|
|
276
|
+
python_version=platform.python_version(),
|
|
277
|
+
is_ci=ci_product is not None,
|
|
278
|
+
ci_product=ci_product,
|
|
279
|
+
cli_version=CURRENT_VERSION,
|
|
280
|
+
cli_args=sys.argv[1:] if len(sys.argv) > 1 else [],
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
@silence_errors
|
|
285
|
+
def flush_telemetry(wait: bool = False) -> None:
|
|
286
|
+
"""Flushes all pending telemetry events."""
|
|
287
|
+
|
|
288
|
+
telemetry = _get_helper()
|
|
289
|
+
if not telemetry:
|
|
290
|
+
return
|
|
291
|
+
|
|
292
|
+
try:
|
|
293
|
+
telemetry.flush(wait=wait)
|
|
294
|
+
except Exception as ex:
|
|
295
|
+
telemetry.log(str(ex))
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
@silence_errors
|
|
299
|
+
def _get_helper() -> Optional[TelemetryHelper]:
|
|
300
|
+
"""Returns the shared TelemetryHelper instance."""
|
|
301
|
+
|
|
302
|
+
if _is_env_true("TB_CLI_TELEMETRY_OPTOUT"):
|
|
303
|
+
return None
|
|
304
|
+
|
|
305
|
+
global _helper_instance
|
|
306
|
+
|
|
307
|
+
if not _helper_instance:
|
|
308
|
+
_helper_instance = TelemetryHelper()
|
|
309
|
+
|
|
310
|
+
return _helper_instance
|
|
@@ -10,11 +10,11 @@ import click
|
|
|
10
10
|
|
|
11
11
|
from tinybird.client import AuthNoTokenException
|
|
12
12
|
from tinybird.feedback_manager import FeedbackManager
|
|
13
|
-
from tinybird.
|
|
14
|
-
from tinybird.
|
|
15
|
-
from tinybird.
|
|
16
|
-
from tinybird.
|
|
17
|
-
from tinybird.
|
|
13
|
+
from tinybird.tb.modules.cli import cli
|
|
14
|
+
from tinybird.tb.modules.common import coro, create_tb_client, gather_with_concurrency
|
|
15
|
+
from tinybird.tb.modules.config import CLIConfig
|
|
16
|
+
from tinybird.tb.modules.exceptions import CLIException
|
|
17
|
+
from tinybird.tb.modules.tinyunit.tinyunit import (
|
|
18
18
|
TestSummaryResults,
|
|
19
19
|
generate_file,
|
|
20
20
|
parse_file,
|
|
@@ -9,7 +9,7 @@ from typing_extensions import override
|
|
|
9
9
|
|
|
10
10
|
from tinybird.client import TinyB
|
|
11
11
|
from tinybird.feedback_manager import FeedbackManager
|
|
12
|
-
from tinybird.
|
|
12
|
+
from tinybird.tb.modules.common import CLIException
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
@dataclass
|
|
@@ -8,13 +8,13 @@ from humanfriendly import parse_timespan
|
|
|
8
8
|
|
|
9
9
|
from tinybird.client import AuthNoTokenException, TinyB
|
|
10
10
|
from tinybird.feedback_manager import FeedbackManager
|
|
11
|
-
from tinybird.
|
|
12
|
-
from tinybird.
|
|
11
|
+
from tinybird.tb.modules.cli import cli
|
|
12
|
+
from tinybird.tb.modules.common import (
|
|
13
13
|
DoesNotExistException,
|
|
14
14
|
coro,
|
|
15
15
|
echo_safe_humanfriendly_tables_format_smart_table,
|
|
16
16
|
)
|
|
17
|
-
from tinybird.
|
|
17
|
+
from tinybird.tb.modules.exceptions import CLITokenException
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
@cli.group()
|
|
@@ -10,10 +10,9 @@ from click import Context
|
|
|
10
10
|
|
|
11
11
|
from tinybird.client import CanNotBeDeletedException, DoesNotExistException, TinyB
|
|
12
12
|
from tinybird.config import get_display_host
|
|
13
|
-
from tinybird.datafile import PipeTypes
|
|
14
13
|
from tinybird.feedback_manager import FeedbackManager
|
|
15
|
-
from tinybird.
|
|
16
|
-
from tinybird.
|
|
14
|
+
from tinybird.tb.modules.cli import cli
|
|
15
|
+
from tinybird.tb.modules.common import (
|
|
17
16
|
_get_workspace_plan_name,
|
|
18
17
|
ask_for_user_token,
|
|
19
18
|
check_user_token,
|
|
@@ -26,8 +25,9 @@ from tinybird.tb_cli_modules.common import (
|
|
|
26
25
|
print_current_workspace,
|
|
27
26
|
switch_workspace,
|
|
28
27
|
)
|
|
29
|
-
from tinybird.
|
|
30
|
-
from tinybird.
|
|
28
|
+
from tinybird.tb.modules.config import CLIConfig
|
|
29
|
+
from tinybird.tb.modules.datafile import PipeTypes
|
|
30
|
+
from tinybird.tb.modules.exceptions import CLIWorkspaceException
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
@cli.group()
|
|
@@ -13,16 +13,16 @@ from click import Context
|
|
|
13
13
|
from tinybird.client import TinyB
|
|
14
14
|
from tinybird.config import get_display_host
|
|
15
15
|
from tinybird.feedback_manager import FeedbackManager
|
|
16
|
-
from tinybird.
|
|
16
|
+
from tinybird.tb.modules.common import (
|
|
17
17
|
ask_for_user_token,
|
|
18
18
|
check_user_token,
|
|
19
19
|
coro,
|
|
20
20
|
echo_safe_humanfriendly_tables_format_smart_table,
|
|
21
21
|
get_current_workspace,
|
|
22
22
|
)
|
|
23
|
-
from tinybird.
|
|
24
|
-
from tinybird.
|
|
25
|
-
from tinybird.
|
|
23
|
+
from tinybird.tb.modules.config import CLIConfig
|
|
24
|
+
from tinybird.tb.modules.exceptions import CLIWorkspaceMembersException
|
|
25
|
+
from tinybird.tb.modules.workspace import workspace
|
|
26
26
|
|
|
27
27
|
ROLES = ["viewer", "guest", "admin"]
|
|
28
28
|
|
|
@@ -53,7 +53,6 @@ from tinybird.config import (
|
|
|
53
53
|
get_display_host,
|
|
54
54
|
write_config,
|
|
55
55
|
)
|
|
56
|
-
from tinybird.tb_cli_modules.table import format_table
|
|
57
56
|
|
|
58
57
|
if TYPE_CHECKING:
|
|
59
58
|
from tinybird.connectors import Connector
|
|
@@ -135,23 +134,6 @@ def echo_safe_humanfriendly_tables_format_smart_table(data: Iterable[Any], colum
|
|
|
135
134
|
raise exc
|
|
136
135
|
|
|
137
136
|
|
|
138
|
-
def echo_safe_format_table(data: Iterable[Any], columns) -> None:
|
|
139
|
-
"""
|
|
140
|
-
There is a bug in the humanfriendly library: it breaks to render the small table for small terminals
|
|
141
|
-
(`format_robust_table`) if we call format_smart_table with an empty dataset. This catches the error and prints
|
|
142
|
-
what we would call an empty "robust_table".
|
|
143
|
-
"""
|
|
144
|
-
try:
|
|
145
|
-
click.echo(format_table(data, columns))
|
|
146
|
-
except ValueError as exc:
|
|
147
|
-
if str(exc) == "max() arg is an empty sequence":
|
|
148
|
-
click.echo("------------")
|
|
149
|
-
click.echo("Empty")
|
|
150
|
-
click.echo("------------")
|
|
151
|
-
else:
|
|
152
|
-
raise exc
|
|
153
|
-
|
|
154
|
-
|
|
155
137
|
def normalize_datasource_name(s: str) -> str:
|
|
156
138
|
s = re.sub(r"[^0-9a-zA-Z_]", "_", s)
|
|
157
139
|
if s[0] in "0123456789":
|
|
@@ -171,7 +153,7 @@ def generate_datafile(
|
|
|
171
153
|
if not f.exists() or force:
|
|
172
154
|
with open(f"{f}", "w") as ds_file:
|
|
173
155
|
ds_file.write(datafile)
|
|
174
|
-
click.echo(FeedbackManager.
|
|
156
|
+
click.echo(FeedbackManager.success_generated_file(file=f, stem=datasource_name, filename=filename))
|
|
175
157
|
|
|
176
158
|
if data and (base / "fixtures").exists():
|
|
177
159
|
# Generating a fixture for Parquet files is not so trivial, since Parquet format
|
|
@@ -184,6 +166,7 @@ def generate_datafile(
|
|
|
184
166
|
newline = b"\n" # TODO: guess
|
|
185
167
|
with open(f, "wb") as fixture_file:
|
|
186
168
|
fixture_file.write(data[: data.rfind(newline)])
|
|
169
|
+
click.echo(FeedbackManager.success_generated_fixture(fixture=f))
|
|
187
170
|
else:
|
|
188
171
|
click.echo(FeedbackManager.error_file_already_exists(file=f))
|
|
189
172
|
|
|
@@ -1060,7 +1043,6 @@ def get_format_from_filename_or_url(filename_or_url: str) -> str:
|
|
|
1060
1043
|
|
|
1061
1044
|
async def push_data(
|
|
1062
1045
|
ctx: Context,
|
|
1063
|
-
client: TinyB,
|
|
1064
1046
|
datasource_name: str,
|
|
1065
1047
|
url,
|
|
1066
1048
|
connector: Optional[str],
|
|
@@ -1073,6 +1055,7 @@ async def push_data(
|
|
|
1073
1055
|
):
|
|
1074
1056
|
if url and type(url) is tuple:
|
|
1075
1057
|
url = url[0]
|
|
1058
|
+
client: TinyB = ctx.obj["client"]
|
|
1076
1059
|
|
|
1077
1060
|
if connector and sql:
|
|
1078
1061
|
load_connector_config(ctx, connector, False, check_uninstalled=False)
|
|
@@ -1108,7 +1091,7 @@ async def push_data(
|
|
|
1108
1091
|
cb.First = True # type: ignore[attr-defined]
|
|
1109
1092
|
cb.prev_done = 0 # type: ignore[attr-defined]
|
|
1110
1093
|
|
|
1111
|
-
click.echo(FeedbackManager.
|
|
1094
|
+
click.echo(FeedbackManager.info_starting_import_process())
|
|
1112
1095
|
|
|
1113
1096
|
if isinstance(url, list):
|
|
1114
1097
|
urls = url
|
|
@@ -1179,16 +1162,17 @@ async def push_data(
|
|
|
1179
1162
|
except Exception as e:
|
|
1180
1163
|
raise CLIException(FeedbackManager.error_exception(error=e))
|
|
1181
1164
|
else:
|
|
1165
|
+
click.echo(FeedbackManager.success_progress_blocks())
|
|
1182
1166
|
if mode == "append" and parser and parser != "clickhouse":
|
|
1183
1167
|
click.echo(FeedbackManager.success_appended_rows(appended_rows=appended_rows))
|
|
1184
1168
|
|
|
1169
|
+
click.echo(FeedbackManager.success_total_rows(datasource=datasource_name, total_rows=total_rows))
|
|
1170
|
+
|
|
1185
1171
|
if mode == "replace":
|
|
1186
1172
|
click.echo(FeedbackManager.success_replaced_datasource(datasource=datasource_name))
|
|
1187
1173
|
else:
|
|
1188
|
-
click.echo(FeedbackManager.
|
|
1189
|
-
|
|
1190
|
-
click.echo(FeedbackManager.success_progress_blocks())
|
|
1191
|
-
|
|
1174
|
+
click.echo(FeedbackManager.success_appended_datasource(datasource=datasource_name))
|
|
1175
|
+
click.echo(FeedbackManager.info_data_pushed(datasource=datasource_name))
|
|
1192
1176
|
finally:
|
|
1193
1177
|
try:
|
|
1194
1178
|
for url in urls:
|
|
@@ -338,14 +338,6 @@ class CLIConfig:
|
|
|
338
338
|
CLIConfig._projects[working_dir] = result
|
|
339
339
|
return result
|
|
340
340
|
|
|
341
|
-
@staticmethod
|
|
342
|
-
def get_llm_config(working_dir: Optional[str] = None) -> Dict[str, Any]:
|
|
343
|
-
return (
|
|
344
|
-
CLIConfig.get_project_config(working_dir)
|
|
345
|
-
.get("llms", {})
|
|
346
|
-
.get("openai", {"model": "gpt-4o-mini", "api_key": None})
|
|
347
|
-
)
|
|
348
|
-
|
|
349
341
|
@staticmethod
|
|
350
342
|
def reset() -> None:
|
|
351
343
|
CLIConfig._global = None
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
tinybird/__cli__.py,sha256=pgYsVLcqL16wtSn6KtKweNZYoYJdEksTgSvQAW7hH64,250
|
|
2
|
+
tinybird/check_pypi.py,sha256=RIhZcTg9dk5SFuvV3yVtFgBael_5_VYl4GzOZJojCfU,971
|
|
3
|
+
tinybird/client.py,sha256=nd97gD2-8Ap8yDonBcVwk9eXDAL43hmIYdo-Pse43RE,50738
|
|
4
|
+
tinybird/config.py,sha256=Z-BX9FrjgsLw1YwcCdF0IztLB97Zpc70VVPplO_pDSY,6089
|
|
5
|
+
tinybird/connectors.py,sha256=lkpVSUmSuViEZBa4QjTK7YmPHUop0a5UFoTrSmlVq6k,15244
|
|
6
|
+
tinybird/context.py,sha256=kutUQ0kCwparowI74_YLXx6wtTzGLRouJ6oGHVBPzBo,1291
|
|
7
|
+
tinybird/datatypes.py,sha256=IHyhZ86ib54Vnd1pbod9y2aS8DDvDKZm1HJGlThdbuQ,10460
|
|
8
|
+
tinybird/feedback_manager.py,sha256=qX8yrjoHUL4kjb_IKzLZPPfg4m54XWUZkdVOrv1ktpo,67814
|
|
9
|
+
tinybird/git_settings.py,sha256=XUL9ZUj59-ZVQJDYmMEq4UpnuuOuQOHGlNcX3JgQHjQ,3954
|
|
10
|
+
tinybird/sql.py,sha256=gfRKjdqEygcE1WOTeQ1QV2Jal8Jzl4RSX8fftu1KSEs,45825
|
|
11
|
+
tinybird/sql_template.py,sha256=IqYRfUxDYBCoOYjqqvn--_8QXLv9FSRnJ0bInx7q1Xs,93051
|
|
12
|
+
tinybird/sql_template_fmt.py,sha256=1z-PuqSZXtzso8Z_mPqUc-NxIxUrNUcVIPezNieZk-M,10196
|
|
13
|
+
tinybird/sql_toolset.py,sha256=xS_yD5N_TZT5d4uPcXdeIYX4GQPz7-7wHywMGdfgqcM,13794
|
|
14
|
+
tinybird/syncasync.py,sha256=fAvq0qkRgqXqXMKwbY2iJNYqLT_r6mDsh1MRpGKrdRU,27763
|
|
15
|
+
tinybird/tornado_template.py,sha256=o2HguxrL1Evnt8o3IvrsI8Zm6JtRQ3zhLJKf1XyR3SQ,41965
|
|
16
|
+
tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
|
|
17
|
+
tinybird/ch_utils/engine.py,sha256=OXkBhlzGjZotjD0vaT-rFIbSGV4tpiHxE8qO_ip0SyQ,40454
|
|
18
|
+
tinybird/tb/cli.py,sha256=6Lu3wsCNepAxjJCWy4c6RhVPArBtm8TlUcSxX--TsBo,783
|
|
19
|
+
tinybird/tb/modules/auth.py,sha256=hynZ-Temot8YBsySUWKSFzZlYadtFPxG3o6lCSu1n6E,9018
|
|
20
|
+
tinybird/tb/modules/branch.py,sha256=X3Ks8SnIeZLgU6wU6tY0-PfLmIJdGho6wlkmC4o4nOg,39119
|
|
21
|
+
tinybird/tb/modules/build.py,sha256=CAZq1BxbGM-uaoGanVEtZT3YwzWN_jJ6Fzgrw8Nwi3g,7798
|
|
22
|
+
tinybird/tb/modules/cicd.py,sha256=mIMU1gNGpN3f5K3rBYN9rFoOT3RogDxIs_zB3LY-iO4,7463
|
|
23
|
+
tinybird/tb/modules/cli.py,sha256=Kz55MfNv01lcYow2TNRJuGZvUZ_2Sa-jfNqeYVmhuJU,61968
|
|
24
|
+
tinybird/tb/modules/common.py,sha256=KBuSgVKqyUE4OVgMqD7BXro346IxcSNkKCf_I1_D9fw,79086
|
|
25
|
+
tinybird/tb/modules/config.py,sha256=ppWvACHrSLkb5hOoQLYNby2w8jR76-8Kx2NBCst7ntQ,11760
|
|
26
|
+
tinybird/tb/modules/connection.py,sha256=ZSqBGoRiJedjHKEyB_fr1ybucOHtaad8d7uqGa2Q92M,28668
|
|
27
|
+
tinybird/tb/modules/create.py,sha256=E5hs03kDdJj7n0_6H3loUOcfNv71Jbks0EBu9Kdre4Q,6898
|
|
28
|
+
tinybird/tb/modules/datafile.py,sha256=16MTuwXqsruhf_UoniDQSHetMIGeFbiUtZcLDJcqoAk,253563
|
|
29
|
+
tinybird/tb/modules/datasource.py,sha256=Wkxj6QBtTKCgl43JFwMWafEHYnnQjGZZHoL2SbuC3KQ,35811
|
|
30
|
+
tinybird/tb/modules/exceptions.py,sha256=4A2sSjCEqKUMqpP3WI00zouCWW4uLaghXXLZBSw04mY,3363
|
|
31
|
+
tinybird/tb/modules/fmt.py,sha256=ZoYLUkyExudbUr-Phy-I2riPBdQd1NXwA1y-XbSEFB8,3385
|
|
32
|
+
tinybird/tb/modules/job.py,sha256=eoBVyA24lYIPonU88Jn7FF9hBKz1kScy9_w_oWreuc4,2952
|
|
33
|
+
tinybird/tb/modules/llm.py,sha256=D6ShlqCorZpRLjE5srI7Ws4VUTH6kSotTi88CvUwoGw,2446
|
|
34
|
+
tinybird/tb/modules/local.py,sha256=nCaN05ntqIGFbBqyhRwpwdLpcYwsx6Vvp0atU_KKaTo,5893
|
|
35
|
+
tinybird/tb/modules/mock.py,sha256=dYxm8_1zXGZnf4X8OYmG9hK1I-csHakD64Hag8mAn4Q,2069
|
|
36
|
+
tinybird/tb/modules/pipe.py,sha256=LIiEK3zUmLcu24zgHy8qWvDD4yQjwnb--AZ1vbs0554,30984
|
|
37
|
+
tinybird/tb/modules/prompts.py,sha256=12Abh1kZItxMa7r3WbTi_TgTYtlV30HcaG_JiodafIA,7255
|
|
38
|
+
tinybird/tb/modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
39
|
+
tinybird/tb/modules/table.py,sha256=hG-PRDVuFp2uph41WpoLRV1yjp3RI2fi_iGGiI0rdxU,7695
|
|
40
|
+
tinybird/tb/modules/tag.py,sha256=1qQWyk1p3Btv3LzM8VbJG-k7x2-pFuAlYCg3QL6QewI,3480
|
|
41
|
+
tinybird/tb/modules/telemetry.py,sha256=iEGnMuCuNhvF6ln__j6X9MSTwL_0Hm-GgFHHHvhfknk,10466
|
|
42
|
+
tinybird/tb/modules/test.py,sha256=psINFpSYT1eGgy32-_4q6CJ7LOcdwBpAfasMA0_tNOU,4330
|
|
43
|
+
tinybird/tb/modules/token.py,sha256=r0oeG1RpOOzHtqbUaHBiOmhE55HfNIvReAAWyKl9fJg,12695
|
|
44
|
+
tinybird/tb/modules/workspace.py,sha256=alZFDKy_qaZk5TIe6gduBeNgpp4h5jp6RDj1SwP5ylg,10909
|
|
45
|
+
tinybird/tb/modules/workspace_members.py,sha256=08W0onEYkKLEC5TkAI07cxN9XSquEm7HnL7OkHAVDjo,8715
|
|
46
|
+
tinybird/tb/modules/tinyunit/tinyunit.py,sha256=IkjRCvb8HnNEE84rtl0I1b9gQVpE_zCE8MvFFet51sg,11716
|
|
47
|
+
tinybird/tb/modules/tinyunit/tinyunit_lib.py,sha256=hGh1ZaXC1af7rKnX7222urkj0QJMhMWclqMy59dOqwE,1922
|
|
48
|
+
tinybird/tb_cli_modules/cicd.py,sha256=0lMkb6CVOFZl5HOwgY8mK4T4mgI7O8335UngLXtCc-c,13851
|
|
49
|
+
tinybird/tb_cli_modules/common.py,sha256=JJwkLUcTbTonf1NWe4tUeenQGMr82d2152hnSs6k6Zc,78706
|
|
50
|
+
tinybird/tb_cli_modules/config.py,sha256=6NTgIdwf0X132A1j6G_YrdPep87ymZ9b5pABabKLzh4,11484
|
|
51
|
+
tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
|
|
52
|
+
tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
53
|
+
tinybird/tb_cli_modules/telemetry.py,sha256=iEGnMuCuNhvF6ln__j6X9MSTwL_0Hm-GgFHHHvhfknk,10466
|
|
54
|
+
tinybird-0.0.1.dev6.dist-info/METADATA,sha256=wgXkBp0lUmzhrrDmTwxzy0FKVfmEIGfZ6no4CQF3uQc,2404
|
|
55
|
+
tinybird-0.0.1.dev6.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
|
|
56
|
+
tinybird-0.0.1.dev6.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
|
|
57
|
+
tinybird-0.0.1.dev6.dist-info/top_level.txt,sha256=pgw6AzERHBcW3YTi2PW4arjxLkulk2msOz_SomfOEuc,45
|
|
58
|
+
tinybird-0.0.1.dev6.dist-info/RECORD,,
|
tinybird/tb_cli.py
DELETED
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import sys
|
|
3
|
-
|
|
4
|
-
if sys.platform == "win32":
|
|
5
|
-
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
|
6
|
-
|
|
7
|
-
import tinybird.tb_cli_modules.auth
|
|
8
|
-
import tinybird.tb_cli_modules.branch
|
|
9
|
-
import tinybird.tb_cli_modules.build
|
|
10
|
-
import tinybird.tb_cli_modules.cli
|
|
11
|
-
import tinybird.tb_cli_modules.common
|
|
12
|
-
import tinybird.tb_cli_modules.connection
|
|
13
|
-
import tinybird.tb_cli_modules.create
|
|
14
|
-
import tinybird.tb_cli_modules.datasource
|
|
15
|
-
import tinybird.tb_cli_modules.fmt
|
|
16
|
-
import tinybird.tb_cli_modules.job
|
|
17
|
-
import tinybird.tb_cli_modules.mock
|
|
18
|
-
import tinybird.tb_cli_modules.pipe
|
|
19
|
-
import tinybird.tb_cli_modules.tag
|
|
20
|
-
import tinybird.tb_cli_modules.test
|
|
21
|
-
import tinybird.tb_cli_modules.token
|
|
22
|
-
import tinybird.tb_cli_modules.workspace
|
|
23
|
-
import tinybird.tb_cli_modules.workspace_members
|
|
24
|
-
|
|
25
|
-
cli = tinybird.tb_cli_modules.cli.cli
|
|
26
|
-
|
|
27
|
-
if __name__ == "__main__":
|
|
28
|
-
cli()
|