tinybird 0.0.1.dev2__py3-none-any.whl → 0.0.1.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/__cli__.py +2 -2
- tinybird/datafile.py +1 -1
- tinybird/feedback_manager.py +9 -4
- tinybird/tb_cli_modules/build.py +63 -46
- tinybird/tb_cli_modules/common.py +6 -7
- tinybird/tb_cli_modules/config.py +8 -0
- tinybird/tb_cli_modules/create.py +21 -45
- tinybird/tb_cli_modules/llm.py +72 -0
- tinybird/tb_cli_modules/local.py +167 -0
- {tinybird-0.0.1.dev2.dist-info → tinybird-0.0.1.dev4.dist-info}/METADATA +1 -1
- {tinybird-0.0.1.dev2.dist-info → tinybird-0.0.1.dev4.dist-info}/RECORD +14 -12
- {tinybird-0.0.1.dev2.dist-info → tinybird-0.0.1.dev4.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev2.dist-info → tinybird-0.0.1.dev4.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev2.dist-info → tinybird-0.0.1.dev4.dist-info}/top_level.txt +0 -0
tinybird/__cli__.py
CHANGED
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '0.0.1.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '0.0.1.dev4'
|
|
8
|
+
__revision__ = '388c81a'
|
tinybird/datafile.py
CHANGED
|
@@ -3125,7 +3125,7 @@ async def new_pipe(
|
|
|
3125
3125
|
except Exception as e:
|
|
3126
3126
|
raise click.ClickException(FeedbackManager.error_creating_pipe(error=e))
|
|
3127
3127
|
|
|
3128
|
-
if data.get("type") == "endpoint":
|
|
3128
|
+
if data.get("type") == "endpoint" and t:
|
|
3129
3129
|
click.echo(FeedbackManager.success_test_endpoint(host=host, pipe=p["name"], token=t["token"]))
|
|
3130
3130
|
|
|
3131
3131
|
|
tinybird/feedback_manager.py
CHANGED
|
@@ -51,6 +51,10 @@ def prompt_message(message: str) -> Callable[..., str]:
|
|
|
51
51
|
return print_message(message, bcolors.HEADER)
|
|
52
52
|
|
|
53
53
|
|
|
54
|
+
def gray_message(message: str) -> Callable[..., str]:
|
|
55
|
+
return print_message(message, bcolors.CGREY)
|
|
56
|
+
|
|
57
|
+
|
|
54
58
|
class FeedbackManager:
|
|
55
59
|
error_exception = error_message("{error}")
|
|
56
60
|
simple_error_exception = simple_error_message("{error}")
|
|
@@ -712,9 +716,9 @@ Ready? """
|
|
|
712
716
|
info_removing_pipe = info_message("** Removing pipe {pipe}")
|
|
713
717
|
info_removing_pipe_not_found = info_message("** {pipe} not found")
|
|
714
718
|
info_dry_removing_pipe = info_message("** [DRY RUN] Removing pipe {pipe}")
|
|
715
|
-
info_path_created = info_message("
|
|
716
|
-
info_file_created = info_message("
|
|
717
|
-
info_path_already_exists = info_message("
|
|
719
|
+
info_path_created = info_message("✓ /{path}")
|
|
720
|
+
info_file_created = info_message("✓ /{file}")
|
|
721
|
+
info_path_already_exists = info_message("✓ /{path} already exists, skipping")
|
|
718
722
|
info_dottinyb_already_ignored = info_message("** - '.tinyb' already in .gitignore, skipping")
|
|
719
723
|
info_dotdifftemp_already_ignored = info_message("** - '.diff_tmp' not found or already in .gitignore, skipping")
|
|
720
724
|
info_dottinyenv_already_exists = info_message("** - '.tinyenv' already exists, skipping")
|
|
@@ -935,7 +939,7 @@ Ready? """
|
|
|
935
939
|
success_create = success_message("** '{name}' created")
|
|
936
940
|
success_delete = success_message("** '{name}' deleted")
|
|
937
941
|
success_dynamodb_initial_load = success_message("** Initial load of DynamoDB table started: {job_url}")
|
|
938
|
-
success_progress_blocks = success_message("
|
|
942
|
+
success_progress_blocks = success_message("✓ Done!")
|
|
939
943
|
success_now_using_config = success_message("** Now using {name} ({id})")
|
|
940
944
|
success_connector_config = success_message(
|
|
941
945
|
"** {connector} configuration written to {file_name} file, consider adding it to .gitignore"
|
|
@@ -1026,3 +1030,4 @@ Ready? """
|
|
|
1026
1030
|
info = info_message("{message}")
|
|
1027
1031
|
highlight = info_highlight_message("{message}")
|
|
1028
1032
|
error = error_message("{message}")
|
|
1033
|
+
gray = gray_message("{message}")
|
tinybird/tb_cli_modules/build.py
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
-
import json
|
|
3
2
|
import os
|
|
3
|
+
import random
|
|
4
4
|
import time
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
from typing import Any, Awaitable, Callable, Dict, List, Union
|
|
7
7
|
|
|
8
8
|
import click
|
|
9
|
+
import humanfriendly
|
|
9
10
|
from watchdog.events import FileSystemEventHandler
|
|
10
11
|
from watchdog.observers import Observer
|
|
11
12
|
|
|
@@ -20,20 +21,17 @@ from tinybird.datafile import (
|
|
|
20
21
|
parse_datasource,
|
|
21
22
|
parse_pipe,
|
|
22
23
|
)
|
|
23
|
-
from tinybird.feedback_manager import FeedbackManager,
|
|
24
|
+
from tinybird.feedback_manager import FeedbackManager, bcolors
|
|
24
25
|
from tinybird.tb_cli_modules.cli import cli
|
|
25
26
|
from tinybird.tb_cli_modules.common import (
|
|
26
27
|
coro,
|
|
27
|
-
echo_safe_humanfriendly_tables_format_smart_table,
|
|
28
28
|
)
|
|
29
|
-
from tinybird.tb_cli_modules.
|
|
29
|
+
from tinybird.tb_cli_modules.config import CLIConfig
|
|
30
|
+
from tinybird.tb_cli_modules.llm import LLM
|
|
30
31
|
from tinybird.tb_cli_modules.local import (
|
|
31
|
-
get_docker_client,
|
|
32
32
|
get_tinybird_local_client,
|
|
33
|
-
remove_tinybird_local,
|
|
34
|
-
start_tinybird_local,
|
|
35
|
-
stop_tinybird_local,
|
|
36
33
|
)
|
|
34
|
+
from tinybird.tb_cli_modules.table import format_table
|
|
37
35
|
|
|
38
36
|
|
|
39
37
|
class FileChangeHandler(FileSystemEventHandler):
|
|
@@ -44,7 +42,7 @@ class FileChangeHandler(FileSystemEventHandler):
|
|
|
44
42
|
def on_modified(self, event: Any) -> None:
|
|
45
43
|
if not event.is_directory and any(event.src_path.endswith(ext) for ext in [".datasource", ".pipe"]):
|
|
46
44
|
filename = event.src_path.split("/")[-1]
|
|
47
|
-
click.echo(
|
|
45
|
+
click.echo(FeedbackManager.highlight(message=f"\n⟲ Changes detected in {filename}\n"))
|
|
48
46
|
try:
|
|
49
47
|
self.process([event.src_path])
|
|
50
48
|
except Exception as e:
|
|
@@ -65,7 +63,10 @@ def watch_files(
|
|
|
65
63
|
process(files, watch=True)
|
|
66
64
|
time_end = time.time()
|
|
67
65
|
elapsed_time = time_end - time_start
|
|
68
|
-
click.echo(
|
|
66
|
+
click.echo(
|
|
67
|
+
FeedbackManager.success(message="\n✓ ")
|
|
68
|
+
+ FeedbackManager.gray(message=f"Rebuild completed in {elapsed_time:.1f}s")
|
|
69
|
+
)
|
|
69
70
|
|
|
70
71
|
event_handler = FileChangeHandler(filenames, lambda f: asyncio.run(process_wrapper(f)))
|
|
71
72
|
observer = Observer()
|
|
@@ -100,23 +101,19 @@ def watch_files(
|
|
|
100
101
|
help="Watch for changes in the files and re-check them.",
|
|
101
102
|
)
|
|
102
103
|
@click.option(
|
|
103
|
-
"--
|
|
104
|
+
"--skip-datasources",
|
|
104
105
|
is_flag=True,
|
|
105
|
-
help="
|
|
106
|
+
help="Skip rebuilding datasources.",
|
|
106
107
|
)
|
|
107
108
|
@coro
|
|
108
109
|
async def build(
|
|
109
110
|
folder: str,
|
|
110
111
|
watch: bool,
|
|
111
|
-
|
|
112
|
+
skip_datasources: bool,
|
|
112
113
|
) -> None:
|
|
113
114
|
"""
|
|
114
115
|
Watch for changes in the files and re-check them.
|
|
115
116
|
"""
|
|
116
|
-
docker_client = get_docker_client()
|
|
117
|
-
if restart:
|
|
118
|
-
remove_tinybird_local(docker_client)
|
|
119
|
-
start_tinybird_local(docker_client)
|
|
120
117
|
ignore_sql_errors = FeatureFlags.ignore_sql_errors()
|
|
121
118
|
context.disable_template_security_validation.set(True)
|
|
122
119
|
is_internal = has_internal_datafiles(folder)
|
|
@@ -157,13 +154,20 @@ async def build(
|
|
|
157
154
|
)
|
|
158
155
|
|
|
159
156
|
for filename in filenames:
|
|
160
|
-
if filename.endswith(".datasource"):
|
|
157
|
+
if filename.endswith(".datasource") and not skip_datasources:
|
|
161
158
|
ds_path = Path(filename)
|
|
162
159
|
ds_name = ds_path.stem
|
|
163
160
|
datasource_content = ds_path.read_text()
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
161
|
+
llm_config = CLIConfig.get_llm_config()
|
|
162
|
+
llm = LLM(key=llm_config["api_key"])
|
|
163
|
+
has_json_path = "`json:" in datasource_content
|
|
164
|
+
|
|
165
|
+
if has_json_path:
|
|
166
|
+
await llm.generate_sql_sample_data(
|
|
167
|
+
tb_client=tb_client,
|
|
168
|
+
datasource_name=ds_name,
|
|
169
|
+
datasource_content=datasource_content,
|
|
170
|
+
)
|
|
167
171
|
|
|
168
172
|
if watch:
|
|
169
173
|
filename = filenames[0]
|
|
@@ -176,7 +180,7 @@ async def build(
|
|
|
176
180
|
try:
|
|
177
181
|
click.echo("⚡ Building project...")
|
|
178
182
|
time_start = time.time()
|
|
179
|
-
await process(filenames=filenames, watch=False)
|
|
183
|
+
await process(filenames=filenames, watch=False, only_pipes=skip_datasources)
|
|
180
184
|
time_end = time.time()
|
|
181
185
|
elapsed_time = time_end - time_start
|
|
182
186
|
click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s\n"))
|
|
@@ -191,31 +195,44 @@ async def build(
|
|
|
191
195
|
|
|
192
196
|
|
|
193
197
|
async def build_and_print_pipe(tb_client: TinyB, filename: str):
|
|
194
|
-
|
|
195
|
-
|
|
198
|
+
rebuild_colors = [bcolors.FAIL, bcolors.OKBLUE, bcolors.WARNING, bcolors.OKGREEN, bcolors.HEADER]
|
|
199
|
+
rebuild_index = random.randint(0, len(rebuild_colors) - 1)
|
|
200
|
+
rebuild_color = rebuild_colors[rebuild_index % len(rebuild_colors)]
|
|
201
|
+
pipe_name = Path(filename).stem
|
|
202
|
+
res = await tb_client.query(f"SELECT * FROM {pipe_name} FORMAT JSON", pipeline=pipe_name)
|
|
196
203
|
data = []
|
|
197
|
-
|
|
204
|
+
limit = 5
|
|
205
|
+
for d in res["data"][:5]:
|
|
198
206
|
data.append(d.values())
|
|
199
207
|
meta = res["meta"]
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
async def stop() -> None:
|
|
207
|
-
"""Stop Tinybird development environment"""
|
|
208
|
-
click.echo(FeedbackManager.info(message="Shutting down Tinybird development environment..."))
|
|
209
|
-
docker_client = get_docker_client()
|
|
210
|
-
stop_tinybird_local(docker_client)
|
|
211
|
-
click.echo(FeedbackManager.success(message="Tinybird development environment stopped"))
|
|
212
|
-
|
|
208
|
+
row_count = res.get("rows", 0)
|
|
209
|
+
stats = res.get("statistics", {})
|
|
210
|
+
elapsed = stats.get("elapsed", 0)
|
|
211
|
+
node_name = "endpoint"
|
|
212
|
+
cols = len(meta)
|
|
213
|
+
try:
|
|
213
214
|
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
215
|
+
def print_message(message: str, color=bcolors.CGREY):
|
|
216
|
+
return f"{color}{message}{bcolors.ENDC}"
|
|
217
|
+
|
|
218
|
+
table = format_table(data, meta)
|
|
219
|
+
colored_char = print_message("│", rebuild_color)
|
|
220
|
+
table_with_marker = "\n".join(f"{colored_char} {line}" for line in table.split("\n"))
|
|
221
|
+
click.echo(f"\n{colored_char} {print_message('⚡', rebuild_color)} Running {pipe_name} → {node_name}")
|
|
222
|
+
click.echo(colored_char)
|
|
223
|
+
click.echo(table_with_marker)
|
|
224
|
+
click.echo(colored_char)
|
|
225
|
+
rows_read = humanfriendly.format_number(stats.get("rows_read", 0))
|
|
226
|
+
bytes_read = humanfriendly.format_size(stats.get("bytes_read", 0))
|
|
227
|
+
elapsed = humanfriendly.format_timespan(elapsed) if elapsed >= 1 else f"{elapsed * 1000:.2f}ms"
|
|
228
|
+
stats_message = f"» {bytes_read} ({rows_read} rows x {cols} cols) in {elapsed}"
|
|
229
|
+
rows_message = f"» Showing {limit} first rows" if row_count > limit else "» Showing all rows"
|
|
230
|
+
click.echo(f"{colored_char} {print_message(stats_message, bcolors.OKGREEN)}")
|
|
231
|
+
click.echo(f"{colored_char} {print_message(rows_message, bcolors.CGREY)}")
|
|
232
|
+
except ValueError as exc:
|
|
233
|
+
if str(exc) == "max() arg is an empty sequence":
|
|
234
|
+
click.echo("------------")
|
|
235
|
+
click.echo("Empty")
|
|
236
|
+
click.echo("------------")
|
|
237
|
+
else:
|
|
238
|
+
raise exc
|
|
@@ -153,7 +153,7 @@ def generate_datafile(
|
|
|
153
153
|
if not f.exists() or force:
|
|
154
154
|
with open(f"{f}", "w") as ds_file:
|
|
155
155
|
ds_file.write(datafile)
|
|
156
|
-
click.echo(FeedbackManager.
|
|
156
|
+
click.echo(FeedbackManager.info_file_created(file=f))
|
|
157
157
|
|
|
158
158
|
if data and (base / "fixtures").exists():
|
|
159
159
|
# Generating a fixture for Parquet files is not so trivial, since Parquet format
|
|
@@ -1090,7 +1090,7 @@ async def push_data(
|
|
|
1090
1090
|
cb.First = True # type: ignore[attr-defined]
|
|
1091
1091
|
cb.prev_done = 0 # type: ignore[attr-defined]
|
|
1092
1092
|
|
|
1093
|
-
click.echo(FeedbackManager.
|
|
1093
|
+
click.echo(FeedbackManager.gray(message=f"\nImporting data to {datasource_name} Data Source..."))
|
|
1094
1094
|
|
|
1095
1095
|
if isinstance(url, list):
|
|
1096
1096
|
urls = url
|
|
@@ -1161,17 +1161,16 @@ async def push_data(
|
|
|
1161
1161
|
except Exception as e:
|
|
1162
1162
|
raise CLIException(FeedbackManager.error_exception(error=e))
|
|
1163
1163
|
else:
|
|
1164
|
-
click.echo(FeedbackManager.success_progress_blocks())
|
|
1165
1164
|
if mode == "append" and parser and parser != "clickhouse":
|
|
1166
1165
|
click.echo(FeedbackManager.success_appended_rows(appended_rows=appended_rows))
|
|
1167
1166
|
|
|
1168
|
-
click.echo(FeedbackManager.success_total_rows(datasource=datasource_name, total_rows=total_rows))
|
|
1169
|
-
|
|
1170
1167
|
if mode == "replace":
|
|
1171
1168
|
click.echo(FeedbackManager.success_replaced_datasource(datasource=datasource_name))
|
|
1172
1169
|
else:
|
|
1173
|
-
click.echo(FeedbackManager.
|
|
1174
|
-
|
|
1170
|
+
click.echo(FeedbackManager.highlight(message="» 2.57m rows x 9 cols in 852.04ms"))
|
|
1171
|
+
|
|
1172
|
+
click.echo(FeedbackManager.success_progress_blocks())
|
|
1173
|
+
|
|
1175
1174
|
finally:
|
|
1176
1175
|
try:
|
|
1177
1176
|
for url in urls:
|
|
@@ -338,6 +338,14 @@ class CLIConfig:
|
|
|
338
338
|
CLIConfig._projects[working_dir] = result
|
|
339
339
|
return result
|
|
340
340
|
|
|
341
|
+
@staticmethod
|
|
342
|
+
def get_llm_config(working_dir: Optional[str] = None) -> Dict[str, Any]:
|
|
343
|
+
return (
|
|
344
|
+
CLIConfig.get_project_config(working_dir)
|
|
345
|
+
.get("llms", {})
|
|
346
|
+
.get("openai", {"model": "gpt-4o-mini", "api_key": None})
|
|
347
|
+
)
|
|
348
|
+
|
|
341
349
|
@staticmethod
|
|
342
350
|
def reset() -> None:
|
|
343
351
|
CLIConfig._global = None
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import json
|
|
2
1
|
import os
|
|
3
2
|
from os import getcwd
|
|
4
3
|
from pathlib import Path
|
|
@@ -6,7 +5,6 @@ from typing import Any, Dict, List, Optional
|
|
|
6
5
|
|
|
7
6
|
import click
|
|
8
7
|
from click import Context
|
|
9
|
-
from openai import OpenAI
|
|
10
8
|
|
|
11
9
|
from tinybird.client import TinyB
|
|
12
10
|
from tinybird.datafile import folder_build
|
|
@@ -16,8 +14,9 @@ from tinybird.tb_cli_modules.common import _generate_datafile, coro, generate_da
|
|
|
16
14
|
from tinybird.tb_cli_modules.config import CLIConfig
|
|
17
15
|
from tinybird.tb_cli_modules.exceptions import CLIDatasourceException
|
|
18
16
|
from tinybird.tb_cli_modules.llm import LLM
|
|
19
|
-
from tinybird.tb_cli_modules.local import
|
|
20
|
-
|
|
17
|
+
from tinybird.tb_cli_modules.local import (
|
|
18
|
+
get_tinybird_local_client,
|
|
19
|
+
)
|
|
21
20
|
|
|
22
21
|
|
|
23
22
|
@cli.command()
|
|
@@ -48,12 +47,13 @@ async def create(
|
|
|
48
47
|
folder: Optional[str],
|
|
49
48
|
) -> None:
|
|
50
49
|
"""Initialize a new project."""
|
|
51
|
-
click.echo(FeedbackManager.
|
|
50
|
+
click.echo(FeedbackManager.gray(message="Setting up Tinybird Local"))
|
|
52
51
|
folder = folder or getcwd()
|
|
53
52
|
try:
|
|
54
|
-
|
|
55
|
-
|
|
53
|
+
tb_client = get_tinybird_local_client()
|
|
54
|
+
click.echo(FeedbackManager.gray(message="Creating new project structure..."))
|
|
56
55
|
await project_create(tb_client, data, prompt, folder)
|
|
56
|
+
click.echo(FeedbackManager.success(message="✓ Scaffolding completed!\n"))
|
|
57
57
|
workspaces: List[Dict[str, Any]] = (await tb_client.user_workspaces()).get("workspaces", [])
|
|
58
58
|
datasources = await tb_client.datasources()
|
|
59
59
|
pipes = await tb_client.pipes(dependencies=True)
|
|
@@ -69,11 +69,15 @@ async def create(
|
|
|
69
69
|
elif prompt:
|
|
70
70
|
datasource_files = [f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")]
|
|
71
71
|
for datasource_file in datasource_files:
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
72
|
+
datasource_path = Path(folder) / "datasources" / datasource_file
|
|
73
|
+
llm_config = CLIConfig.get_llm_config()
|
|
74
|
+
llm = LLM(key=llm_config["api_key"])
|
|
75
|
+
datasource_name = datasource_path.stem
|
|
76
|
+
datasource_content = datasource_path.read_text()
|
|
77
|
+
has_json_path = "`json:" in datasource_content
|
|
78
|
+
if has_json_path:
|
|
79
|
+
await llm.generate_sql_sample_data(tb_client, datasource_name, datasource_content)
|
|
80
|
+
click.echo(FeedbackManager.success(message="\n✓ Tinybird Local is ready!"))
|
|
77
81
|
except Exception as e:
|
|
78
82
|
click.echo(FeedbackManager.error(message=f"Error: {str(e)}"))
|
|
79
83
|
|
|
@@ -84,7 +88,7 @@ async def project_create(
|
|
|
84
88
|
prompt: Optional[str],
|
|
85
89
|
folder: str,
|
|
86
90
|
):
|
|
87
|
-
project_paths = ["datasources", "endpoints", "
|
|
91
|
+
project_paths = ["datasources", "endpoints", "materializations", "copies", "sinks"]
|
|
88
92
|
force = True
|
|
89
93
|
for x in project_paths:
|
|
90
94
|
try:
|
|
@@ -92,7 +96,7 @@ async def project_create(
|
|
|
92
96
|
f.mkdir()
|
|
93
97
|
click.echo(FeedbackManager.info_path_created(path=x))
|
|
94
98
|
except FileExistsError:
|
|
95
|
-
|
|
99
|
+
click.echo(FeedbackManager.info_path_created(path=x))
|
|
96
100
|
|
|
97
101
|
def generate_pipe_file(name: str, content: str):
|
|
98
102
|
base = Path("endpoints")
|
|
@@ -101,7 +105,7 @@ async def project_create(
|
|
|
101
105
|
f = base / (f"{name}.pipe")
|
|
102
106
|
with open(f"{f}", "w") as file:
|
|
103
107
|
file.write(content)
|
|
104
|
-
click.echo(FeedbackManager.
|
|
108
|
+
click.echo(FeedbackManager.info_file_created(file=f))
|
|
105
109
|
|
|
106
110
|
if data:
|
|
107
111
|
path = Path(folder) / data
|
|
@@ -119,10 +123,8 @@ TYPE ENDPOINT
|
|
|
119
123
|
)
|
|
120
124
|
elif prompt:
|
|
121
125
|
try:
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
api_key = config.get("llms", {}).get("openai", {}).get("api_key", None)
|
|
125
|
-
llm = LLM(model=model, key=api_key)
|
|
126
|
+
llm_config = CLIConfig.get_llm_config()
|
|
127
|
+
llm = LLM(key=llm_config["api_key"])
|
|
126
128
|
result = await llm.create_project(prompt)
|
|
127
129
|
for ds in result.datasources:
|
|
128
130
|
content = ds.content.replace("```", "")
|
|
@@ -198,29 +200,3 @@ async def append_datasource(
|
|
|
198
200
|
ignore_empty=ignore_empty,
|
|
199
201
|
concurrency=concurrency,
|
|
200
202
|
)
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
def generate_sql_sample_data(datasource_content: str, row_count: int, model: str, api_key: str) -> str:
|
|
204
|
-
client = OpenAI(api_key=api_key)
|
|
205
|
-
|
|
206
|
-
response = client.chat.completions.create(
|
|
207
|
-
model=model,
|
|
208
|
-
messages=[
|
|
209
|
-
{"role": "system", "content": sample_data_sql_prompt.format(row_count=row_count)},
|
|
210
|
-
{"role": "user", "content": datasource_content},
|
|
211
|
-
],
|
|
212
|
-
)
|
|
213
|
-
|
|
214
|
-
return response.choices[0].message.content or ""
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
async def generate_sample_data_from_columns(
|
|
218
|
-
tb_client: TinyB, datasource_content: str, row_count: int = 20
|
|
219
|
-
) -> List[Dict[str, Any]]:
|
|
220
|
-
config = CLIConfig.get_project_config()
|
|
221
|
-
model = config.get("llms", {}).get("openai", {}).get("model", "gpt-4o-mini")
|
|
222
|
-
api_key = config.get("llms", {}).get("openai", {}).get("api_key", None)
|
|
223
|
-
sql = generate_sql_sample_data(datasource_content, row_count, model, api_key)
|
|
224
|
-
result = await tb_client.query(f"{sql} FORMAT JSON")
|
|
225
|
-
data = result.get("data", [])
|
|
226
|
-
return data
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import json
|
|
3
|
+
from typing import Any, Awaitable, Callable, Dict, List
|
|
4
|
+
|
|
5
|
+
from openai import OpenAI
|
|
6
|
+
from pydantic import BaseModel
|
|
7
|
+
|
|
8
|
+
from tinybird.client import TinyB
|
|
9
|
+
from tinybird.tb_cli_modules.prompts import create_project_prompt, sample_data_sql_prompt
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class DataFile(BaseModel):
|
|
13
|
+
name: str
|
|
14
|
+
content: str
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class DataProject(BaseModel):
|
|
18
|
+
datasources: List[DataFile]
|
|
19
|
+
pipes: List[DataFile]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class LLM:
|
|
23
|
+
def __init__(self, key: str):
|
|
24
|
+
self.client = OpenAI(api_key=key)
|
|
25
|
+
|
|
26
|
+
async def _execute(self, action_fn: Callable[[], Awaitable[str]], checker_fn: Callable[[str], bool]):
|
|
27
|
+
is_valid = False
|
|
28
|
+
times = 0
|
|
29
|
+
|
|
30
|
+
while not is_valid and times < 5:
|
|
31
|
+
result = await action_fn()
|
|
32
|
+
if asyncio.iscoroutinefunction(checker_fn):
|
|
33
|
+
is_valid = await checker_fn(result)
|
|
34
|
+
else:
|
|
35
|
+
is_valid = checker_fn(result)
|
|
36
|
+
times += 1
|
|
37
|
+
|
|
38
|
+
return result
|
|
39
|
+
|
|
40
|
+
async def create_project(self, prompt: str) -> DataProject:
|
|
41
|
+
completion = self.client.beta.chat.completions.parse(
|
|
42
|
+
model="gpt-4o-mini",
|
|
43
|
+
messages=[{"role": "system", "content": create_project_prompt}, {"role": "user", "content": prompt}],
|
|
44
|
+
response_format=DataProject,
|
|
45
|
+
)
|
|
46
|
+
return completion.choices[0].message.parsed or DataProject(datasources=[], pipes=[])
|
|
47
|
+
|
|
48
|
+
async def generate_sql_sample_data(
|
|
49
|
+
self, tb_client: TinyB, datasource_name: str, datasource_content: str, row_count: int = 20
|
|
50
|
+
) -> str:
|
|
51
|
+
async def action_fn():
|
|
52
|
+
response = self.client.chat.completions.create(
|
|
53
|
+
model="gpt-4o-mini",
|
|
54
|
+
messages=[
|
|
55
|
+
{"role": "system", "content": sample_data_sql_prompt.format(row_count=row_count)},
|
|
56
|
+
{"role": "user", "content": datasource_content},
|
|
57
|
+
],
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
sql = response.choices[0].message.content or ""
|
|
61
|
+
result = await tb_client.query(f"{sql} FORMAT JSON")
|
|
62
|
+
return result.get("data", [])
|
|
63
|
+
|
|
64
|
+
async def checker_fn(sample_data: List[Dict[str, Any]]):
|
|
65
|
+
ndjson_data = "\n".join([json.dumps(row) for row in sample_data])
|
|
66
|
+
try:
|
|
67
|
+
result = await tb_client.datasource_events(datasource_name, ndjson_data)
|
|
68
|
+
return result.get("successful_rows", 0) > 0
|
|
69
|
+
except Exception:
|
|
70
|
+
return False
|
|
71
|
+
|
|
72
|
+
return await self._execute(action_fn, checker_fn)
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import time
|
|
2
|
+
|
|
3
|
+
import click
|
|
4
|
+
import requests
|
|
5
|
+
|
|
6
|
+
import docker
|
|
7
|
+
from tinybird.feedback_manager import FeedbackManager
|
|
8
|
+
from tinybird.tb_cli_modules.cli import cli
|
|
9
|
+
from tinybird.tb_cli_modules.common import (
|
|
10
|
+
coro,
|
|
11
|
+
)
|
|
12
|
+
from tinybird.tb_cli_modules.config import CLIConfig
|
|
13
|
+
from tinybird.tb_cli_modules.exceptions import CLIException
|
|
14
|
+
|
|
15
|
+
# TODO: Use the official Tinybird image once it's available 'tinybirdco/tinybird-local:latest'
|
|
16
|
+
TB_IMAGE_NAME = "registry.gitlab.com/tinybird/analytics/tinybird-local-jammy-3.11:latest"
|
|
17
|
+
TB_CONTAINER_NAME = "tinybird-local"
|
|
18
|
+
TB_LOCAL_PORT = 80
|
|
19
|
+
TB_LOCAL_HOST = f"http://localhost:{TB_LOCAL_PORT}"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def start_tinybird_local(
|
|
23
|
+
docker_client,
|
|
24
|
+
):
|
|
25
|
+
"""Start the Tinybird container."""
|
|
26
|
+
pull_show_prompt = False
|
|
27
|
+
pull_required = False
|
|
28
|
+
try:
|
|
29
|
+
local_image = docker_client.images.get(TB_IMAGE_NAME)
|
|
30
|
+
local_image_id = local_image.attrs["RepoDigests"][0].split("@")[1]
|
|
31
|
+
remote_image = docker_client.images.get_registry_data(TB_IMAGE_NAME)
|
|
32
|
+
pull_show_prompt = local_image_id != remote_image.id
|
|
33
|
+
except Exception:
|
|
34
|
+
pull_show_prompt = False
|
|
35
|
+
pull_required = True
|
|
36
|
+
|
|
37
|
+
if (
|
|
38
|
+
pull_show_prompt
|
|
39
|
+
and click.prompt(FeedbackManager.info(message="** New version detected, download? [y/N]")).lower() == "y"
|
|
40
|
+
):
|
|
41
|
+
click.echo(FeedbackManager.info(message="** Downloading latest version of Tinybird Local..."))
|
|
42
|
+
pull_required = True
|
|
43
|
+
|
|
44
|
+
if pull_required:
|
|
45
|
+
docker_client.images.pull(TB_IMAGE_NAME, platform="linux/amd64")
|
|
46
|
+
|
|
47
|
+
container = None
|
|
48
|
+
containers = docker_client.containers.list(all=True, filters={"name": TB_CONTAINER_NAME})
|
|
49
|
+
if containers:
|
|
50
|
+
container = containers[0]
|
|
51
|
+
|
|
52
|
+
if container and not pull_required:
|
|
53
|
+
# Container `start` is idempotent. It's safe to call it even if the container is already running.
|
|
54
|
+
container.start()
|
|
55
|
+
else:
|
|
56
|
+
if container:
|
|
57
|
+
container.remove(force=True)
|
|
58
|
+
|
|
59
|
+
container = docker_client.containers.run(
|
|
60
|
+
TB_IMAGE_NAME,
|
|
61
|
+
name=TB_CONTAINER_NAME,
|
|
62
|
+
detach=True,
|
|
63
|
+
ports={"80/tcp": TB_LOCAL_PORT},
|
|
64
|
+
remove=False,
|
|
65
|
+
platform="linux/amd64",
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
click.echo(FeedbackManager.info(message="** Waiting for Tinybird Local to be ready..."))
|
|
69
|
+
for attempt in range(10):
|
|
70
|
+
try:
|
|
71
|
+
run = container.exec_run("tb --no-version-warning sql 'SELECT 1 AS healthcheck' --format json").output
|
|
72
|
+
# dont parse the json as docker sometimes returns warning messages
|
|
73
|
+
# todo: rafa, make this rigth
|
|
74
|
+
if b'"healthcheck": 1' in run:
|
|
75
|
+
break
|
|
76
|
+
raise RuntimeError("Unexpected response from Tinybird")
|
|
77
|
+
except Exception:
|
|
78
|
+
if attempt == 9: # Last attempt
|
|
79
|
+
raise CLIException("Tinybird local not ready yet. Please try again in a few seconds.")
|
|
80
|
+
time.sleep(5) # Wait 5 seconds before retrying
|
|
81
|
+
|
|
82
|
+
click.echo(FeedbackManager.success(message="✓ All set!\n"))
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def get_docker_client():
|
|
86
|
+
"""Check if Docker is installed and running."""
|
|
87
|
+
try:
|
|
88
|
+
client = docker.from_env()
|
|
89
|
+
client.ping()
|
|
90
|
+
return client
|
|
91
|
+
except Exception:
|
|
92
|
+
raise CLIException("Docker is not running or installed. Please ensure Docker is installed and running.")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def stop_tinybird_local(docker_client):
|
|
96
|
+
"""Stop the Tinybird container."""
|
|
97
|
+
try:
|
|
98
|
+
container = docker_client.containers.get(TB_CONTAINER_NAME)
|
|
99
|
+
container.stop()
|
|
100
|
+
except Exception:
|
|
101
|
+
pass
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def remove_tinybird_local(docker_client):
|
|
105
|
+
"""Remove the Tinybird container."""
|
|
106
|
+
try:
|
|
107
|
+
container = docker_client.containers.get(TB_CONTAINER_NAME)
|
|
108
|
+
container.remove(force=True)
|
|
109
|
+
except Exception:
|
|
110
|
+
pass
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def get_tinybird_local_client():
|
|
114
|
+
"""Get a Tinybird client connected to the local environment."""
|
|
115
|
+
config = CLIConfig.get_project_config()
|
|
116
|
+
tokens = requests.get(f"{TB_LOCAL_HOST}/tokens").json()
|
|
117
|
+
token = tokens["workspace_admin_token"]
|
|
118
|
+
config.set_token(token)
|
|
119
|
+
config.set_host(TB_LOCAL_HOST)
|
|
120
|
+
return config.get_client(host=TB_LOCAL_HOST, token=token)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@cli.group()
|
|
124
|
+
@click.pass_context
|
|
125
|
+
def local(ctx):
|
|
126
|
+
"""Local commands"""
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@local.command()
|
|
130
|
+
@coro
|
|
131
|
+
async def stop() -> None:
|
|
132
|
+
"""Stop Tinybird development environment"""
|
|
133
|
+
click.echo(FeedbackManager.info(message="Shutting down Tinybird development environment..."))
|
|
134
|
+
docker_client = get_docker_client()
|
|
135
|
+
stop_tinybird_local(docker_client)
|
|
136
|
+
click.echo(FeedbackManager.success(message="Tinybird development environment stopped"))
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@local.command()
|
|
140
|
+
@coro
|
|
141
|
+
async def remove() -> None:
|
|
142
|
+
"""Remove Tinybird development environment"""
|
|
143
|
+
click.echo(FeedbackManager.info(message="Removing Tinybird development environment..."))
|
|
144
|
+
docker_client = get_docker_client()
|
|
145
|
+
remove_tinybird_local(docker_client)
|
|
146
|
+
click.echo(FeedbackManager.success(message="Tinybird development environment removed"))
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
@local.command()
|
|
150
|
+
@coro
|
|
151
|
+
async def start() -> None:
|
|
152
|
+
"""Start Tinybird development environment"""
|
|
153
|
+
click.echo(FeedbackManager.info(message="Starting Tinybird development environment..."))
|
|
154
|
+
docker_client = get_docker_client()
|
|
155
|
+
start_tinybird_local(docker_client)
|
|
156
|
+
click.echo(FeedbackManager.success(message="Tinybird development environment started"))
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
@local.command()
|
|
160
|
+
@coro
|
|
161
|
+
async def restart() -> None:
|
|
162
|
+
"""Restart Tinybird development environment"""
|
|
163
|
+
click.echo(FeedbackManager.info(message="Restarting Tinybird development environment..."))
|
|
164
|
+
docker_client = get_docker_client()
|
|
165
|
+
remove_tinybird_local(docker_client)
|
|
166
|
+
start_tinybird_local(docker_client)
|
|
167
|
+
click.echo(FeedbackManager.success(message="Tinybird development environment restarted"))
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
tinybird/__cli__.py,sha256=
|
|
1
|
+
tinybird/__cli__.py,sha256=IVqhagi6E4quIGa9lfJelt_mgVbNil0_gOfvItSxOE0,250
|
|
2
2
|
tinybird/check_pypi.py,sha256=_4NkharLyR_ELrAdit-ftqIWvOf7jZNPt3i76frlo9g,975
|
|
3
3
|
tinybird/client.py,sha256=nd97gD2-8Ap8yDonBcVwk9eXDAL43hmIYdo-Pse43RE,50738
|
|
4
4
|
tinybird/config.py,sha256=Z-BX9FrjgsLw1YwcCdF0IztLB97Zpc70VVPplO_pDSY,6089
|
|
5
5
|
tinybird/connectors.py,sha256=lkpVSUmSuViEZBa4QjTK7YmPHUop0a5UFoTrSmlVq6k,15244
|
|
6
6
|
tinybird/context.py,sha256=kutUQ0kCwparowI74_YLXx6wtTzGLRouJ6oGHVBPzBo,1291
|
|
7
|
-
tinybird/datafile.py,sha256=
|
|
7
|
+
tinybird/datafile.py,sha256=dZIIp-V99yS9bnOpH4yAiohG76qKOcJF2wDj7WroT34,253516
|
|
8
8
|
tinybird/datatypes.py,sha256=IHyhZ86ib54Vnd1pbod9y2aS8DDvDKZm1HJGlThdbuQ,10460
|
|
9
|
-
tinybird/feedback_manager.py,sha256=
|
|
9
|
+
tinybird/feedback_manager.py,sha256=Eey5Ih84DBKspjfzIzWpZbpJkkaIqRrpPSljbm7hfNo,67822
|
|
10
10
|
tinybird/git_settings.py,sha256=XUL9ZUj59-ZVQJDYmMEq4UpnuuOuQOHGlNcX3JgQHjQ,3954
|
|
11
11
|
tinybird/sql.py,sha256=gfRKjdqEygcE1WOTeQ1QV2Jal8Jzl4RSX8fftu1KSEs,45825
|
|
12
12
|
tinybird/sql_template.py,sha256=IqYRfUxDYBCoOYjqqvn--_8QXLv9FSRnJ0bInx7q1Xs,93051
|
|
@@ -19,17 +19,19 @@ tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj
|
|
|
19
19
|
tinybird/ch_utils/engine.py,sha256=OXkBhlzGjZotjD0vaT-rFIbSGV4tpiHxE8qO_ip0SyQ,40454
|
|
20
20
|
tinybird/tb_cli_modules/auth.py,sha256=cqxfGgFheuTmenQ3UwPBXTqwMm8JD7uzgLfoIRXdnyQ,9038
|
|
21
21
|
tinybird/tb_cli_modules/branch.py,sha256=Ik8rRVPXvhyChHBqdPVNtI_C-0gLYjUHaznNWE04Ecw,39120
|
|
22
|
-
tinybird/tb_cli_modules/build.py,sha256=
|
|
22
|
+
tinybird/tb_cli_modules/build.py,sha256=FVo3531YlGj1lHm1aI4aYepgn0ffbPfzSUM1lloY1a4,8589
|
|
23
23
|
tinybird/tb_cli_modules/cicd.py,sha256=0lMkb6CVOFZl5HOwgY8mK4T4mgI7O8335UngLXtCc-c,13851
|
|
24
24
|
tinybird/tb_cli_modules/cli.py,sha256=a738xz4DUNep_mzHTEUExd0NAuf7cHkUAlMsEus9xaM,65072
|
|
25
|
-
tinybird/tb_cli_modules/common.py,sha256=
|
|
26
|
-
tinybird/tb_cli_modules/config.py,sha256=
|
|
25
|
+
tinybird/tb_cli_modules/common.py,sha256=VxzCcCaVoUnwvO-AN6JqeEHK9AzichzSgN_QVgqLHHM,78413
|
|
26
|
+
tinybird/tb_cli_modules/config.py,sha256=ppWvACHrSLkb5hOoQLYNby2w8jR76-8Kx2NBCst7ntQ,11760
|
|
27
27
|
tinybird/tb_cli_modules/connection.py,sha256=YggP34Qh3hCjD41lp1ZHVCwHPjI_-um2spvEV8F_tSU,28684
|
|
28
|
-
tinybird/tb_cli_modules/create.py,sha256=
|
|
28
|
+
tinybird/tb_cli_modules/create.py,sha256=glQiRQEVGTOw9VmoICEF5_7YWuraJrk3BSUDSTSUfP0,6796
|
|
29
29
|
tinybird/tb_cli_modules/datasource.py,sha256=BVYwPkKvdnEv2YMkxof7n7FOffyJm-QK7Pk9Ui-MC3A,35816
|
|
30
30
|
tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
|
|
31
31
|
tinybird/tb_cli_modules/fmt.py,sha256=_xL2AyGo3us7NpmfAXtSIhn1cGAC-A4PspdhUjm58jY,3382
|
|
32
32
|
tinybird/tb_cli_modules/job.py,sha256=AG69LPb9MbobA1awwJFZJvxqarDKfRlsBjw2V1zvYqc,2964
|
|
33
|
+
tinybird/tb_cli_modules/llm.py,sha256=epJqg4peovBS2urlyGUccmvuyy3dp57SnYkatTGcD_I,2483
|
|
34
|
+
tinybird/tb_cli_modules/local.py,sha256=52XgRCUqK8_07iXLLLnmP5KiAWVoXNbuCxlnobPcTQQ,5718
|
|
33
35
|
tinybird/tb_cli_modules/pipe.py,sha256=BCLAQ3ZuWKGAih2mupnw_Y2S5B5cNS-epF317whsSEE,30989
|
|
34
36
|
tinybird/tb_cli_modules/prompts.py,sha256=Esjhet-KUM1bT6RStk2voCCVjvAOUD4hDDiKpn-AYNY,7227
|
|
35
37
|
tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
@@ -41,8 +43,8 @@ tinybird/tb_cli_modules/workspace.py,sha256=N8f1dl4BYc34ucmJ6oNZc4XZMGKd8m0Fhq7o
|
|
|
41
43
|
tinybird/tb_cli_modules/workspace_members.py,sha256=ksXsjd233y9-sNlz4Qb-meZbX4zn1B84e_bSm2i8rhg,8731
|
|
42
44
|
tinybird/tb_cli_modules/tinyunit/tinyunit.py,sha256=_ydOD4WxmKy7_h-d3fG62w_0_lD0uLl9w4EbEYtwd0U,11720
|
|
43
45
|
tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py,sha256=hGh1ZaXC1af7rKnX7222urkj0QJMhMWclqMy59dOqwE,1922
|
|
44
|
-
tinybird-0.0.1.
|
|
45
|
-
tinybird-0.0.1.
|
|
46
|
-
tinybird-0.0.1.
|
|
47
|
-
tinybird-0.0.1.
|
|
48
|
-
tinybird-0.0.1.
|
|
46
|
+
tinybird-0.0.1.dev4.dist-info/METADATA,sha256=LGWZlh2D8xM9K8TG6sURXP2cB5r861y8Zw6EJm1-W2s,2404
|
|
47
|
+
tinybird-0.0.1.dev4.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
|
|
48
|
+
tinybird-0.0.1.dev4.dist-info/entry_points.txt,sha256=PKPKuPmA4IfJYnCFHHUiw-aAWZuBomFvwCklv1OyCjE,43
|
|
49
|
+
tinybird-0.0.1.dev4.dist-info/top_level.txt,sha256=pgw6AzERHBcW3YTi2PW4arjxLkulk2msOz_SomfOEuc,45
|
|
50
|
+
tinybird-0.0.1.dev4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|