tinybird 0.0.1.dev2__py3-none-any.whl → 0.0.1.dev4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of tinybird might be problematic. Click here for more details.

tinybird/__cli__.py CHANGED
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
4
4
  __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
5
5
  __author__ = 'Tinybird'
6
6
  __author_email__ = 'support@tinybird.co'
7
- __version__ = '0.0.1.dev2'
8
- __revision__ = 'bdbc260'
7
+ __version__ = '0.0.1.dev4'
8
+ __revision__ = '388c81a'
tinybird/datafile.py CHANGED
@@ -3125,7 +3125,7 @@ async def new_pipe(
3125
3125
  except Exception as e:
3126
3126
  raise click.ClickException(FeedbackManager.error_creating_pipe(error=e))
3127
3127
 
3128
- if data.get("type") == "endpoint":
3128
+ if data.get("type") == "endpoint" and t:
3129
3129
  click.echo(FeedbackManager.success_test_endpoint(host=host, pipe=p["name"], token=t["token"]))
3130
3130
 
3131
3131
 
@@ -51,6 +51,10 @@ def prompt_message(message: str) -> Callable[..., str]:
51
51
  return print_message(message, bcolors.HEADER)
52
52
 
53
53
 
54
+ def gray_message(message: str) -> Callable[..., str]:
55
+ return print_message(message, bcolors.CGREY)
56
+
57
+
54
58
  class FeedbackManager:
55
59
  error_exception = error_message("{error}")
56
60
  simple_error_exception = simple_error_message("{error}")
@@ -712,9 +716,9 @@ Ready? """
712
716
  info_removing_pipe = info_message("** Removing pipe {pipe}")
713
717
  info_removing_pipe_not_found = info_message("** {pipe} not found")
714
718
  info_dry_removing_pipe = info_message("** [DRY RUN] Removing pipe {pipe}")
715
- info_path_created = info_message("** - /{path} created")
716
- info_file_created = info_message("** - {file} created")
717
- info_path_already_exists = info_message("** - /{path} already exists, skipping")
719
+ info_path_created = info_message(" /{path}")
720
+ info_file_created = info_message(" /{file}")
721
+ info_path_already_exists = info_message(" /{path} already exists, skipping")
718
722
  info_dottinyb_already_ignored = info_message("** - '.tinyb' already in .gitignore, skipping")
719
723
  info_dotdifftemp_already_ignored = info_message("** - '.diff_tmp' not found or already in .gitignore, skipping")
720
724
  info_dottinyenv_already_exists = info_message("** - '.tinyenv' already exists, skipping")
@@ -935,7 +939,7 @@ Ready? """
935
939
  success_create = success_message("** '{name}' created")
936
940
  success_delete = success_message("** '{name}' deleted")
937
941
  success_dynamodb_initial_load = success_message("** Initial load of DynamoDB table started: {job_url}")
938
- success_progress_blocks = success_message("** \N{FRONT-FACING BABY CHICK} done")
942
+ success_progress_blocks = success_message(" Done!")
939
943
  success_now_using_config = success_message("** Now using {name} ({id})")
940
944
  success_connector_config = success_message(
941
945
  "** {connector} configuration written to {file_name} file, consider adding it to .gitignore"
@@ -1026,3 +1030,4 @@ Ready? """
1026
1030
  info = info_message("{message}")
1027
1031
  highlight = info_highlight_message("{message}")
1028
1032
  error = error_message("{message}")
1033
+ gray = gray_message("{message}")
@@ -1,11 +1,12 @@
1
1
  import asyncio
2
- import json
3
2
  import os
3
+ import random
4
4
  import time
5
5
  from pathlib import Path
6
6
  from typing import Any, Awaitable, Callable, Dict, List, Union
7
7
 
8
8
  import click
9
+ import humanfriendly
9
10
  from watchdog.events import FileSystemEventHandler
10
11
  from watchdog.observers import Observer
11
12
 
@@ -20,20 +21,17 @@ from tinybird.datafile import (
20
21
  parse_datasource,
21
22
  parse_pipe,
22
23
  )
23
- from tinybird.feedback_manager import FeedbackManager, info_highlight_message, success_message
24
+ from tinybird.feedback_manager import FeedbackManager, bcolors
24
25
  from tinybird.tb_cli_modules.cli import cli
25
26
  from tinybird.tb_cli_modules.common import (
26
27
  coro,
27
- echo_safe_humanfriendly_tables_format_smart_table,
28
28
  )
29
- from tinybird.tb_cli_modules.create import generate_sample_data_from_columns
29
+ from tinybird.tb_cli_modules.config import CLIConfig
30
+ from tinybird.tb_cli_modules.llm import LLM
30
31
  from tinybird.tb_cli_modules.local import (
31
- get_docker_client,
32
32
  get_tinybird_local_client,
33
- remove_tinybird_local,
34
- start_tinybird_local,
35
- stop_tinybird_local,
36
33
  )
34
+ from tinybird.tb_cli_modules.table import format_table
37
35
 
38
36
 
39
37
  class FileChangeHandler(FileSystemEventHandler):
@@ -44,7 +42,7 @@ class FileChangeHandler(FileSystemEventHandler):
44
42
  def on_modified(self, event: Any) -> None:
45
43
  if not event.is_directory and any(event.src_path.endswith(ext) for ext in [".datasource", ".pipe"]):
46
44
  filename = event.src_path.split("/")[-1]
47
- click.echo(info_highlight_message(f"\n⟲ Changes detected in {filename}\n")())
45
+ click.echo(FeedbackManager.highlight(message=f"\n⟲ Changes detected in {filename}\n"))
48
46
  try:
49
47
  self.process([event.src_path])
50
48
  except Exception as e:
@@ -65,7 +63,10 @@ def watch_files(
65
63
  process(files, watch=True)
66
64
  time_end = time.time()
67
65
  elapsed_time = time_end - time_start
68
- click.echo(success_message(f"\n✓ Rebuild completed in {elapsed_time:.1f}s")())
66
+ click.echo(
67
+ FeedbackManager.success(message="\n✓ ")
68
+ + FeedbackManager.gray(message=f"Rebuild completed in {elapsed_time:.1f}s")
69
+ )
69
70
 
70
71
  event_handler = FileChangeHandler(filenames, lambda f: asyncio.run(process_wrapper(f)))
71
72
  observer = Observer()
@@ -100,23 +101,19 @@ def watch_files(
100
101
  help="Watch for changes in the files and re-check them.",
101
102
  )
102
103
  @click.option(
103
- "--restart",
104
+ "--skip-datasources",
104
105
  is_flag=True,
105
- help="Restart the Tinybird development environment before building the first time.",
106
+ help="Skip rebuilding datasources.",
106
107
  )
107
108
  @coro
108
109
  async def build(
109
110
  folder: str,
110
111
  watch: bool,
111
- restart: bool,
112
+ skip_datasources: bool,
112
113
  ) -> None:
113
114
  """
114
115
  Watch for changes in the files and re-check them.
115
116
  """
116
- docker_client = get_docker_client()
117
- if restart:
118
- remove_tinybird_local(docker_client)
119
- start_tinybird_local(docker_client)
120
117
  ignore_sql_errors = FeatureFlags.ignore_sql_errors()
121
118
  context.disable_template_security_validation.set(True)
122
119
  is_internal = has_internal_datafiles(folder)
@@ -157,13 +154,20 @@ async def build(
157
154
  )
158
155
 
159
156
  for filename in filenames:
160
- if filename.endswith(".datasource"):
157
+ if filename.endswith(".datasource") and not skip_datasources:
161
158
  ds_path = Path(filename)
162
159
  ds_name = ds_path.stem
163
160
  datasource_content = ds_path.read_text()
164
- sample_data = await generate_sample_data_from_columns(tb_client, datasource_content)
165
- ndjson_data = "\n".join([json.dumps(row) for row in sample_data])
166
- await tb_client.datasource_events(ds_name, ndjson_data)
161
+ llm_config = CLIConfig.get_llm_config()
162
+ llm = LLM(key=llm_config["api_key"])
163
+ has_json_path = "`json:" in datasource_content
164
+
165
+ if has_json_path:
166
+ await llm.generate_sql_sample_data(
167
+ tb_client=tb_client,
168
+ datasource_name=ds_name,
169
+ datasource_content=datasource_content,
170
+ )
167
171
 
168
172
  if watch:
169
173
  filename = filenames[0]
@@ -176,7 +180,7 @@ async def build(
176
180
  try:
177
181
  click.echo("⚡ Building project...")
178
182
  time_start = time.time()
179
- await process(filenames=filenames, watch=False)
183
+ await process(filenames=filenames, watch=False, only_pipes=skip_datasources)
180
184
  time_end = time.time()
181
185
  elapsed_time = time_end - time_start
182
186
  click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s\n"))
@@ -191,31 +195,44 @@ async def build(
191
195
 
192
196
 
193
197
  async def build_and_print_pipe(tb_client: TinyB, filename: str):
194
- pipe_name = os.path.basename(filename.split(".")[0])
195
- res = await tb_client.query(f"SELECT * FROM {pipe_name} LIMIT 5 FORMAT JSON", pipeline=pipe_name)
198
+ rebuild_colors = [bcolors.FAIL, bcolors.OKBLUE, bcolors.WARNING, bcolors.OKGREEN, bcolors.HEADER]
199
+ rebuild_index = random.randint(0, len(rebuild_colors) - 1)
200
+ rebuild_color = rebuild_colors[rebuild_index % len(rebuild_colors)]
201
+ pipe_name = Path(filename).stem
202
+ res = await tb_client.query(f"SELECT * FROM {pipe_name} FORMAT JSON", pipeline=pipe_name)
196
203
  data = []
197
- for d in res["data"]:
204
+ limit = 5
205
+ for d in res["data"][:5]:
198
206
  data.append(d.values())
199
207
  meta = res["meta"]
200
- column_names = [col["name"] for col in meta]
201
- echo_safe_humanfriendly_tables_format_smart_table(data, column_names=column_names)
202
-
203
-
204
- @cli.command()
205
- @coro
206
- async def stop() -> None:
207
- """Stop Tinybird development environment"""
208
- click.echo(FeedbackManager.info(message="Shutting down Tinybird development environment..."))
209
- docker_client = get_docker_client()
210
- stop_tinybird_local(docker_client)
211
- click.echo(FeedbackManager.success(message="Tinybird development environment stopped"))
212
-
208
+ row_count = res.get("rows", 0)
209
+ stats = res.get("statistics", {})
210
+ elapsed = stats.get("elapsed", 0)
211
+ node_name = "endpoint"
212
+ cols = len(meta)
213
+ try:
213
214
 
214
- @cli.command()
215
- @coro
216
- async def start() -> None:
217
- """Start Tinybird development environment"""
218
- click.echo(FeedbackManager.info(message="Starting Tinybird development environment..."))
219
- docker_client = get_docker_client()
220
- start_tinybird_local(docker_client)
221
- click.echo(FeedbackManager.success(message="Tinybird development environment started"))
215
+ def print_message(message: str, color=bcolors.CGREY):
216
+ return f"{color}{message}{bcolors.ENDC}"
217
+
218
+ table = format_table(data, meta)
219
+ colored_char = print_message("│", rebuild_color)
220
+ table_with_marker = "\n".join(f"{colored_char} {line}" for line in table.split("\n"))
221
+ click.echo(f"\n{colored_char} {print_message('⚡', rebuild_color)} Running {pipe_name} → {node_name}")
222
+ click.echo(colored_char)
223
+ click.echo(table_with_marker)
224
+ click.echo(colored_char)
225
+ rows_read = humanfriendly.format_number(stats.get("rows_read", 0))
226
+ bytes_read = humanfriendly.format_size(stats.get("bytes_read", 0))
227
+ elapsed = humanfriendly.format_timespan(elapsed) if elapsed >= 1 else f"{elapsed * 1000:.2f}ms"
228
+ stats_message = f"» {bytes_read} ({rows_read} rows x {cols} cols) in {elapsed}"
229
+ rows_message = f"» Showing {limit} first rows" if row_count > limit else "» Showing all rows"
230
+ click.echo(f"{colored_char} {print_message(stats_message, bcolors.OKGREEN)}")
231
+ click.echo(f"{colored_char} {print_message(rows_message, bcolors.CGREY)}")
232
+ except ValueError as exc:
233
+ if str(exc) == "max() arg is an empty sequence":
234
+ click.echo("------------")
235
+ click.echo("Empty")
236
+ click.echo("------------")
237
+ else:
238
+ raise exc
@@ -153,7 +153,7 @@ def generate_datafile(
153
153
  if not f.exists() or force:
154
154
  with open(f"{f}", "w") as ds_file:
155
155
  ds_file.write(datafile)
156
- click.echo(FeedbackManager.success(message=f"** Generated {f}"))
156
+ click.echo(FeedbackManager.info_file_created(file=f))
157
157
 
158
158
  if data and (base / "fixtures").exists():
159
159
  # Generating a fixture for Parquet files is not so trivial, since Parquet format
@@ -1090,7 +1090,7 @@ async def push_data(
1090
1090
  cb.First = True # type: ignore[attr-defined]
1091
1091
  cb.prev_done = 0 # type: ignore[attr-defined]
1092
1092
 
1093
- click.echo(FeedbackManager.info_starting_import_process())
1093
+ click.echo(FeedbackManager.gray(message=f"\nImporting data to {datasource_name} Data Source..."))
1094
1094
 
1095
1095
  if isinstance(url, list):
1096
1096
  urls = url
@@ -1161,17 +1161,16 @@ async def push_data(
1161
1161
  except Exception as e:
1162
1162
  raise CLIException(FeedbackManager.error_exception(error=e))
1163
1163
  else:
1164
- click.echo(FeedbackManager.success_progress_blocks())
1165
1164
  if mode == "append" and parser and parser != "clickhouse":
1166
1165
  click.echo(FeedbackManager.success_appended_rows(appended_rows=appended_rows))
1167
1166
 
1168
- click.echo(FeedbackManager.success_total_rows(datasource=datasource_name, total_rows=total_rows))
1169
-
1170
1167
  if mode == "replace":
1171
1168
  click.echo(FeedbackManager.success_replaced_datasource(datasource=datasource_name))
1172
1169
  else:
1173
- click.echo(FeedbackManager.success_appended_datasource(datasource=datasource_name))
1174
- click.echo(FeedbackManager.info_data_pushed(datasource=datasource_name))
1170
+ click.echo(FeedbackManager.highlight(message="» 2.57m rows x 9 cols in 852.04ms"))
1171
+
1172
+ click.echo(FeedbackManager.success_progress_blocks())
1173
+
1175
1174
  finally:
1176
1175
  try:
1177
1176
  for url in urls:
@@ -338,6 +338,14 @@ class CLIConfig:
338
338
  CLIConfig._projects[working_dir] = result
339
339
  return result
340
340
 
341
+ @staticmethod
342
+ def get_llm_config(working_dir: Optional[str] = None) -> Dict[str, Any]:
343
+ return (
344
+ CLIConfig.get_project_config(working_dir)
345
+ .get("llms", {})
346
+ .get("openai", {"model": "gpt-4o-mini", "api_key": None})
347
+ )
348
+
341
349
  @staticmethod
342
350
  def reset() -> None:
343
351
  CLIConfig._global = None
@@ -1,4 +1,3 @@
1
- import json
2
1
  import os
3
2
  from os import getcwd
4
3
  from pathlib import Path
@@ -6,7 +5,6 @@ from typing import Any, Dict, List, Optional
6
5
 
7
6
  import click
8
7
  from click import Context
9
- from openai import OpenAI
10
8
 
11
9
  from tinybird.client import TinyB
12
10
  from tinybird.datafile import folder_build
@@ -16,8 +14,9 @@ from tinybird.tb_cli_modules.common import _generate_datafile, coro, generate_da
16
14
  from tinybird.tb_cli_modules.config import CLIConfig
17
15
  from tinybird.tb_cli_modules.exceptions import CLIDatasourceException
18
16
  from tinybird.tb_cli_modules.llm import LLM
19
- from tinybird.tb_cli_modules.local import get_docker_client, set_up_tinybird_local
20
- from tinybird.tb_cli_modules.prompts import sample_data_sql_prompt
17
+ from tinybird.tb_cli_modules.local import (
18
+ get_tinybird_local_client,
19
+ )
21
20
 
22
21
 
23
22
  @cli.command()
@@ -48,12 +47,13 @@ async def create(
48
47
  folder: Optional[str],
49
48
  ) -> None:
50
49
  """Initialize a new project."""
51
- click.echo(FeedbackManager.highlight(message="Setting up Tinybird development environment..."))
50
+ click.echo(FeedbackManager.gray(message="Setting up Tinybird Local"))
52
51
  folder = folder or getcwd()
53
52
  try:
54
- docker_client = get_docker_client()
55
- tb_client = set_up_tinybird_local(docker_client)
53
+ tb_client = get_tinybird_local_client()
54
+ click.echo(FeedbackManager.gray(message="Creating new project structure..."))
56
55
  await project_create(tb_client, data, prompt, folder)
56
+ click.echo(FeedbackManager.success(message="✓ Scaffolding completed!\n"))
57
57
  workspaces: List[Dict[str, Any]] = (await tb_client.user_workspaces()).get("workspaces", [])
58
58
  datasources = await tb_client.datasources()
59
59
  pipes = await tb_client.pipes(dependencies=True)
@@ -69,11 +69,15 @@ async def create(
69
69
  elif prompt:
70
70
  datasource_files = [f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")]
71
71
  for datasource_file in datasource_files:
72
- datasource_content = Path(folder) / "datasources" / datasource_file
73
- sample_data = await generate_sample_data_from_columns(tb_client, datasource_content)
74
- ndjson_data = "\n".join([json.dumps(row) for row in sample_data])
75
- await tb_client.datasource_events(datasource_file, ndjson_data)
76
- click.echo(FeedbackManager.success(message="\n✔ Tinybird development environment is ready"))
72
+ datasource_path = Path(folder) / "datasources" / datasource_file
73
+ llm_config = CLIConfig.get_llm_config()
74
+ llm = LLM(key=llm_config["api_key"])
75
+ datasource_name = datasource_path.stem
76
+ datasource_content = datasource_path.read_text()
77
+ has_json_path = "`json:" in datasource_content
78
+ if has_json_path:
79
+ await llm.generate_sql_sample_data(tb_client, datasource_name, datasource_content)
80
+ click.echo(FeedbackManager.success(message="\n✓ Tinybird Local is ready!"))
77
81
  except Exception as e:
78
82
  click.echo(FeedbackManager.error(message=f"Error: {str(e)}"))
79
83
 
@@ -84,7 +88,7 @@ async def project_create(
84
88
  prompt: Optional[str],
85
89
  folder: str,
86
90
  ):
87
- project_paths = ["datasources", "endpoints", "copies", "sinks", "playgrounds", "materializations"]
91
+ project_paths = ["datasources", "endpoints", "materializations", "copies", "sinks"]
88
92
  force = True
89
93
  for x in project_paths:
90
94
  try:
@@ -92,7 +96,7 @@ async def project_create(
92
96
  f.mkdir()
93
97
  click.echo(FeedbackManager.info_path_created(path=x))
94
98
  except FileExistsError:
95
- pass
99
+ click.echo(FeedbackManager.info_path_created(path=x))
96
100
 
97
101
  def generate_pipe_file(name: str, content: str):
98
102
  base = Path("endpoints")
@@ -101,7 +105,7 @@ async def project_create(
101
105
  f = base / (f"{name}.pipe")
102
106
  with open(f"{f}", "w") as file:
103
107
  file.write(content)
104
- click.echo(FeedbackManager.success(message=f"** Generated {f}"))
108
+ click.echo(FeedbackManager.info_file_created(file=f))
105
109
 
106
110
  if data:
107
111
  path = Path(folder) / data
@@ -119,10 +123,8 @@ TYPE ENDPOINT
119
123
  )
120
124
  elif prompt:
121
125
  try:
122
- config = CLIConfig.get_project_config()
123
- model = config.get("llms", {}).get("openai", {}).get("model", "gpt-4o-mini")
124
- api_key = config.get("llms", {}).get("openai", {}).get("api_key", None)
125
- llm = LLM(model=model, key=api_key)
126
+ llm_config = CLIConfig.get_llm_config()
127
+ llm = LLM(key=llm_config["api_key"])
126
128
  result = await llm.create_project(prompt)
127
129
  for ds in result.datasources:
128
130
  content = ds.content.replace("```", "")
@@ -198,29 +200,3 @@ async def append_datasource(
198
200
  ignore_empty=ignore_empty,
199
201
  concurrency=concurrency,
200
202
  )
201
-
202
-
203
- def generate_sql_sample_data(datasource_content: str, row_count: int, model: str, api_key: str) -> str:
204
- client = OpenAI(api_key=api_key)
205
-
206
- response = client.chat.completions.create(
207
- model=model,
208
- messages=[
209
- {"role": "system", "content": sample_data_sql_prompt.format(row_count=row_count)},
210
- {"role": "user", "content": datasource_content},
211
- ],
212
- )
213
-
214
- return response.choices[0].message.content or ""
215
-
216
-
217
- async def generate_sample_data_from_columns(
218
- tb_client: TinyB, datasource_content: str, row_count: int = 20
219
- ) -> List[Dict[str, Any]]:
220
- config = CLIConfig.get_project_config()
221
- model = config.get("llms", {}).get("openai", {}).get("model", "gpt-4o-mini")
222
- api_key = config.get("llms", {}).get("openai", {}).get("api_key", None)
223
- sql = generate_sql_sample_data(datasource_content, row_count, model, api_key)
224
- result = await tb_client.query(f"{sql} FORMAT JSON")
225
- data = result.get("data", [])
226
- return data
@@ -0,0 +1,72 @@
1
+ import asyncio
2
+ import json
3
+ from typing import Any, Awaitable, Callable, Dict, List
4
+
5
+ from openai import OpenAI
6
+ from pydantic import BaseModel
7
+
8
+ from tinybird.client import TinyB
9
+ from tinybird.tb_cli_modules.prompts import create_project_prompt, sample_data_sql_prompt
10
+
11
+
12
+ class DataFile(BaseModel):
13
+ name: str
14
+ content: str
15
+
16
+
17
+ class DataProject(BaseModel):
18
+ datasources: List[DataFile]
19
+ pipes: List[DataFile]
20
+
21
+
22
+ class LLM:
23
+ def __init__(self, key: str):
24
+ self.client = OpenAI(api_key=key)
25
+
26
+ async def _execute(self, action_fn: Callable[[], Awaitable[str]], checker_fn: Callable[[str], bool]):
27
+ is_valid = False
28
+ times = 0
29
+
30
+ while not is_valid and times < 5:
31
+ result = await action_fn()
32
+ if asyncio.iscoroutinefunction(checker_fn):
33
+ is_valid = await checker_fn(result)
34
+ else:
35
+ is_valid = checker_fn(result)
36
+ times += 1
37
+
38
+ return result
39
+
40
+ async def create_project(self, prompt: str) -> DataProject:
41
+ completion = self.client.beta.chat.completions.parse(
42
+ model="gpt-4o-mini",
43
+ messages=[{"role": "system", "content": create_project_prompt}, {"role": "user", "content": prompt}],
44
+ response_format=DataProject,
45
+ )
46
+ return completion.choices[0].message.parsed or DataProject(datasources=[], pipes=[])
47
+
48
+ async def generate_sql_sample_data(
49
+ self, tb_client: TinyB, datasource_name: str, datasource_content: str, row_count: int = 20
50
+ ) -> str:
51
+ async def action_fn():
52
+ response = self.client.chat.completions.create(
53
+ model="gpt-4o-mini",
54
+ messages=[
55
+ {"role": "system", "content": sample_data_sql_prompt.format(row_count=row_count)},
56
+ {"role": "user", "content": datasource_content},
57
+ ],
58
+ )
59
+
60
+ sql = response.choices[0].message.content or ""
61
+ result = await tb_client.query(f"{sql} FORMAT JSON")
62
+ return result.get("data", [])
63
+
64
+ async def checker_fn(sample_data: List[Dict[str, Any]]):
65
+ ndjson_data = "\n".join([json.dumps(row) for row in sample_data])
66
+ try:
67
+ result = await tb_client.datasource_events(datasource_name, ndjson_data)
68
+ return result.get("successful_rows", 0) > 0
69
+ except Exception:
70
+ return False
71
+
72
+ return await self._execute(action_fn, checker_fn)
@@ -0,0 +1,167 @@
1
+ import time
2
+
3
+ import click
4
+ import requests
5
+
6
+ import docker
7
+ from tinybird.feedback_manager import FeedbackManager
8
+ from tinybird.tb_cli_modules.cli import cli
9
+ from tinybird.tb_cli_modules.common import (
10
+ coro,
11
+ )
12
+ from tinybird.tb_cli_modules.config import CLIConfig
13
+ from tinybird.tb_cli_modules.exceptions import CLIException
14
+
15
+ # TODO: Use the official Tinybird image once it's available 'tinybirdco/tinybird-local:latest'
16
+ TB_IMAGE_NAME = "registry.gitlab.com/tinybird/analytics/tinybird-local-jammy-3.11:latest"
17
+ TB_CONTAINER_NAME = "tinybird-local"
18
+ TB_LOCAL_PORT = 80
19
+ TB_LOCAL_HOST = f"http://localhost:{TB_LOCAL_PORT}"
20
+
21
+
22
+ def start_tinybird_local(
23
+ docker_client,
24
+ ):
25
+ """Start the Tinybird container."""
26
+ pull_show_prompt = False
27
+ pull_required = False
28
+ try:
29
+ local_image = docker_client.images.get(TB_IMAGE_NAME)
30
+ local_image_id = local_image.attrs["RepoDigests"][0].split("@")[1]
31
+ remote_image = docker_client.images.get_registry_data(TB_IMAGE_NAME)
32
+ pull_show_prompt = local_image_id != remote_image.id
33
+ except Exception:
34
+ pull_show_prompt = False
35
+ pull_required = True
36
+
37
+ if (
38
+ pull_show_prompt
39
+ and click.prompt(FeedbackManager.info(message="** New version detected, download? [y/N]")).lower() == "y"
40
+ ):
41
+ click.echo(FeedbackManager.info(message="** Downloading latest version of Tinybird Local..."))
42
+ pull_required = True
43
+
44
+ if pull_required:
45
+ docker_client.images.pull(TB_IMAGE_NAME, platform="linux/amd64")
46
+
47
+ container = None
48
+ containers = docker_client.containers.list(all=True, filters={"name": TB_CONTAINER_NAME})
49
+ if containers:
50
+ container = containers[0]
51
+
52
+ if container and not pull_required:
53
+ # Container `start` is idempotent. It's safe to call it even if the container is already running.
54
+ container.start()
55
+ else:
56
+ if container:
57
+ container.remove(force=True)
58
+
59
+ container = docker_client.containers.run(
60
+ TB_IMAGE_NAME,
61
+ name=TB_CONTAINER_NAME,
62
+ detach=True,
63
+ ports={"80/tcp": TB_LOCAL_PORT},
64
+ remove=False,
65
+ platform="linux/amd64",
66
+ )
67
+
68
+ click.echo(FeedbackManager.info(message="** Waiting for Tinybird Local to be ready..."))
69
+ for attempt in range(10):
70
+ try:
71
+ run = container.exec_run("tb --no-version-warning sql 'SELECT 1 AS healthcheck' --format json").output
72
+ # dont parse the json as docker sometimes returns warning messages
73
+ # todo: rafa, make this rigth
74
+ if b'"healthcheck": 1' in run:
75
+ break
76
+ raise RuntimeError("Unexpected response from Tinybird")
77
+ except Exception:
78
+ if attempt == 9: # Last attempt
79
+ raise CLIException("Tinybird local not ready yet. Please try again in a few seconds.")
80
+ time.sleep(5) # Wait 5 seconds before retrying
81
+
82
+ click.echo(FeedbackManager.success(message="✓ All set!\n"))
83
+
84
+
85
+ def get_docker_client():
86
+ """Check if Docker is installed and running."""
87
+ try:
88
+ client = docker.from_env()
89
+ client.ping()
90
+ return client
91
+ except Exception:
92
+ raise CLIException("Docker is not running or installed. Please ensure Docker is installed and running.")
93
+
94
+
95
+ def stop_tinybird_local(docker_client):
96
+ """Stop the Tinybird container."""
97
+ try:
98
+ container = docker_client.containers.get(TB_CONTAINER_NAME)
99
+ container.stop()
100
+ except Exception:
101
+ pass
102
+
103
+
104
+ def remove_tinybird_local(docker_client):
105
+ """Remove the Tinybird container."""
106
+ try:
107
+ container = docker_client.containers.get(TB_CONTAINER_NAME)
108
+ container.remove(force=True)
109
+ except Exception:
110
+ pass
111
+
112
+
113
+ def get_tinybird_local_client():
114
+ """Get a Tinybird client connected to the local environment."""
115
+ config = CLIConfig.get_project_config()
116
+ tokens = requests.get(f"{TB_LOCAL_HOST}/tokens").json()
117
+ token = tokens["workspace_admin_token"]
118
+ config.set_token(token)
119
+ config.set_host(TB_LOCAL_HOST)
120
+ return config.get_client(host=TB_LOCAL_HOST, token=token)
121
+
122
+
123
+ @cli.group()
124
+ @click.pass_context
125
+ def local(ctx):
126
+ """Local commands"""
127
+
128
+
129
+ @local.command()
130
+ @coro
131
+ async def stop() -> None:
132
+ """Stop Tinybird development environment"""
133
+ click.echo(FeedbackManager.info(message="Shutting down Tinybird development environment..."))
134
+ docker_client = get_docker_client()
135
+ stop_tinybird_local(docker_client)
136
+ click.echo(FeedbackManager.success(message="Tinybird development environment stopped"))
137
+
138
+
139
+ @local.command()
140
+ @coro
141
+ async def remove() -> None:
142
+ """Remove Tinybird development environment"""
143
+ click.echo(FeedbackManager.info(message="Removing Tinybird development environment..."))
144
+ docker_client = get_docker_client()
145
+ remove_tinybird_local(docker_client)
146
+ click.echo(FeedbackManager.success(message="Tinybird development environment removed"))
147
+
148
+
149
+ @local.command()
150
+ @coro
151
+ async def start() -> None:
152
+ """Start Tinybird development environment"""
153
+ click.echo(FeedbackManager.info(message="Starting Tinybird development environment..."))
154
+ docker_client = get_docker_client()
155
+ start_tinybird_local(docker_client)
156
+ click.echo(FeedbackManager.success(message="Tinybird development environment started"))
157
+
158
+
159
+ @local.command()
160
+ @coro
161
+ async def restart() -> None:
162
+ """Restart Tinybird development environment"""
163
+ click.echo(FeedbackManager.info(message="Restarting Tinybird development environment..."))
164
+ docker_client = get_docker_client()
165
+ remove_tinybird_local(docker_client)
166
+ start_tinybird_local(docker_client)
167
+ click.echo(FeedbackManager.success(message="Tinybird development environment restarted"))
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: tinybird
3
- Version: 0.0.1.dev2
3
+ Version: 0.0.1.dev4
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/cli/introduction.html
6
6
  Author: Tinybird
@@ -1,12 +1,12 @@
1
- tinybird/__cli__.py,sha256=PHs2gdn0u2epRONYAzC2TOYLhI2SR6FPDDS5NDrS0TY,250
1
+ tinybird/__cli__.py,sha256=IVqhagi6E4quIGa9lfJelt_mgVbNil0_gOfvItSxOE0,250
2
2
  tinybird/check_pypi.py,sha256=_4NkharLyR_ELrAdit-ftqIWvOf7jZNPt3i76frlo9g,975
3
3
  tinybird/client.py,sha256=nd97gD2-8Ap8yDonBcVwk9eXDAL43hmIYdo-Pse43RE,50738
4
4
  tinybird/config.py,sha256=Z-BX9FrjgsLw1YwcCdF0IztLB97Zpc70VVPplO_pDSY,6089
5
5
  tinybird/connectors.py,sha256=lkpVSUmSuViEZBa4QjTK7YmPHUop0a5UFoTrSmlVq6k,15244
6
6
  tinybird/context.py,sha256=kutUQ0kCwparowI74_YLXx6wtTzGLRouJ6oGHVBPzBo,1291
7
- tinybird/datafile.py,sha256=JpZX33Ilq71PiuDBs4TNF1xJoUMtTPLynIbHuo7Bn80,253510
7
+ tinybird/datafile.py,sha256=dZIIp-V99yS9bnOpH4yAiohG76qKOcJF2wDj7WroT34,253516
8
8
  tinybird/datatypes.py,sha256=IHyhZ86ib54Vnd1pbod9y2aS8DDvDKZm1HJGlThdbuQ,10460
9
- tinybird/feedback_manager.py,sha256=FT5wJ06gYPGofVgpR-XdZBxUlV9oV94rLx-vzh9ArB8,67724
9
+ tinybird/feedback_manager.py,sha256=Eey5Ih84DBKspjfzIzWpZbpJkkaIqRrpPSljbm7hfNo,67822
10
10
  tinybird/git_settings.py,sha256=XUL9ZUj59-ZVQJDYmMEq4UpnuuOuQOHGlNcX3JgQHjQ,3954
11
11
  tinybird/sql.py,sha256=gfRKjdqEygcE1WOTeQ1QV2Jal8Jzl4RSX8fftu1KSEs,45825
12
12
  tinybird/sql_template.py,sha256=IqYRfUxDYBCoOYjqqvn--_8QXLv9FSRnJ0bInx7q1Xs,93051
@@ -19,17 +19,19 @@ tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj
19
19
  tinybird/ch_utils/engine.py,sha256=OXkBhlzGjZotjD0vaT-rFIbSGV4tpiHxE8qO_ip0SyQ,40454
20
20
  tinybird/tb_cli_modules/auth.py,sha256=cqxfGgFheuTmenQ3UwPBXTqwMm8JD7uzgLfoIRXdnyQ,9038
21
21
  tinybird/tb_cli_modules/branch.py,sha256=Ik8rRVPXvhyChHBqdPVNtI_C-0gLYjUHaznNWE04Ecw,39120
22
- tinybird/tb_cli_modules/build.py,sha256=pF6Ts2k0qCwCi1ktrGyJo1NcdWsU9O3okDdb3SiFdZo,7615
22
+ tinybird/tb_cli_modules/build.py,sha256=FVo3531YlGj1lHm1aI4aYepgn0ffbPfzSUM1lloY1a4,8589
23
23
  tinybird/tb_cli_modules/cicd.py,sha256=0lMkb6CVOFZl5HOwgY8mK4T4mgI7O8335UngLXtCc-c,13851
24
24
  tinybird/tb_cli_modules/cli.py,sha256=a738xz4DUNep_mzHTEUExd0NAuf7cHkUAlMsEus9xaM,65072
25
- tinybird/tb_cli_modules/common.py,sha256=pS7kisPe4vChirSY2og5e0Cu-RNDow7X3Qdk1BEHL78,78571
26
- tinybird/tb_cli_modules/config.py,sha256=6NTgIdwf0X132A1j6G_YrdPep87ymZ9b5pABabKLzh4,11484
25
+ tinybird/tb_cli_modules/common.py,sha256=VxzCcCaVoUnwvO-AN6JqeEHK9AzichzSgN_QVgqLHHM,78413
26
+ tinybird/tb_cli_modules/config.py,sha256=ppWvACHrSLkb5hOoQLYNby2w8jR76-8Kx2NBCst7ntQ,11760
27
27
  tinybird/tb_cli_modules/connection.py,sha256=YggP34Qh3hCjD41lp1ZHVCwHPjI_-um2spvEV8F_tSU,28684
28
- tinybird/tb_cli_modules/create.py,sha256=wcr-GIgCz_t135UIDFt_gNxUQ0huI3oXGM5bIuC665E,7808
28
+ tinybird/tb_cli_modules/create.py,sha256=glQiRQEVGTOw9VmoICEF5_7YWuraJrk3BSUDSTSUfP0,6796
29
29
  tinybird/tb_cli_modules/datasource.py,sha256=BVYwPkKvdnEv2YMkxof7n7FOffyJm-QK7Pk9Ui-MC3A,35816
30
30
  tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
31
31
  tinybird/tb_cli_modules/fmt.py,sha256=_xL2AyGo3us7NpmfAXtSIhn1cGAC-A4PspdhUjm58jY,3382
32
32
  tinybird/tb_cli_modules/job.py,sha256=AG69LPb9MbobA1awwJFZJvxqarDKfRlsBjw2V1zvYqc,2964
33
+ tinybird/tb_cli_modules/llm.py,sha256=epJqg4peovBS2urlyGUccmvuyy3dp57SnYkatTGcD_I,2483
34
+ tinybird/tb_cli_modules/local.py,sha256=52XgRCUqK8_07iXLLLnmP5KiAWVoXNbuCxlnobPcTQQ,5718
33
35
  tinybird/tb_cli_modules/pipe.py,sha256=BCLAQ3ZuWKGAih2mupnw_Y2S5B5cNS-epF317whsSEE,30989
34
36
  tinybird/tb_cli_modules/prompts.py,sha256=Esjhet-KUM1bT6RStk2voCCVjvAOUD4hDDiKpn-AYNY,7227
35
37
  tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
@@ -41,8 +43,8 @@ tinybird/tb_cli_modules/workspace.py,sha256=N8f1dl4BYc34ucmJ6oNZc4XZMGKd8m0Fhq7o
41
43
  tinybird/tb_cli_modules/workspace_members.py,sha256=ksXsjd233y9-sNlz4Qb-meZbX4zn1B84e_bSm2i8rhg,8731
42
44
  tinybird/tb_cli_modules/tinyunit/tinyunit.py,sha256=_ydOD4WxmKy7_h-d3fG62w_0_lD0uLl9w4EbEYtwd0U,11720
43
45
  tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py,sha256=hGh1ZaXC1af7rKnX7222urkj0QJMhMWclqMy59dOqwE,1922
44
- tinybird-0.0.1.dev2.dist-info/METADATA,sha256=C0w-b-Ur4VfPVJya8uxCZvv2cBxlgaDePZnV2m2-Y9k,2404
45
- tinybird-0.0.1.dev2.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
46
- tinybird-0.0.1.dev2.dist-info/entry_points.txt,sha256=PKPKuPmA4IfJYnCFHHUiw-aAWZuBomFvwCklv1OyCjE,43
47
- tinybird-0.0.1.dev2.dist-info/top_level.txt,sha256=pgw6AzERHBcW3YTi2PW4arjxLkulk2msOz_SomfOEuc,45
48
- tinybird-0.0.1.dev2.dist-info/RECORD,,
46
+ tinybird-0.0.1.dev4.dist-info/METADATA,sha256=LGWZlh2D8xM9K8TG6sURXP2cB5r861y8Zw6EJm1-W2s,2404
47
+ tinybird-0.0.1.dev4.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
48
+ tinybird-0.0.1.dev4.dist-info/entry_points.txt,sha256=PKPKuPmA4IfJYnCFHHUiw-aAWZuBomFvwCklv1OyCjE,43
49
+ tinybird-0.0.1.dev4.dist-info/top_level.txt,sha256=pgw6AzERHBcW3YTi2PW4arjxLkulk2msOz_SomfOEuc,45
50
+ tinybird-0.0.1.dev4.dist-info/RECORD,,