tinybird 0.0.1.dev14__tar.gz → 0.0.1.dev16__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of tinybird might be problematic. Click here for more details.

Files changed (98) hide show
  1. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/PKG-INFO +1 -1
  2. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/client.py +3 -1
  3. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/datafile.py +3 -3
  4. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/feedback_manager.py +12 -1
  5. {tinybird-0.0.1.dev14/tinybird/tb/modules → tinybird-0.0.1.dev16/tinybird}/prompts.py +26 -42
  6. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/cli.py +1 -0
  7. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/build.py +17 -93
  8. tinybird-0.0.1.dev16/tinybird/tb/modules/build_shell.py +133 -0
  9. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/cli.py +14 -21
  10. tinybird-0.0.1.dev16/tinybird/tb/modules/create.py +411 -0
  11. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/build.py +29 -21
  12. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/build_pipe.py +4 -0
  13. tinybird-0.0.1.dev16/tinybird/tb/modules/datafile/common.py +1850 -0
  14. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/parse_datasource.py +1 -0
  15. tinybird-0.0.1.dev16/tinybird/tb/modules/llm.py +95 -0
  16. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/local.py +2 -47
  17. tinybird-0.0.1.dev16/tinybird/tb/modules/local_common.py +54 -0
  18. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/login.py +0 -1
  19. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/mock.py +6 -5
  20. tinybird-0.0.1.dev16/tinybird/tb/modules/test.py +138 -0
  21. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird.egg-info/PKG-INFO +1 -1
  22. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird.egg-info/SOURCES.txt +3 -1
  23. tinybird-0.0.1.dev14/tinybird/tb/modules/create.py +0 -176
  24. tinybird-0.0.1.dev14/tinybird/tb/modules/datafile/common.py +0 -910
  25. tinybird-0.0.1.dev14/tinybird/tb/modules/llm.py +0 -73
  26. tinybird-0.0.1.dev14/tinybird/tb/modules/test.py +0 -107
  27. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/setup.cfg +0 -0
  28. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/__cli__.py +0 -0
  29. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/ch_utils/constants.py +0 -0
  30. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/ch_utils/engine.py +0 -0
  31. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/check_pypi.py +0 -0
  32. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/config.py +0 -0
  33. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/connectors.py +0 -0
  34. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/context.py +0 -0
  35. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/datatypes.py +0 -0
  36. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/git_settings.py +0 -0
  37. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/sql.py +0 -0
  38. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/sql_template.py +0 -0
  39. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/sql_template_fmt.py +0 -0
  40. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/sql_toolset.py +0 -0
  41. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/syncasync.py +0 -0
  42. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/auth.py +0 -0
  43. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/branch.py +0 -0
  44. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/cicd.py +0 -0
  45. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/common.py +0 -0
  46. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/config.py +0 -0
  47. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/connection.py +0 -0
  48. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/build_common.py +0 -0
  49. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/build_datasource.py +0 -0
  50. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/diff.py +0 -0
  51. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/exceptions.py +0 -0
  52. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/fixture.py +0 -0
  53. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/format_common.py +0 -0
  54. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/format_datasource.py +0 -0
  55. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/format_pipe.py +0 -0
  56. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/parse_pipe.py +0 -0
  57. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/pipe_checker.py +0 -0
  58. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/pull.py +0 -0
  59. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datasource.py +0 -0
  60. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/exceptions.py +0 -0
  61. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/fmt.py +0 -0
  62. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/job.py +0 -0
  63. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/pipe.py +0 -0
  64. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/regions.py +0 -0
  65. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/table.py +0 -0
  66. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/tag.py +0 -0
  67. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/telemetry.py +0 -0
  68. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/tinyunit/tinyunit.py +0 -0
  69. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -0
  70. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/token.py +0 -0
  71. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/workspace.py +0 -0
  72. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/workspace_members.py +0 -0
  73. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli.py +0 -0
  74. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/auth.py +0 -0
  75. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/branch.py +0 -0
  76. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/cicd.py +0 -0
  77. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/cli.py +0 -0
  78. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/common.py +0 -0
  79. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/config.py +0 -0
  80. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/connection.py +0 -0
  81. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/datasource.py +0 -0
  82. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/exceptions.py +0 -0
  83. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/fmt.py +0 -0
  84. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/job.py +0 -0
  85. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/pipe.py +0 -0
  86. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/regions.py +0 -0
  87. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/tag.py +0 -0
  88. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/telemetry.py +0 -0
  89. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/test.py +0 -0
  90. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
  91. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
  92. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/workspace.py +0 -0
  93. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/workspace_members.py +0 -0
  94. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tornado_template.py +0 -0
  95. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird.egg-info/dependency_links.txt +0 -0
  96. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird.egg-info/entry_points.txt +0 -0
  97. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird.egg-info/requires.txt +0 -0
  98. {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: tinybird
3
- Version: 0.0.1.dev14
3
+ Version: 0.0.1.dev16
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/cli/introduction.html
6
6
  Author: Tinybird
@@ -188,7 +188,9 @@ class TinyB:
188
188
  if response.status_code == 599:
189
189
  raise TimeoutException("timeout")
190
190
  if "Content-Type" in response.headers and (
191
- response.headers["Content-Type"] == "text/plain" or "text/csv" in response.headers["Content-Type"]
191
+ response.headers["Content-Type"] == "text/plain"
192
+ or "text/csv" in response.headers["Content-Type"]
193
+ or "ndjson" in response.headers["Content-Type"]
192
194
  ):
193
195
  return response.content.decode("utf-8")
194
196
  if response.status_code >= 400 and response.status_code not in [400, 403, 404, 409, 429]:
@@ -1051,9 +1051,9 @@ def parse(
1051
1051
  def _f(sql: str, **kwargs: Any) -> None:
1052
1052
  if not parser_state.current_node:
1053
1053
  raise ParseException("SQL must be called after a NODE command")
1054
- parser_state.current_node[var_name] = (
1055
- textwrap.dedent(sql).rstrip() if "%" not in sql.strip()[0] else sql.strip()
1056
- )
1054
+ _sql = textwrap.dedent(sql).rstrip() if "%" not in sql.strip()[0] else sql.strip()
1055
+ _sql = eval_var(_sql)
1056
+ parser_state.current_node[var_name] = _sql
1057
1057
 
1058
1058
  # HACK this cast is needed because Mypy
1059
1059
  return cast(Callable[[str, KwArg(Any)], None], _f)
@@ -1,5 +1,6 @@
1
1
  from collections import namedtuple
2
2
  from typing import Any, Callable
3
+ import os
3
4
 
4
5
  FeedbackMessage = namedtuple("FeedbackMessage", "message")
5
6
 
@@ -26,6 +27,16 @@ def print_message(message: str, color: str = bcolors.ENDC) -> Callable[..., str]
26
27
  def error_message(message: str) -> Callable[..., str]:
27
28
  return print_message(f"\n** {message}", bcolors.FAIL)
28
29
 
30
+ def error_exception(message: Exception) -> Callable[..., str]:
31
+ def formatter(**kwargs: Any) -> str:
32
+ color = bcolors.FAIL
33
+ if isinstance(kwargs.get('error', ''), Exception) and os.environ.get('TB_DEBUG', '') != '':
34
+ import traceback
35
+ stack_trace = f"Traceback:\n{''.join(traceback.format_tb(kwargs['error'].__traceback__))}"
36
+ return f"{color}{message.format(**kwargs)}{stack_trace}{bcolors.ENDC}"
37
+ return f"{color}{message.format(**kwargs)}{bcolors.ENDC}"
38
+ return formatter
39
+
29
40
 
30
41
  def simple_error_message(message: str) -> Callable[..., str]:
31
42
  return print_message(f"{message}", bcolors.FAIL)
@@ -56,7 +67,7 @@ def gray_message(message: str) -> Callable[..., str]:
56
67
 
57
68
 
58
69
  class FeedbackManager:
59
- error_exception = error_message("{error}")
70
+ error_exception = error_exception("{error}")
60
71
  simple_error_exception = simple_error_message("{error}")
61
72
  error_exception_trace = error_message("{error}\n** Trace:\n{trace}")
62
73
  error_notoken = error_message(
@@ -76,46 +76,7 @@ SQL >
76
76
  </instructions>
77
77
  """
78
78
 
79
- sample_data_prompt = """Your role is to generate sample data for a given data source. A data source is a table in Tinybird. You will be given a schema for the data source, and you will need to generate a set of rows that fit that schema.
80
-
81
- The schema will be provided to you at the end of this prompt in the following format:
82
-
83
- ```
84
- SCHEMA >
85
- `span_id` String `json:$.span_id`,
86
- `operation_id` String `json:$.operation_name`,
87
- `duration` Nullable(Float32) `json:$.duration`,
88
- ```
89
-
90
- In the example above, `span_id` are column names, String and Float32 are column types, and `json:$.duration` is a JSON path expression that points to the duration value in the JSON data. Use this information to generate valid data.
91
-
92
- You will generate {row_count} rows of data in JSON format, newlines separated. Your reponse will only contain the {row_count} rows of data, without any other text. Do not format the data in any way, and do not place it inside any markers for a code block of any kind. Most importantly, do not use backticks on the first or last line or anywhere else in the response. This is extremely important, ensure that your response doesn't include any backticks.
93
-
94
- Remember that for a JSON to be valid, there can't be a trailing comma after the last value in an object.
95
-
96
- Make sure to include all columns in the schema, and use valid JSON. You may leave some columns empty only if the column type allows it (e.g. Nullable(String)). Use realistic values for the data, and make sure to include a variety of values in each column.
97
-
98
- For columns that represent timestamps, use values that are close to the current date and time. The current date and time is {current_datetime}"""
99
-
100
- sample_data_with_errors_prompt = """Your role is to generate incorrect sample data for a given data source. A data source is a table in Tinybird. You will be given a schema for the data source, and you will need to generate a set of rows that mostly fit that schema, but with some errors.
101
-
102
- The schema will be provided to you at the end of this prompt in the following format:
103
-
104
- ```
105
- SCHEMA >
106
- `span_id` String `json:$.span_id`,
107
- `operation_id` String `json:$.operation_name`,
108
- `duration` Nullable(Float32) `json:$.duration`,
109
- ```
110
-
111
- In the example above, `span_id` are column names, String and Float32 are column types, and `json:$.duration` is a JSON path expression that points to the duration value in the JSON data. Use this information to generate valid data.
112
-
113
- You will generate {row_count} rows of data in JSON format, newlines separated. Your reponse will be a single string containing all {row_count} rows, without any other text. Make sure to include all columns in the schema, and use valid JSON. Use realistic values for the data, and make sure to include a variety of values in each column. In every set of {row_count} rows, there should be at least one row that has an error. The error could be in any column, and it could be a type error (e.g. a string where a number is expected), or it could be an error in the structure of the JSON (e.g. a value in the wrong place in the document).
114
-
115
- For columns that represent timestamps, use values that are close to the current date and time. The current date and time is {current_datetime}"""
116
-
117
-
118
- sample_data_sql_prompt = """
79
+ generate_sql_mock_data_prompt = """
119
80
  Given the schema for a Tinybird datasource, return a can you create a clickhouse sql query to generate some random data that matches that schema.
120
81
 
121
82
  Response format MUST be just a valid clickhouse sql query.
@@ -142,13 +103,13 @@ SELECT
142
103
  rand() % 50 AS pvp_kills, -- Random PvP kills between 0 and 49
143
104
  rand() % 200 AS quest_completions, -- Random quest completions between 0 and 199
144
105
  now() - rand() % 86400 AS timestamp -- Random timestamp within the last day
145
- FROM numbers({row_count})
106
+ FROM numbers({rows})
146
107
 
147
108
  # Instructions:
148
109
 
149
110
  - The query MUST return a random sample of data that matches the schema.
150
111
  - The query MUST return a valid clickhouse sql query.
151
- - The query MUST return a sample of EXACTLY {row_count} rows.
112
+ - The query MUST return a sample of EXACTLY {rows} rows.
152
113
  - The query MUST be valid for clickhouse and Tinybird.
153
114
  - Return JUST the sql query, without any other text or symbols.
154
115
  - Do NOT include ```clickhouse or ```sql or any other wrapping text.
@@ -160,3 +121,26 @@ FROM numbers({row_count})
160
121
  {context}
161
122
 
162
123
  """
124
+
125
+ create_test_calls_prompt = """
126
+ You are a Tinybird expert. You will be given a pipe endpoint containing different nodes with SQL and Tinybird templating syntax. You will generate URLs to test it with different parameters combinations.
127
+
128
+ <test>
129
+ <test_1>:
130
+ name: <test_name_1>
131
+ description: <description_1>
132
+ parameters: <url_encoded_parameters_1>
133
+ <test_2>:
134
+ name: <test_name_2>
135
+ description: <description_2>
136
+ parameters: <url_encoded_parameters_2>
137
+ </test>
138
+ <instructions>
139
+ - The test name must be unique.
140
+ - The test command must be a valid Tinybird command that can be run in the terminal.
141
+ - The test command can have as many parameters as are needed to test the pipe.
142
+ - The parameter within Tinybird templating syntax looks like this one {{String(my_param_name, default_value)}}.
143
+ - If there are no parameters in the , you can omit parametrs and generate a single test command.
144
+ - Extra context: {context}
145
+ </instructions>
146
+ """
@@ -14,6 +14,7 @@ import tinybird.tb.modules.create
14
14
  import tinybird.tb.modules.datasource
15
15
  import tinybird.tb.modules.fmt
16
16
  import tinybird.tb.modules.job
17
+ import tinybird.tb.modules.local
17
18
  import tinybird.tb.modules.login
18
19
  import tinybird.tb.modules.mock
19
20
  import tinybird.tb.modules.pipe
@@ -1,23 +1,20 @@
1
1
  import asyncio
2
- import cmd
3
2
  import os
4
- import random
5
- import subprocess
6
- import sys
7
3
  import threading
8
4
  import time
9
5
  from pathlib import Path
10
6
  from typing import Any, Awaitable, Callable, List, Union
11
7
 
12
8
  import click
13
- import humanfriendly
9
+ from click import Context
14
10
  from watchdog.events import FileSystemEventHandler
15
11
  from watchdog.observers import Observer
16
12
 
17
13
  import tinybird.context as context
18
14
  from tinybird.client import TinyB
19
15
  from tinybird.config import FeatureFlags
20
- from tinybird.feedback_manager import FeedbackManager, bcolors
16
+ from tinybird.feedback_manager import FeedbackManager
17
+ from tinybird.tb.modules.build_shell import BuildShell, print_table_formatted
21
18
  from tinybird.tb.modules.cli import cli
22
19
  from tinybird.tb.modules.common import coro, push_data
23
20
  from tinybird.tb.modules.datafile.build import folder_build
@@ -26,44 +23,7 @@ from tinybird.tb.modules.datafile.exceptions import ParseException
26
23
  from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
27
24
  from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
28
25
  from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
29
- from tinybird.tb.modules.local import get_tinybird_local_client
30
- from tinybird.tb.modules.table import format_table
31
-
32
-
33
- class BuildShell(cmd.Cmd):
34
- prompt = "\n\001\033[1;32m\002TB > \001\033[0m\002"
35
-
36
- def __init__(self, folder: str):
37
- super().__init__()
38
- self.folder = folder
39
-
40
- def do_exit(self, arg):
41
- sys.exit(0)
42
-
43
- def do_quit(self, arg):
44
- sys.exit(0)
45
-
46
- def default(self, argline):
47
- click.echo("")
48
- if argline.startswith("tb build"):
49
- click.echo(FeedbackManager.error(message="Build command is already running"))
50
- else:
51
- arg_stripped = argline.strip().lower()
52
- if not arg_stripped:
53
- return
54
- if arg_stripped.startswith("tb"):
55
- extra_args = f" --folder {self.folder}" if arg_stripped.startswith("tb mock") else ""
56
- subprocess.run(arg_stripped + extra_args, shell=True, text=True)
57
- elif arg_stripped.startswith("with") or arg_stripped.startswith("select"):
58
- subprocess.run(f'tb sql "{arg_stripped}"', shell=True, text=True)
59
- elif arg_stripped.startswith("mock "):
60
- subprocess.run(f"tb {arg_stripped} --folder {self.folder}", shell=True, text=True)
61
- else:
62
- click.echo(FeedbackManager.error(message="Invalid command"))
63
-
64
- def reprint_prompt(self):
65
- self.stdout.write(self.prompt)
66
- self.stdout.flush()
26
+ from tinybird.tb.modules.local_common import get_tinybird_local_client
67
27
 
68
28
 
69
29
  class FileChangeHandler(FileSystemEventHandler):
@@ -72,7 +32,12 @@ class FileChangeHandler(FileSystemEventHandler):
72
32
  self.process = process
73
33
 
74
34
  def on_modified(self, event: Any) -> None:
75
- if not event.is_directory and any(event.src_path.endswith(ext) for ext in [".datasource", ".pipe", ".ndjson"]):
35
+ is_not_vendor = "vendor/" not in event.src_path
36
+ if (
37
+ is_not_vendor
38
+ and not event.is_directory
39
+ and any(event.src_path.endswith(ext) for ext in [".datasource", ".pipe", ".ndjson"])
40
+ ):
76
41
  filename = event.src_path.split("/")[-1]
77
42
  click.echo(FeedbackManager.highlight(message=f"\n\n⟲ Changes detected in {filename}\n"))
78
43
  try:
@@ -132,16 +97,12 @@ def watch_files(
132
97
  is_flag=True,
133
98
  help="Watch for changes in the files and re-check them.",
134
99
  )
135
- @click.option(
136
- "--skip-datasources",
137
- is_flag=True,
138
- help="Skip rebuilding datasources.",
139
- )
100
+ @click.pass_context
140
101
  @coro
141
102
  async def build(
103
+ ctx: Context,
142
104
  folder: str,
143
105
  watch: bool,
144
- skip_datasources: bool,
145
106
  ) -> None:
146
107
  """
147
108
  Watch for changes in the files and re-check them.
@@ -171,7 +132,7 @@ async def build(
171
132
 
172
133
  parser(filename)
173
134
 
174
- async def process(filenames: List[str], watch: bool = False, only_pipes: bool = False):
135
+ async def process(filenames: List[str], watch: bool = False):
175
136
  datafiles = [f for f in filenames if f.endswith(".datasource") or f.endswith(".pipe")]
176
137
  if len(datafiles) > 0:
177
138
  check_filenames(filenames=datafiles)
@@ -180,7 +141,7 @@ async def build(
180
141
  filenames=datafiles,
181
142
  ignore_sql_errors=ignore_sql_errors,
182
143
  is_internal=is_internal,
183
- only_pipes=only_pipes,
144
+ watch=watch,
184
145
  )
185
146
 
186
147
  filename = filenames[0]
@@ -209,7 +170,7 @@ async def build(
209
170
  try:
210
171
  click.echo("⚡ Building project...\n")
211
172
  time_start = time.time()
212
- await process(filenames=filenames, watch=False, only_pipes=skip_datasources)
173
+ await process(filenames=filenames, watch=False)
213
174
  time_end = time.time()
214
175
  elapsed_time = time_end - time_start
215
176
  for filename in filenames:
@@ -226,7 +187,7 @@ async def build(
226
187
  await build_once(filenames)
227
188
 
228
189
  if watch:
229
- shell = BuildShell(folder=folder)
190
+ shell = BuildShell(folder=folder, client=tb_client)
230
191
  click.echo(FeedbackManager.highlight(message="◎ Watching for changes..."))
231
192
  watcher_thread = threading.Thread(target=watch_files, args=(filenames, process, shell, folder), daemon=True)
232
193
  watcher_thread.start()
@@ -234,48 +195,11 @@ async def build(
234
195
 
235
196
 
236
197
  async def build_and_print_resource(tb_client: TinyB, filename: str):
237
- rebuild_colors = [bcolors.FAIL, bcolors.OKBLUE, bcolors.WARNING, bcolors.OKGREEN, bcolors.HEADER]
238
- rebuild_index = random.randint(0, len(rebuild_colors) - 1)
239
- rebuild_color = rebuild_colors[rebuild_index % len(rebuild_colors)]
240
198
  resource_path = Path(filename)
241
199
  name = resource_path.stem
242
200
  pipeline = name if filename.endswith(".pipe") else None
243
201
  res = await tb_client.query(f"SELECT * FROM {name} FORMAT JSON", pipeline=pipeline)
244
- data = []
245
- limit = 5
246
- for d in res["data"][:5]:
247
- data.append(d.values())
248
- meta = res["meta"]
249
- row_count = res.get("rows", 0)
250
- stats = res.get("statistics", {})
251
- elapsed = stats.get("elapsed", 0)
252
- cols = len(meta)
253
- try:
254
-
255
- def print_message(message: str, color=bcolors.CGREY):
256
- return f"{color}{message}{bcolors.ENDC}"
257
-
258
- table = format_table(data, meta)
259
- colored_char = print_message("│", rebuild_color)
260
- table_with_marker = "\n".join(f"{colored_char} {line}" for line in table.split("\n"))
261
- click.echo(f"\n{colored_char} {print_message('⚡', rebuild_color)} Running {name}")
262
- click.echo(colored_char)
263
- click.echo(table_with_marker)
264
- click.echo(colored_char)
265
- rows_read = humanfriendly.format_number(stats.get("rows_read", 0))
266
- bytes_read = humanfriendly.format_size(stats.get("bytes_read", 0))
267
- elapsed = humanfriendly.format_timespan(elapsed) if elapsed >= 1 else f"{elapsed * 1000:.2f}ms"
268
- stats_message = f"» {bytes_read} ({rows_read} rows x {cols} cols) in {elapsed}"
269
- rows_message = f"» Showing {limit} first rows" if row_count > limit else "» Showing all rows"
270
- click.echo(f"{colored_char} {print_message(stats_message, bcolors.OKGREEN)}")
271
- click.echo(f"{colored_char} {print_message(rows_message, bcolors.CGREY)}")
272
- except ValueError as exc:
273
- if str(exc) == "max() arg is an empty sequence":
274
- click.echo("------------")
275
- click.echo("Empty")
276
- click.echo("------------")
277
- else:
278
- raise exc
202
+ print_table_formatted(res, name)
279
203
 
280
204
 
281
205
  async def append_datasource(
@@ -0,0 +1,133 @@
1
+ import asyncio
2
+ import cmd
3
+ import random
4
+ import subprocess
5
+ import sys
6
+
7
+ import click
8
+ import humanfriendly
9
+
10
+ from tinybird.client import TinyB
11
+ from tinybird.feedback_manager import FeedbackManager, bcolors
12
+ from tinybird.tb.modules.exceptions import CLIException
13
+ from tinybird.tb.modules.table import format_table
14
+
15
+
16
+ class BuildShell(cmd.Cmd):
17
+ prompt = "\n\001\033[1;32m\002TB > \001\033[0m\002"
18
+
19
+ def __init__(self, folder: str, client: TinyB):
20
+ super().__init__()
21
+ self.folder = folder
22
+ self.client = client
23
+
24
+ def do_exit(self, arg):
25
+ sys.exit(0)
26
+
27
+ def do_quit(self, arg):
28
+ sys.exit(0)
29
+
30
+ def default(self, argline):
31
+ click.echo("")
32
+ if argline.startswith("tb build"):
33
+ click.echo(FeedbackManager.error(message="Build command is already running"))
34
+ else:
35
+ arg_stripped = argline.strip().lower()
36
+ if not arg_stripped:
37
+ return
38
+ if arg_stripped.startswith("tb"):
39
+ arg_stripped = arg_stripped.replace("tb", "tb --local")
40
+ extra_args = f" --folder {self.folder}" if arg_stripped.startswith("tb mock") else ""
41
+ subprocess.run(arg_stripped + extra_args, shell=True, text=True)
42
+ elif arg_stripped.startswith("with") or arg_stripped.startswith("select"):
43
+ try:
44
+ self.run_sql(self.client, argline)
45
+ except Exception as e:
46
+ click.echo(FeedbackManager.error(message=str(e)))
47
+
48
+ elif arg_stripped.startswith("mock "):
49
+ subprocess.run(f"tb {arg_stripped} --folder {self.folder}", shell=True, text=True)
50
+ else:
51
+ click.echo(FeedbackManager.error(message="Invalid command"))
52
+
53
+ def reprint_prompt(self):
54
+ self.stdout.write(self.prompt)
55
+ self.stdout.flush()
56
+
57
+ def run_sql(self, query, rows_limit=20):
58
+ try:
59
+ q = query.strip()
60
+ if q.startswith("insert"):
61
+ click.echo(FeedbackManager.info_append_data())
62
+ raise CLIException(FeedbackManager.error_invalid_query())
63
+ if q.startswith("delete"):
64
+ raise CLIException(FeedbackManager.error_invalid_query())
65
+
66
+ # fuck my life
67
+ def run_query_in_thread():
68
+ loop = asyncio.new_event_loop()
69
+ asyncio.set_event_loop(loop)
70
+ try:
71
+ return loop.run_until_complete(
72
+ self.client.query(f"SELECT * FROM ({query}) LIMIT {rows_limit} FORMAT JSON")
73
+ )
74
+ finally:
75
+ loop.close()
76
+
77
+ # Run the query in a separate thread
78
+ import concurrent.futures
79
+
80
+ with concurrent.futures.ThreadPoolExecutor() as executor:
81
+ res = executor.submit(run_query_in_thread).result()
82
+
83
+ except Exception as e:
84
+ raise CLIException(FeedbackManager.error_exception(error=str(e)))
85
+
86
+ if isinstance(res, dict) and "error" in res:
87
+ raise CLIException(FeedbackManager.error_exception(error=res["error"]))
88
+
89
+ if isinstance(res, dict) and "data" in res and res["data"]:
90
+ print_table_formatted(res, "QUERY")
91
+ else:
92
+ click.echo(FeedbackManager.info_no_rows())
93
+
94
+
95
+ def print_table_formatted(res: dict, name: str):
96
+ rebuild_colors = [bcolors.FAIL, bcolors.OKBLUE, bcolors.WARNING, bcolors.OKGREEN, bcolors.HEADER]
97
+ rebuild_index = random.randint(0, len(rebuild_colors) - 1)
98
+ rebuild_color = rebuild_colors[rebuild_index % len(rebuild_colors)]
99
+ data = []
100
+ limit = 5
101
+ for d in res["data"][:5]:
102
+ data.append(d.values())
103
+ meta = res["meta"]
104
+ row_count = res.get("rows", 0)
105
+ stats = res.get("statistics", {})
106
+ elapsed = stats.get("elapsed", 0)
107
+ cols = len(meta)
108
+ try:
109
+
110
+ def print_message(message: str, color=bcolors.CGREY):
111
+ return f"{color}{message}{bcolors.ENDC}"
112
+
113
+ table = format_table(data, meta)
114
+ colored_char = print_message("│", rebuild_color)
115
+ table_with_marker = "\n".join(f"{colored_char} {line}" for line in table.split("\n"))
116
+ click.echo(f"\n{colored_char} {print_message('⚡', rebuild_color)} Running {name}")
117
+ click.echo(colored_char)
118
+ click.echo(table_with_marker)
119
+ click.echo(colored_char)
120
+ rows_read = humanfriendly.format_number(stats.get("rows_read", 0))
121
+ bytes_read = humanfriendly.format_size(stats.get("bytes_read", 0))
122
+ elapsed = humanfriendly.format_timespan(elapsed) if elapsed >= 1 else f"{elapsed * 1000:.2f}ms"
123
+ stats_message = f"» {bytes_read} ({rows_read} rows x {cols} cols) in {elapsed}"
124
+ rows_message = f"» Showing {limit} first rows" if row_count > limit else "» Showing all rows"
125
+ click.echo(f"{colored_char} {print_message(stats_message, bcolors.OKGREEN)}")
126
+ click.echo(f"{colored_char} {print_message(rows_message, bcolors.CGREY)}")
127
+ except ValueError as exc:
128
+ if str(exc) == "max() arg is an empty sequence":
129
+ click.echo("------------")
130
+ click.echo("Empty")
131
+ click.echo("------------")
132
+ else:
133
+ raise exc
@@ -10,7 +10,7 @@ import pprint
10
10
  import re
11
11
  import shutil
12
12
  import sys
13
- from os import environ, getcwd
13
+ from os import getcwd
14
14
  from pathlib import Path
15
15
  from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
16
16
 
@@ -26,7 +26,7 @@ from tinybird.client import (
26
26
  DoesNotExistException,
27
27
  TinyB,
28
28
  )
29
- from tinybird.config import CURRENT_VERSION, SUPPORTED_CONNECTORS, VERSION, FeatureFlags, get_config
29
+ from tinybird.config import SUPPORTED_CONNECTORS, VERSION, FeatureFlags, get_config
30
30
  from tinybird.feedback_manager import FeedbackManager
31
31
  from tinybird.tb.modules.common import (
32
32
  OLDEST_ROLLBACK,
@@ -50,6 +50,7 @@ from tinybird.tb.modules.config import CLIConfig
50
50
  from tinybird.tb.modules.datafile.build import build_graph, folder_push
51
51
  from tinybird.tb.modules.datafile.common import (
52
52
  Datafile,
53
+ DatafileSyntaxError,
53
54
  get_project_filenames,
54
55
  get_resource_versions,
55
56
  has_internal_datafiles,
@@ -62,6 +63,7 @@ from tinybird.tb.modules.datafile.exceptions import (
62
63
  from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
63
64
  from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
64
65
  from tinybird.tb.modules.datafile.pull import folder_pull
66
+ from tinybird.tb.modules.local_common import get_tinybird_local_client
65
67
  from tinybird.tb.modules.telemetry import add_telemetry_event
66
68
 
67
69
  __old_click_echo = click.echo
@@ -106,13 +108,8 @@ DEFAULT_PATTERNS: List[Tuple[str, Union[str, Callable[[str], str]]]] = [
106
108
  @click.option(
107
109
  "--with-headers", help="Flag to enable connector to export with headers", is_flag=True, default=False, hidden=True
108
110
  )
109
- @click.option(
110
- "--version-warning/--no-version-warning",
111
- envvar="TB_VERSION_WARNING",
112
- default=True,
113
- help="Don't print version warning message if there's a new available version. You can use TB_VERSION_WARNING envar",
114
- )
115
111
  @click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
112
+ @click.option("--local", is_flag=True, default=False, help="Run in local mode")
116
113
  @click.version_option(version=VERSION)
117
114
  @click.pass_context
118
115
  @coro
@@ -135,8 +132,8 @@ async def cli(
135
132
  sf_storage_integration: str,
136
133
  sf_stage,
137
134
  with_headers: bool,
138
- version_warning: bool,
139
135
  show_tokens: bool,
136
+ local: bool,
140
137
  ) -> None:
141
138
  """
142
139
  Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
@@ -151,17 +148,6 @@ async def cli(
151
148
  if getenv_bool("TB_DISABLE_SSL_CHECKS", False):
152
149
  click.echo(FeedbackManager.warning_disabled_ssl_checks())
153
150
 
154
- # ensure that ctx.obj exists and is a dict (in case `cli()` is called)
155
- # by means other than the `if` block below
156
- if not environ.get("PYTEST", None) and version_warning and not token:
157
- from tinybird.check_pypi import CheckPypi
158
-
159
- latest_version = await CheckPypi().get_latest_version()
160
-
161
- if "x.y.z" not in CURRENT_VERSION and latest_version != CURRENT_VERSION:
162
- click.echo(FeedbackManager.warning_update_version(latest_version=latest_version))
163
- click.echo(FeedbackManager.warning_current_version(current_version=CURRENT_VERSION))
164
-
165
151
  if debug:
166
152
  logging.basicConfig(level=logging.DEBUG)
167
153
 
@@ -245,7 +231,11 @@ async def cli(
245
231
 
246
232
  logging.debug("debug enabled")
247
233
 
248
- ctx.ensure_object(dict)["client"] = _get_tb_client(config.get("token", None), config["host"], semver)
234
+ ctx.ensure_object(dict)["client"] = (
235
+ await get_tinybird_local_client()
236
+ if local
237
+ else _get_tb_client(config.get("token", None), config["host"], semver)
238
+ )
249
239
 
250
240
  for connector in SUPPORTED_CONNECTORS:
251
241
  load_connector_config(ctx, connector, debug, check_uninstalled=True)
@@ -361,6 +351,9 @@ def check(filenames: List[str], debug: bool) -> None:
361
351
  for x in doc.nodes:
362
352
  pp.pprint(x)
363
353
 
354
+ except DatafileSyntaxError as e:
355
+ # TODO(eclbg): add the filename to the error message
356
+ raise CLIException(e)
364
357
  except ParseException as e:
365
358
  raise CLIException(FeedbackManager.error_exception(error=e))
366
359