tinybird 0.0.1.dev14__tar.gz → 0.0.1.dev16__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/PKG-INFO +1 -1
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/client.py +3 -1
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/datafile.py +3 -3
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/feedback_manager.py +12 -1
- {tinybird-0.0.1.dev14/tinybird/tb/modules → tinybird-0.0.1.dev16/tinybird}/prompts.py +26 -42
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/cli.py +1 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/build.py +17 -93
- tinybird-0.0.1.dev16/tinybird/tb/modules/build_shell.py +133 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/cli.py +14 -21
- tinybird-0.0.1.dev16/tinybird/tb/modules/create.py +411 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/build.py +29 -21
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/build_pipe.py +4 -0
- tinybird-0.0.1.dev16/tinybird/tb/modules/datafile/common.py +1850 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/parse_datasource.py +1 -0
- tinybird-0.0.1.dev16/tinybird/tb/modules/llm.py +95 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/local.py +2 -47
- tinybird-0.0.1.dev16/tinybird/tb/modules/local_common.py +54 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/login.py +0 -1
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/mock.py +6 -5
- tinybird-0.0.1.dev16/tinybird/tb/modules/test.py +138 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird.egg-info/PKG-INFO +1 -1
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird.egg-info/SOURCES.txt +3 -1
- tinybird-0.0.1.dev14/tinybird/tb/modules/create.py +0 -176
- tinybird-0.0.1.dev14/tinybird/tb/modules/datafile/common.py +0 -910
- tinybird-0.0.1.dev14/tinybird/tb/modules/llm.py +0 -73
- tinybird-0.0.1.dev14/tinybird/tb/modules/test.py +0 -107
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/setup.cfg +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/__cli__.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/ch_utils/constants.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/ch_utils/engine.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/check_pypi.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/config.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/connectors.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/context.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/datatypes.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/git_settings.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/sql.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/sql_template.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/sql_template_fmt.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/sql_toolset.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/syncasync.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/auth.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/branch.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/cicd.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/common.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/config.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/connection.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/build_common.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/build_datasource.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/diff.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/exceptions.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/fixture.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/format_common.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/format_datasource.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/format_pipe.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/parse_pipe.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/pipe_checker.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datafile/pull.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/datasource.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/exceptions.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/fmt.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/job.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/pipe.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/regions.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/table.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/tag.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/telemetry.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/tinyunit/tinyunit.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/token.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/workspace.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb/modules/workspace_members.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/auth.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/branch.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/cicd.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/cli.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/common.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/config.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/connection.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/datasource.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/exceptions.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/fmt.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/job.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/pipe.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/regions.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/tag.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/telemetry.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/test.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/workspace.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tb_cli_modules/workspace_members.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird/tornado_template.py +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird.egg-info/dependency_links.txt +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird.egg-info/entry_points.txt +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird.egg-info/requires.txt +0 -0
- {tinybird-0.0.1.dev14 → tinybird-0.0.1.dev16}/tinybird.egg-info/top_level.txt +0 -0
|
@@ -188,7 +188,9 @@ class TinyB:
|
|
|
188
188
|
if response.status_code == 599:
|
|
189
189
|
raise TimeoutException("timeout")
|
|
190
190
|
if "Content-Type" in response.headers and (
|
|
191
|
-
response.headers["Content-Type"] == "text/plain"
|
|
191
|
+
response.headers["Content-Type"] == "text/plain"
|
|
192
|
+
or "text/csv" in response.headers["Content-Type"]
|
|
193
|
+
or "ndjson" in response.headers["Content-Type"]
|
|
192
194
|
):
|
|
193
195
|
return response.content.decode("utf-8")
|
|
194
196
|
if response.status_code >= 400 and response.status_code not in [400, 403, 404, 409, 429]:
|
|
@@ -1051,9 +1051,9 @@ def parse(
|
|
|
1051
1051
|
def _f(sql: str, **kwargs: Any) -> None:
|
|
1052
1052
|
if not parser_state.current_node:
|
|
1053
1053
|
raise ParseException("SQL must be called after a NODE command")
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1054
|
+
_sql = textwrap.dedent(sql).rstrip() if "%" not in sql.strip()[0] else sql.strip()
|
|
1055
|
+
_sql = eval_var(_sql)
|
|
1056
|
+
parser_state.current_node[var_name] = _sql
|
|
1057
1057
|
|
|
1058
1058
|
# HACK this cast is needed because Mypy
|
|
1059
1059
|
return cast(Callable[[str, KwArg(Any)], None], _f)
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from collections import namedtuple
|
|
2
2
|
from typing import Any, Callable
|
|
3
|
+
import os
|
|
3
4
|
|
|
4
5
|
FeedbackMessage = namedtuple("FeedbackMessage", "message")
|
|
5
6
|
|
|
@@ -26,6 +27,16 @@ def print_message(message: str, color: str = bcolors.ENDC) -> Callable[..., str]
|
|
|
26
27
|
def error_message(message: str) -> Callable[..., str]:
|
|
27
28
|
return print_message(f"\n** {message}", bcolors.FAIL)
|
|
28
29
|
|
|
30
|
+
def error_exception(message: Exception) -> Callable[..., str]:
|
|
31
|
+
def formatter(**kwargs: Any) -> str:
|
|
32
|
+
color = bcolors.FAIL
|
|
33
|
+
if isinstance(kwargs.get('error', ''), Exception) and os.environ.get('TB_DEBUG', '') != '':
|
|
34
|
+
import traceback
|
|
35
|
+
stack_trace = f"Traceback:\n{''.join(traceback.format_tb(kwargs['error'].__traceback__))}"
|
|
36
|
+
return f"{color}{message.format(**kwargs)}{stack_trace}{bcolors.ENDC}"
|
|
37
|
+
return f"{color}{message.format(**kwargs)}{bcolors.ENDC}"
|
|
38
|
+
return formatter
|
|
39
|
+
|
|
29
40
|
|
|
30
41
|
def simple_error_message(message: str) -> Callable[..., str]:
|
|
31
42
|
return print_message(f"{message}", bcolors.FAIL)
|
|
@@ -56,7 +67,7 @@ def gray_message(message: str) -> Callable[..., str]:
|
|
|
56
67
|
|
|
57
68
|
|
|
58
69
|
class FeedbackManager:
|
|
59
|
-
error_exception =
|
|
70
|
+
error_exception = error_exception("{error}")
|
|
60
71
|
simple_error_exception = simple_error_message("{error}")
|
|
61
72
|
error_exception_trace = error_message("{error}\n** Trace:\n{trace}")
|
|
62
73
|
error_notoken = error_message(
|
|
@@ -76,46 +76,7 @@ SQL >
|
|
|
76
76
|
</instructions>
|
|
77
77
|
"""
|
|
78
78
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
The schema will be provided to you at the end of this prompt in the following format:
|
|
82
|
-
|
|
83
|
-
```
|
|
84
|
-
SCHEMA >
|
|
85
|
-
`span_id` String `json:$.span_id`,
|
|
86
|
-
`operation_id` String `json:$.operation_name`,
|
|
87
|
-
`duration` Nullable(Float32) `json:$.duration`,
|
|
88
|
-
```
|
|
89
|
-
|
|
90
|
-
In the example above, `span_id` are column names, String and Float32 are column types, and `json:$.duration` is a JSON path expression that points to the duration value in the JSON data. Use this information to generate valid data.
|
|
91
|
-
|
|
92
|
-
You will generate {row_count} rows of data in JSON format, newlines separated. Your reponse will only contain the {row_count} rows of data, without any other text. Do not format the data in any way, and do not place it inside any markers for a code block of any kind. Most importantly, do not use backticks on the first or last line or anywhere else in the response. This is extremely important, ensure that your response doesn't include any backticks.
|
|
93
|
-
|
|
94
|
-
Remember that for a JSON to be valid, there can't be a trailing comma after the last value in an object.
|
|
95
|
-
|
|
96
|
-
Make sure to include all columns in the schema, and use valid JSON. You may leave some columns empty only if the column type allows it (e.g. Nullable(String)). Use realistic values for the data, and make sure to include a variety of values in each column.
|
|
97
|
-
|
|
98
|
-
For columns that represent timestamps, use values that are close to the current date and time. The current date and time is {current_datetime}"""
|
|
99
|
-
|
|
100
|
-
sample_data_with_errors_prompt = """Your role is to generate incorrect sample data for a given data source. A data source is a table in Tinybird. You will be given a schema for the data source, and you will need to generate a set of rows that mostly fit that schema, but with some errors.
|
|
101
|
-
|
|
102
|
-
The schema will be provided to you at the end of this prompt in the following format:
|
|
103
|
-
|
|
104
|
-
```
|
|
105
|
-
SCHEMA >
|
|
106
|
-
`span_id` String `json:$.span_id`,
|
|
107
|
-
`operation_id` String `json:$.operation_name`,
|
|
108
|
-
`duration` Nullable(Float32) `json:$.duration`,
|
|
109
|
-
```
|
|
110
|
-
|
|
111
|
-
In the example above, `span_id` are column names, String and Float32 are column types, and `json:$.duration` is a JSON path expression that points to the duration value in the JSON data. Use this information to generate valid data.
|
|
112
|
-
|
|
113
|
-
You will generate {row_count} rows of data in JSON format, newlines separated. Your reponse will be a single string containing all {row_count} rows, without any other text. Make sure to include all columns in the schema, and use valid JSON. Use realistic values for the data, and make sure to include a variety of values in each column. In every set of {row_count} rows, there should be at least one row that has an error. The error could be in any column, and it could be a type error (e.g. a string where a number is expected), or it could be an error in the structure of the JSON (e.g. a value in the wrong place in the document).
|
|
114
|
-
|
|
115
|
-
For columns that represent timestamps, use values that are close to the current date and time. The current date and time is {current_datetime}"""
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
sample_data_sql_prompt = """
|
|
79
|
+
generate_sql_mock_data_prompt = """
|
|
119
80
|
Given the schema for a Tinybird datasource, return a can you create a clickhouse sql query to generate some random data that matches that schema.
|
|
120
81
|
|
|
121
82
|
Response format MUST be just a valid clickhouse sql query.
|
|
@@ -142,13 +103,13 @@ SELECT
|
|
|
142
103
|
rand() % 50 AS pvp_kills, -- Random PvP kills between 0 and 49
|
|
143
104
|
rand() % 200 AS quest_completions, -- Random quest completions between 0 and 199
|
|
144
105
|
now() - rand() % 86400 AS timestamp -- Random timestamp within the last day
|
|
145
|
-
FROM numbers({
|
|
106
|
+
FROM numbers({rows})
|
|
146
107
|
|
|
147
108
|
# Instructions:
|
|
148
109
|
|
|
149
110
|
- The query MUST return a random sample of data that matches the schema.
|
|
150
111
|
- The query MUST return a valid clickhouse sql query.
|
|
151
|
-
- The query MUST return a sample of EXACTLY {
|
|
112
|
+
- The query MUST return a sample of EXACTLY {rows} rows.
|
|
152
113
|
- The query MUST be valid for clickhouse and Tinybird.
|
|
153
114
|
- Return JUST the sql query, without any other text or symbols.
|
|
154
115
|
- Do NOT include ```clickhouse or ```sql or any other wrapping text.
|
|
@@ -160,3 +121,26 @@ FROM numbers({row_count})
|
|
|
160
121
|
{context}
|
|
161
122
|
|
|
162
123
|
"""
|
|
124
|
+
|
|
125
|
+
create_test_calls_prompt = """
|
|
126
|
+
You are a Tinybird expert. You will be given a pipe endpoint containing different nodes with SQL and Tinybird templating syntax. You will generate URLs to test it with different parameters combinations.
|
|
127
|
+
|
|
128
|
+
<test>
|
|
129
|
+
<test_1>:
|
|
130
|
+
name: <test_name_1>
|
|
131
|
+
description: <description_1>
|
|
132
|
+
parameters: <url_encoded_parameters_1>
|
|
133
|
+
<test_2>:
|
|
134
|
+
name: <test_name_2>
|
|
135
|
+
description: <description_2>
|
|
136
|
+
parameters: <url_encoded_parameters_2>
|
|
137
|
+
</test>
|
|
138
|
+
<instructions>
|
|
139
|
+
- The test name must be unique.
|
|
140
|
+
- The test command must be a valid Tinybird command that can be run in the terminal.
|
|
141
|
+
- The test command can have as many parameters as are needed to test the pipe.
|
|
142
|
+
- The parameter within Tinybird templating syntax looks like this one {{String(my_param_name, default_value)}}.
|
|
143
|
+
- If there are no parameters in the , you can omit parametrs and generate a single test command.
|
|
144
|
+
- Extra context: {context}
|
|
145
|
+
</instructions>
|
|
146
|
+
"""
|
|
@@ -14,6 +14,7 @@ import tinybird.tb.modules.create
|
|
|
14
14
|
import tinybird.tb.modules.datasource
|
|
15
15
|
import tinybird.tb.modules.fmt
|
|
16
16
|
import tinybird.tb.modules.job
|
|
17
|
+
import tinybird.tb.modules.local
|
|
17
18
|
import tinybird.tb.modules.login
|
|
18
19
|
import tinybird.tb.modules.mock
|
|
19
20
|
import tinybird.tb.modules.pipe
|
|
@@ -1,23 +1,20 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
-
import cmd
|
|
3
2
|
import os
|
|
4
|
-
import random
|
|
5
|
-
import subprocess
|
|
6
|
-
import sys
|
|
7
3
|
import threading
|
|
8
4
|
import time
|
|
9
5
|
from pathlib import Path
|
|
10
6
|
from typing import Any, Awaitable, Callable, List, Union
|
|
11
7
|
|
|
12
8
|
import click
|
|
13
|
-
import
|
|
9
|
+
from click import Context
|
|
14
10
|
from watchdog.events import FileSystemEventHandler
|
|
15
11
|
from watchdog.observers import Observer
|
|
16
12
|
|
|
17
13
|
import tinybird.context as context
|
|
18
14
|
from tinybird.client import TinyB
|
|
19
15
|
from tinybird.config import FeatureFlags
|
|
20
|
-
from tinybird.feedback_manager import FeedbackManager
|
|
16
|
+
from tinybird.feedback_manager import FeedbackManager
|
|
17
|
+
from tinybird.tb.modules.build_shell import BuildShell, print_table_formatted
|
|
21
18
|
from tinybird.tb.modules.cli import cli
|
|
22
19
|
from tinybird.tb.modules.common import coro, push_data
|
|
23
20
|
from tinybird.tb.modules.datafile.build import folder_build
|
|
@@ -26,44 +23,7 @@ from tinybird.tb.modules.datafile.exceptions import ParseException
|
|
|
26
23
|
from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
|
|
27
24
|
from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
|
|
28
25
|
from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
|
|
29
|
-
from tinybird.tb.modules.
|
|
30
|
-
from tinybird.tb.modules.table import format_table
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
class BuildShell(cmd.Cmd):
|
|
34
|
-
prompt = "\n\001\033[1;32m\002TB > \001\033[0m\002"
|
|
35
|
-
|
|
36
|
-
def __init__(self, folder: str):
|
|
37
|
-
super().__init__()
|
|
38
|
-
self.folder = folder
|
|
39
|
-
|
|
40
|
-
def do_exit(self, arg):
|
|
41
|
-
sys.exit(0)
|
|
42
|
-
|
|
43
|
-
def do_quit(self, arg):
|
|
44
|
-
sys.exit(0)
|
|
45
|
-
|
|
46
|
-
def default(self, argline):
|
|
47
|
-
click.echo("")
|
|
48
|
-
if argline.startswith("tb build"):
|
|
49
|
-
click.echo(FeedbackManager.error(message="Build command is already running"))
|
|
50
|
-
else:
|
|
51
|
-
arg_stripped = argline.strip().lower()
|
|
52
|
-
if not arg_stripped:
|
|
53
|
-
return
|
|
54
|
-
if arg_stripped.startswith("tb"):
|
|
55
|
-
extra_args = f" --folder {self.folder}" if arg_stripped.startswith("tb mock") else ""
|
|
56
|
-
subprocess.run(arg_stripped + extra_args, shell=True, text=True)
|
|
57
|
-
elif arg_stripped.startswith("with") or arg_stripped.startswith("select"):
|
|
58
|
-
subprocess.run(f'tb sql "{arg_stripped}"', shell=True, text=True)
|
|
59
|
-
elif arg_stripped.startswith("mock "):
|
|
60
|
-
subprocess.run(f"tb {arg_stripped} --folder {self.folder}", shell=True, text=True)
|
|
61
|
-
else:
|
|
62
|
-
click.echo(FeedbackManager.error(message="Invalid command"))
|
|
63
|
-
|
|
64
|
-
def reprint_prompt(self):
|
|
65
|
-
self.stdout.write(self.prompt)
|
|
66
|
-
self.stdout.flush()
|
|
26
|
+
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
67
27
|
|
|
68
28
|
|
|
69
29
|
class FileChangeHandler(FileSystemEventHandler):
|
|
@@ -72,7 +32,12 @@ class FileChangeHandler(FileSystemEventHandler):
|
|
|
72
32
|
self.process = process
|
|
73
33
|
|
|
74
34
|
def on_modified(self, event: Any) -> None:
|
|
75
|
-
|
|
35
|
+
is_not_vendor = "vendor/" not in event.src_path
|
|
36
|
+
if (
|
|
37
|
+
is_not_vendor
|
|
38
|
+
and not event.is_directory
|
|
39
|
+
and any(event.src_path.endswith(ext) for ext in [".datasource", ".pipe", ".ndjson"])
|
|
40
|
+
):
|
|
76
41
|
filename = event.src_path.split("/")[-1]
|
|
77
42
|
click.echo(FeedbackManager.highlight(message=f"\n\n⟲ Changes detected in {filename}\n"))
|
|
78
43
|
try:
|
|
@@ -132,16 +97,12 @@ def watch_files(
|
|
|
132
97
|
is_flag=True,
|
|
133
98
|
help="Watch for changes in the files and re-check them.",
|
|
134
99
|
)
|
|
135
|
-
@click.
|
|
136
|
-
"--skip-datasources",
|
|
137
|
-
is_flag=True,
|
|
138
|
-
help="Skip rebuilding datasources.",
|
|
139
|
-
)
|
|
100
|
+
@click.pass_context
|
|
140
101
|
@coro
|
|
141
102
|
async def build(
|
|
103
|
+
ctx: Context,
|
|
142
104
|
folder: str,
|
|
143
105
|
watch: bool,
|
|
144
|
-
skip_datasources: bool,
|
|
145
106
|
) -> None:
|
|
146
107
|
"""
|
|
147
108
|
Watch for changes in the files and re-check them.
|
|
@@ -171,7 +132,7 @@ async def build(
|
|
|
171
132
|
|
|
172
133
|
parser(filename)
|
|
173
134
|
|
|
174
|
-
async def process(filenames: List[str], watch: bool = False
|
|
135
|
+
async def process(filenames: List[str], watch: bool = False):
|
|
175
136
|
datafiles = [f for f in filenames if f.endswith(".datasource") or f.endswith(".pipe")]
|
|
176
137
|
if len(datafiles) > 0:
|
|
177
138
|
check_filenames(filenames=datafiles)
|
|
@@ -180,7 +141,7 @@ async def build(
|
|
|
180
141
|
filenames=datafiles,
|
|
181
142
|
ignore_sql_errors=ignore_sql_errors,
|
|
182
143
|
is_internal=is_internal,
|
|
183
|
-
|
|
144
|
+
watch=watch,
|
|
184
145
|
)
|
|
185
146
|
|
|
186
147
|
filename = filenames[0]
|
|
@@ -209,7 +170,7 @@ async def build(
|
|
|
209
170
|
try:
|
|
210
171
|
click.echo("⚡ Building project...\n")
|
|
211
172
|
time_start = time.time()
|
|
212
|
-
await process(filenames=filenames, watch=False
|
|
173
|
+
await process(filenames=filenames, watch=False)
|
|
213
174
|
time_end = time.time()
|
|
214
175
|
elapsed_time = time_end - time_start
|
|
215
176
|
for filename in filenames:
|
|
@@ -226,7 +187,7 @@ async def build(
|
|
|
226
187
|
await build_once(filenames)
|
|
227
188
|
|
|
228
189
|
if watch:
|
|
229
|
-
shell = BuildShell(folder=folder)
|
|
190
|
+
shell = BuildShell(folder=folder, client=tb_client)
|
|
230
191
|
click.echo(FeedbackManager.highlight(message="◎ Watching for changes..."))
|
|
231
192
|
watcher_thread = threading.Thread(target=watch_files, args=(filenames, process, shell, folder), daemon=True)
|
|
232
193
|
watcher_thread.start()
|
|
@@ -234,48 +195,11 @@ async def build(
|
|
|
234
195
|
|
|
235
196
|
|
|
236
197
|
async def build_and_print_resource(tb_client: TinyB, filename: str):
|
|
237
|
-
rebuild_colors = [bcolors.FAIL, bcolors.OKBLUE, bcolors.WARNING, bcolors.OKGREEN, bcolors.HEADER]
|
|
238
|
-
rebuild_index = random.randint(0, len(rebuild_colors) - 1)
|
|
239
|
-
rebuild_color = rebuild_colors[rebuild_index % len(rebuild_colors)]
|
|
240
198
|
resource_path = Path(filename)
|
|
241
199
|
name = resource_path.stem
|
|
242
200
|
pipeline = name if filename.endswith(".pipe") else None
|
|
243
201
|
res = await tb_client.query(f"SELECT * FROM {name} FORMAT JSON", pipeline=pipeline)
|
|
244
|
-
|
|
245
|
-
limit = 5
|
|
246
|
-
for d in res["data"][:5]:
|
|
247
|
-
data.append(d.values())
|
|
248
|
-
meta = res["meta"]
|
|
249
|
-
row_count = res.get("rows", 0)
|
|
250
|
-
stats = res.get("statistics", {})
|
|
251
|
-
elapsed = stats.get("elapsed", 0)
|
|
252
|
-
cols = len(meta)
|
|
253
|
-
try:
|
|
254
|
-
|
|
255
|
-
def print_message(message: str, color=bcolors.CGREY):
|
|
256
|
-
return f"{color}{message}{bcolors.ENDC}"
|
|
257
|
-
|
|
258
|
-
table = format_table(data, meta)
|
|
259
|
-
colored_char = print_message("│", rebuild_color)
|
|
260
|
-
table_with_marker = "\n".join(f"{colored_char} {line}" for line in table.split("\n"))
|
|
261
|
-
click.echo(f"\n{colored_char} {print_message('⚡', rebuild_color)} Running {name}")
|
|
262
|
-
click.echo(colored_char)
|
|
263
|
-
click.echo(table_with_marker)
|
|
264
|
-
click.echo(colored_char)
|
|
265
|
-
rows_read = humanfriendly.format_number(stats.get("rows_read", 0))
|
|
266
|
-
bytes_read = humanfriendly.format_size(stats.get("bytes_read", 0))
|
|
267
|
-
elapsed = humanfriendly.format_timespan(elapsed) if elapsed >= 1 else f"{elapsed * 1000:.2f}ms"
|
|
268
|
-
stats_message = f"» {bytes_read} ({rows_read} rows x {cols} cols) in {elapsed}"
|
|
269
|
-
rows_message = f"» Showing {limit} first rows" if row_count > limit else "» Showing all rows"
|
|
270
|
-
click.echo(f"{colored_char} {print_message(stats_message, bcolors.OKGREEN)}")
|
|
271
|
-
click.echo(f"{colored_char} {print_message(rows_message, bcolors.CGREY)}")
|
|
272
|
-
except ValueError as exc:
|
|
273
|
-
if str(exc) == "max() arg is an empty sequence":
|
|
274
|
-
click.echo("------------")
|
|
275
|
-
click.echo("Empty")
|
|
276
|
-
click.echo("------------")
|
|
277
|
-
else:
|
|
278
|
-
raise exc
|
|
202
|
+
print_table_formatted(res, name)
|
|
279
203
|
|
|
280
204
|
|
|
281
205
|
async def append_datasource(
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import cmd
|
|
3
|
+
import random
|
|
4
|
+
import subprocess
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
import humanfriendly
|
|
9
|
+
|
|
10
|
+
from tinybird.client import TinyB
|
|
11
|
+
from tinybird.feedback_manager import FeedbackManager, bcolors
|
|
12
|
+
from tinybird.tb.modules.exceptions import CLIException
|
|
13
|
+
from tinybird.tb.modules.table import format_table
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class BuildShell(cmd.Cmd):
|
|
17
|
+
prompt = "\n\001\033[1;32m\002TB > \001\033[0m\002"
|
|
18
|
+
|
|
19
|
+
def __init__(self, folder: str, client: TinyB):
|
|
20
|
+
super().__init__()
|
|
21
|
+
self.folder = folder
|
|
22
|
+
self.client = client
|
|
23
|
+
|
|
24
|
+
def do_exit(self, arg):
|
|
25
|
+
sys.exit(0)
|
|
26
|
+
|
|
27
|
+
def do_quit(self, arg):
|
|
28
|
+
sys.exit(0)
|
|
29
|
+
|
|
30
|
+
def default(self, argline):
|
|
31
|
+
click.echo("")
|
|
32
|
+
if argline.startswith("tb build"):
|
|
33
|
+
click.echo(FeedbackManager.error(message="Build command is already running"))
|
|
34
|
+
else:
|
|
35
|
+
arg_stripped = argline.strip().lower()
|
|
36
|
+
if not arg_stripped:
|
|
37
|
+
return
|
|
38
|
+
if arg_stripped.startswith("tb"):
|
|
39
|
+
arg_stripped = arg_stripped.replace("tb", "tb --local")
|
|
40
|
+
extra_args = f" --folder {self.folder}" if arg_stripped.startswith("tb mock") else ""
|
|
41
|
+
subprocess.run(arg_stripped + extra_args, shell=True, text=True)
|
|
42
|
+
elif arg_stripped.startswith("with") or arg_stripped.startswith("select"):
|
|
43
|
+
try:
|
|
44
|
+
self.run_sql(self.client, argline)
|
|
45
|
+
except Exception as e:
|
|
46
|
+
click.echo(FeedbackManager.error(message=str(e)))
|
|
47
|
+
|
|
48
|
+
elif arg_stripped.startswith("mock "):
|
|
49
|
+
subprocess.run(f"tb {arg_stripped} --folder {self.folder}", shell=True, text=True)
|
|
50
|
+
else:
|
|
51
|
+
click.echo(FeedbackManager.error(message="Invalid command"))
|
|
52
|
+
|
|
53
|
+
def reprint_prompt(self):
|
|
54
|
+
self.stdout.write(self.prompt)
|
|
55
|
+
self.stdout.flush()
|
|
56
|
+
|
|
57
|
+
def run_sql(self, query, rows_limit=20):
|
|
58
|
+
try:
|
|
59
|
+
q = query.strip()
|
|
60
|
+
if q.startswith("insert"):
|
|
61
|
+
click.echo(FeedbackManager.info_append_data())
|
|
62
|
+
raise CLIException(FeedbackManager.error_invalid_query())
|
|
63
|
+
if q.startswith("delete"):
|
|
64
|
+
raise CLIException(FeedbackManager.error_invalid_query())
|
|
65
|
+
|
|
66
|
+
# fuck my life
|
|
67
|
+
def run_query_in_thread():
|
|
68
|
+
loop = asyncio.new_event_loop()
|
|
69
|
+
asyncio.set_event_loop(loop)
|
|
70
|
+
try:
|
|
71
|
+
return loop.run_until_complete(
|
|
72
|
+
self.client.query(f"SELECT * FROM ({query}) LIMIT {rows_limit} FORMAT JSON")
|
|
73
|
+
)
|
|
74
|
+
finally:
|
|
75
|
+
loop.close()
|
|
76
|
+
|
|
77
|
+
# Run the query in a separate thread
|
|
78
|
+
import concurrent.futures
|
|
79
|
+
|
|
80
|
+
with concurrent.futures.ThreadPoolExecutor() as executor:
|
|
81
|
+
res = executor.submit(run_query_in_thread).result()
|
|
82
|
+
|
|
83
|
+
except Exception as e:
|
|
84
|
+
raise CLIException(FeedbackManager.error_exception(error=str(e)))
|
|
85
|
+
|
|
86
|
+
if isinstance(res, dict) and "error" in res:
|
|
87
|
+
raise CLIException(FeedbackManager.error_exception(error=res["error"]))
|
|
88
|
+
|
|
89
|
+
if isinstance(res, dict) and "data" in res and res["data"]:
|
|
90
|
+
print_table_formatted(res, "QUERY")
|
|
91
|
+
else:
|
|
92
|
+
click.echo(FeedbackManager.info_no_rows())
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def print_table_formatted(res: dict, name: str):
|
|
96
|
+
rebuild_colors = [bcolors.FAIL, bcolors.OKBLUE, bcolors.WARNING, bcolors.OKGREEN, bcolors.HEADER]
|
|
97
|
+
rebuild_index = random.randint(0, len(rebuild_colors) - 1)
|
|
98
|
+
rebuild_color = rebuild_colors[rebuild_index % len(rebuild_colors)]
|
|
99
|
+
data = []
|
|
100
|
+
limit = 5
|
|
101
|
+
for d in res["data"][:5]:
|
|
102
|
+
data.append(d.values())
|
|
103
|
+
meta = res["meta"]
|
|
104
|
+
row_count = res.get("rows", 0)
|
|
105
|
+
stats = res.get("statistics", {})
|
|
106
|
+
elapsed = stats.get("elapsed", 0)
|
|
107
|
+
cols = len(meta)
|
|
108
|
+
try:
|
|
109
|
+
|
|
110
|
+
def print_message(message: str, color=bcolors.CGREY):
|
|
111
|
+
return f"{color}{message}{bcolors.ENDC}"
|
|
112
|
+
|
|
113
|
+
table = format_table(data, meta)
|
|
114
|
+
colored_char = print_message("│", rebuild_color)
|
|
115
|
+
table_with_marker = "\n".join(f"{colored_char} {line}" for line in table.split("\n"))
|
|
116
|
+
click.echo(f"\n{colored_char} {print_message('⚡', rebuild_color)} Running {name}")
|
|
117
|
+
click.echo(colored_char)
|
|
118
|
+
click.echo(table_with_marker)
|
|
119
|
+
click.echo(colored_char)
|
|
120
|
+
rows_read = humanfriendly.format_number(stats.get("rows_read", 0))
|
|
121
|
+
bytes_read = humanfriendly.format_size(stats.get("bytes_read", 0))
|
|
122
|
+
elapsed = humanfriendly.format_timespan(elapsed) if elapsed >= 1 else f"{elapsed * 1000:.2f}ms"
|
|
123
|
+
stats_message = f"» {bytes_read} ({rows_read} rows x {cols} cols) in {elapsed}"
|
|
124
|
+
rows_message = f"» Showing {limit} first rows" if row_count > limit else "» Showing all rows"
|
|
125
|
+
click.echo(f"{colored_char} {print_message(stats_message, bcolors.OKGREEN)}")
|
|
126
|
+
click.echo(f"{colored_char} {print_message(rows_message, bcolors.CGREY)}")
|
|
127
|
+
except ValueError as exc:
|
|
128
|
+
if str(exc) == "max() arg is an empty sequence":
|
|
129
|
+
click.echo("------------")
|
|
130
|
+
click.echo("Empty")
|
|
131
|
+
click.echo("------------")
|
|
132
|
+
else:
|
|
133
|
+
raise exc
|
|
@@ -10,7 +10,7 @@ import pprint
|
|
|
10
10
|
import re
|
|
11
11
|
import shutil
|
|
12
12
|
import sys
|
|
13
|
-
from os import
|
|
13
|
+
from os import getcwd
|
|
14
14
|
from pathlib import Path
|
|
15
15
|
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
|
|
16
16
|
|
|
@@ -26,7 +26,7 @@ from tinybird.client import (
|
|
|
26
26
|
DoesNotExistException,
|
|
27
27
|
TinyB,
|
|
28
28
|
)
|
|
29
|
-
from tinybird.config import
|
|
29
|
+
from tinybird.config import SUPPORTED_CONNECTORS, VERSION, FeatureFlags, get_config
|
|
30
30
|
from tinybird.feedback_manager import FeedbackManager
|
|
31
31
|
from tinybird.tb.modules.common import (
|
|
32
32
|
OLDEST_ROLLBACK,
|
|
@@ -50,6 +50,7 @@ from tinybird.tb.modules.config import CLIConfig
|
|
|
50
50
|
from tinybird.tb.modules.datafile.build import build_graph, folder_push
|
|
51
51
|
from tinybird.tb.modules.datafile.common import (
|
|
52
52
|
Datafile,
|
|
53
|
+
DatafileSyntaxError,
|
|
53
54
|
get_project_filenames,
|
|
54
55
|
get_resource_versions,
|
|
55
56
|
has_internal_datafiles,
|
|
@@ -62,6 +63,7 @@ from tinybird.tb.modules.datafile.exceptions import (
|
|
|
62
63
|
from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
|
|
63
64
|
from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
|
|
64
65
|
from tinybird.tb.modules.datafile.pull import folder_pull
|
|
66
|
+
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
65
67
|
from tinybird.tb.modules.telemetry import add_telemetry_event
|
|
66
68
|
|
|
67
69
|
__old_click_echo = click.echo
|
|
@@ -106,13 +108,8 @@ DEFAULT_PATTERNS: List[Tuple[str, Union[str, Callable[[str], str]]]] = [
|
|
|
106
108
|
@click.option(
|
|
107
109
|
"--with-headers", help="Flag to enable connector to export with headers", is_flag=True, default=False, hidden=True
|
|
108
110
|
)
|
|
109
|
-
@click.option(
|
|
110
|
-
"--version-warning/--no-version-warning",
|
|
111
|
-
envvar="TB_VERSION_WARNING",
|
|
112
|
-
default=True,
|
|
113
|
-
help="Don't print version warning message if there's a new available version. You can use TB_VERSION_WARNING envar",
|
|
114
|
-
)
|
|
115
111
|
@click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
|
|
112
|
+
@click.option("--local", is_flag=True, default=False, help="Run in local mode")
|
|
116
113
|
@click.version_option(version=VERSION)
|
|
117
114
|
@click.pass_context
|
|
118
115
|
@coro
|
|
@@ -135,8 +132,8 @@ async def cli(
|
|
|
135
132
|
sf_storage_integration: str,
|
|
136
133
|
sf_stage,
|
|
137
134
|
with_headers: bool,
|
|
138
|
-
version_warning: bool,
|
|
139
135
|
show_tokens: bool,
|
|
136
|
+
local: bool,
|
|
140
137
|
) -> None:
|
|
141
138
|
"""
|
|
142
139
|
Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
|
|
@@ -151,17 +148,6 @@ async def cli(
|
|
|
151
148
|
if getenv_bool("TB_DISABLE_SSL_CHECKS", False):
|
|
152
149
|
click.echo(FeedbackManager.warning_disabled_ssl_checks())
|
|
153
150
|
|
|
154
|
-
# ensure that ctx.obj exists and is a dict (in case `cli()` is called)
|
|
155
|
-
# by means other than the `if` block below
|
|
156
|
-
if not environ.get("PYTEST", None) and version_warning and not token:
|
|
157
|
-
from tinybird.check_pypi import CheckPypi
|
|
158
|
-
|
|
159
|
-
latest_version = await CheckPypi().get_latest_version()
|
|
160
|
-
|
|
161
|
-
if "x.y.z" not in CURRENT_VERSION and latest_version != CURRENT_VERSION:
|
|
162
|
-
click.echo(FeedbackManager.warning_update_version(latest_version=latest_version))
|
|
163
|
-
click.echo(FeedbackManager.warning_current_version(current_version=CURRENT_VERSION))
|
|
164
|
-
|
|
165
151
|
if debug:
|
|
166
152
|
logging.basicConfig(level=logging.DEBUG)
|
|
167
153
|
|
|
@@ -245,7 +231,11 @@ async def cli(
|
|
|
245
231
|
|
|
246
232
|
logging.debug("debug enabled")
|
|
247
233
|
|
|
248
|
-
ctx.ensure_object(dict)["client"] =
|
|
234
|
+
ctx.ensure_object(dict)["client"] = (
|
|
235
|
+
await get_tinybird_local_client()
|
|
236
|
+
if local
|
|
237
|
+
else _get_tb_client(config.get("token", None), config["host"], semver)
|
|
238
|
+
)
|
|
249
239
|
|
|
250
240
|
for connector in SUPPORTED_CONNECTORS:
|
|
251
241
|
load_connector_config(ctx, connector, debug, check_uninstalled=True)
|
|
@@ -361,6 +351,9 @@ def check(filenames: List[str], debug: bool) -> None:
|
|
|
361
351
|
for x in doc.nodes:
|
|
362
352
|
pp.pprint(x)
|
|
363
353
|
|
|
354
|
+
except DatafileSyntaxError as e:
|
|
355
|
+
# TODO(eclbg): add the filename to the error message
|
|
356
|
+
raise CLIException(e)
|
|
364
357
|
except ParseException as e:
|
|
365
358
|
raise CLIException(FeedbackManager.error_exception(error=e))
|
|
366
359
|
|