tinybird 0.0.1.dev291__py3-none-any.whl → 1.0.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tinybird/ch_utils/constants.py +5 -0
- tinybird/connectors.py +1 -7
- tinybird/context.py +3 -3
- tinybird/datafile/common.py +10 -8
- tinybird/datafile/parse_pipe.py +2 -2
- tinybird/feedback_manager.py +3 -0
- tinybird/prompts.py +1 -0
- tinybird/service_datasources.py +223 -0
- tinybird/sql_template.py +26 -11
- tinybird/sql_template_fmt.py +14 -4
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/cli.py +1 -0
- tinybird/tb/client.py +104 -26
- tinybird/tb/config.py +24 -0
- tinybird/tb/modules/agent/agent.py +103 -67
- tinybird/tb/modules/agent/banner.py +15 -15
- tinybird/tb/modules/agent/explore_agent.py +5 -0
- tinybird/tb/modules/agent/mock_agent.py +5 -1
- tinybird/tb/modules/agent/models.py +6 -2
- tinybird/tb/modules/agent/prompts.py +49 -2
- tinybird/tb/modules/agent/tools/deploy.py +1 -1
- tinybird/tb/modules/agent/tools/execute_query.py +15 -18
- tinybird/tb/modules/agent/tools/request_endpoint.py +1 -1
- tinybird/tb/modules/agent/tools/run_command.py +9 -0
- tinybird/tb/modules/agent/utils.py +38 -48
- tinybird/tb/modules/branch.py +150 -0
- tinybird/tb/modules/build.py +58 -13
- tinybird/tb/modules/build_common.py +209 -25
- tinybird/tb/modules/cli.py +129 -16
- tinybird/tb/modules/common.py +172 -146
- tinybird/tb/modules/connection.py +125 -194
- tinybird/tb/modules/connection_kafka.py +382 -0
- tinybird/tb/modules/copy.py +3 -1
- tinybird/tb/modules/create.py +83 -150
- tinybird/tb/modules/datafile/build.py +27 -38
- tinybird/tb/modules/datafile/build_datasource.py +21 -25
- tinybird/tb/modules/datafile/diff.py +1 -1
- tinybird/tb/modules/datafile/format_pipe.py +46 -7
- tinybird/tb/modules/datafile/playground.py +59 -68
- tinybird/tb/modules/datafile/pull.py +2 -3
- tinybird/tb/modules/datasource.py +477 -308
- tinybird/tb/modules/deployment.py +2 -0
- tinybird/tb/modules/deployment_common.py +84 -44
- tinybird/tb/modules/deprecations.py +4 -4
- tinybird/tb/modules/dev_server.py +33 -12
- tinybird/tb/modules/exceptions.py +14 -0
- tinybird/tb/modules/feedback_manager.py +1 -1
- tinybird/tb/modules/info.py +69 -12
- tinybird/tb/modules/infra.py +4 -5
- tinybird/tb/modules/job_common.py +15 -0
- tinybird/tb/modules/local.py +143 -23
- tinybird/tb/modules/local_common.py +347 -19
- tinybird/tb/modules/local_logs.py +209 -0
- tinybird/tb/modules/login.py +21 -2
- tinybird/tb/modules/login_common.py +254 -12
- tinybird/tb/modules/mock.py +5 -54
- tinybird/tb/modules/mock_common.py +0 -54
- tinybird/tb/modules/open.py +10 -5
- tinybird/tb/modules/project.py +14 -5
- tinybird/tb/modules/shell.py +15 -7
- tinybird/tb/modules/sink.py +3 -1
- tinybird/tb/modules/telemetry.py +11 -3
- tinybird/tb/modules/test.py +13 -9
- tinybird/tb/modules/test_common.py +13 -87
- tinybird/tb/modules/tinyunit/tinyunit.py +0 -14
- tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -6
- tinybird/tb/modules/watch.py +5 -3
- tinybird/tb_cli_modules/common.py +2 -2
- tinybird/tb_cli_modules/telemetry.py +1 -1
- tinybird/tornado_template.py +6 -7
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/METADATA +32 -6
- tinybird-1.0.5.dist-info/RECORD +132 -0
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/WHEEL +1 -1
- tinybird-0.0.1.dev291.dist-info/RECORD +0 -128
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/top_level.txt +0 -0
|
@@ -8,7 +8,7 @@ from pydantic_ai.retries import AsyncTenacityTransport, wait_retry_after
|
|
|
8
8
|
from tenacity import AsyncRetrying, retry_if_exception_type, stop_after_attempt, wait_exponential
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
def create_retrying_client(token: str, workspace_id: str):
|
|
11
|
+
def create_retrying_client(token: str, workspace_id: str, feature: Optional[str] = None):
|
|
12
12
|
"""Create a client with smart retry handling for multiple error types."""
|
|
13
13
|
|
|
14
14
|
def should_retry_status(response):
|
|
@@ -29,7 +29,10 @@ def create_retrying_client(token: str, workspace_id: str):
|
|
|
29
29
|
),
|
|
30
30
|
validate_response=should_retry_status,
|
|
31
31
|
)
|
|
32
|
-
|
|
32
|
+
params = {"token": token, "workspace_id": workspace_id}
|
|
33
|
+
if feature:
|
|
34
|
+
params["feature"] = feature
|
|
35
|
+
return AsyncClient(transport=transport, params=params)
|
|
33
36
|
|
|
34
37
|
|
|
35
38
|
def create_model(
|
|
@@ -38,6 +41,7 @@ def create_model(
|
|
|
38
41
|
workspace_id: str,
|
|
39
42
|
model: AnthropicModelName = "claude-4-sonnet-20250514",
|
|
40
43
|
run_id: Optional[str] = None,
|
|
44
|
+
feature: Optional[str] = None,
|
|
41
45
|
):
|
|
42
46
|
default_headers = {}
|
|
43
47
|
if run_id:
|
|
@@ -35,8 +35,10 @@ available_commands = [
|
|
|
35
35
|
"`tb sink ls`: List all sinks",
|
|
36
36
|
"`tb workspace current`: Show the current workspace",
|
|
37
37
|
"`tb workspace clear --yes`: Delete all resources in the workspace (Only available in Tinybird Local)",
|
|
38
|
+
"`tb workspace ls`: List all workspaces",
|
|
38
39
|
"`tb local start --skip-new-version`: Start Tinybird Local container in non-interactive mode",
|
|
39
40
|
"`tb local restart --skip-new-version --yes`: Restart Tinybird Local container in non-interactive mode",
|
|
41
|
+
"`tb pull --only-vendored`: Pull only the vendored datasources from other workspaces",
|
|
40
42
|
]
|
|
41
43
|
|
|
42
44
|
plan_instructions = """
|
|
@@ -166,7 +168,6 @@ SQL >
|
|
|
166
168
|
|
|
167
169
|
def resources_prompt(project: Project) -> str:
|
|
168
170
|
files = project.get_project_files()
|
|
169
|
-
fixture_files = project.get_fixture_files()
|
|
170
171
|
|
|
171
172
|
resources_content = "# Existing resources in the project:\n"
|
|
172
173
|
if files:
|
|
@@ -184,7 +185,36 @@ def resources_prompt(project: Project) -> str:
|
|
|
184
185
|
else:
|
|
185
186
|
resources_content += "No resources found"
|
|
186
187
|
|
|
188
|
+
return resources_content
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def vendor_files_prompt(project: Project) -> str:
|
|
192
|
+
files = project.get_vendored_files()
|
|
193
|
+
content = "# Datasources shared from other workspaces:\n"
|
|
194
|
+
if files:
|
|
195
|
+
resources: list[dict[str, Any]] = []
|
|
196
|
+
for filename in files:
|
|
197
|
+
file_path = Path(filename)
|
|
198
|
+
workspace_name = file_path.parent.parent.name
|
|
199
|
+
resource = {
|
|
200
|
+
"path": str(file_path.relative_to(project.folder)),
|
|
201
|
+
"type": get_resource_type(file_path),
|
|
202
|
+
"name": f"{workspace_name}.{file_path.stem}",
|
|
203
|
+
"content": file_path.read_text(),
|
|
204
|
+
"origin_workspace": workspace_name,
|
|
205
|
+
}
|
|
206
|
+
resources.append(resource)
|
|
207
|
+
content += format_as_xml(resources, root_tag="resources", item_tag="resource")
|
|
208
|
+
else:
|
|
209
|
+
content += "No datasources shared from other workspaces"
|
|
210
|
+
|
|
211
|
+
return content
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def fixtures_prompt(project: Project) -> str:
|
|
215
|
+
fixture_files = project.get_fixture_files()
|
|
187
216
|
fixture_content = "# Fixture files in the project:\n"
|
|
217
|
+
|
|
188
218
|
if fixture_files:
|
|
189
219
|
fixtures: list[dict[str, Any]] = []
|
|
190
220
|
for filename in fixture_files:
|
|
@@ -199,7 +229,7 @@ def resources_prompt(project: Project) -> str:
|
|
|
199
229
|
else:
|
|
200
230
|
fixture_content += "No fixture files found"
|
|
201
231
|
|
|
202
|
-
return
|
|
232
|
+
return fixture_content
|
|
203
233
|
|
|
204
234
|
|
|
205
235
|
def service_datasources_prompt() -> str:
|
|
@@ -954,6 +984,23 @@ After changes have been deployed and promoted, if you want to deploy other chang
|
|
|
954
984
|
If after running a deployment, the error contains a recommended forward query, use it to update the .datasource file.
|
|
955
985
|
</dev_notes>
|
|
956
986
|
|
|
987
|
+
## Sharing datasources with other workspaces
|
|
988
|
+
To share a Data Source, in the .datasource file you want to share, add the destination workspace(s). For example:
|
|
989
|
+
|
|
990
|
+
```
|
|
991
|
+
SHARED_WITH >
|
|
992
|
+
destination_workspace,
|
|
993
|
+
other_destination_workspace
|
|
994
|
+
```
|
|
995
|
+
|
|
996
|
+
## Working with shared datasources:
|
|
997
|
+
|
|
998
|
+
The following limitations apply to shared datasources:
|
|
999
|
+
- Shared datasources are read-only.
|
|
1000
|
+
- You can't share a shared datasource, only the original.
|
|
1001
|
+
- You can't check the quarantine of a shared datasource.
|
|
1002
|
+
- You can't create a Materialized View from a shared datasource.
|
|
1003
|
+
|
|
957
1004
|
# Working with any type of pipe file:
|
|
958
1005
|
{pipe_instructions}
|
|
959
1006
|
{pipe_example}
|
|
@@ -24,7 +24,7 @@ def deploy(ctx: RunContext[TinybirdAgentContext], allow_destructive_operations:
|
|
|
24
24
|
FeedbackManager.warning(message="Destructive operations flag is enabled due to a file deleted recently")
|
|
25
25
|
)
|
|
26
26
|
|
|
27
|
-
click.echo()
|
|
27
|
+
click.echo("")
|
|
28
28
|
confirmation = show_confirmation(
|
|
29
29
|
title="Deploy the project?",
|
|
30
30
|
skip_confirmation=False,
|
|
@@ -121,27 +121,24 @@ def execute_query(
|
|
|
121
121
|
bytes_read = humanfriendly.format_size(stats["bytes_read"])
|
|
122
122
|
|
|
123
123
|
click.echo(FeedbackManager.info_query_stats(seconds=seconds, rows=rows_read, bytes=bytes_read))
|
|
124
|
-
click.echo()
|
|
124
|
+
click.echo("")
|
|
125
125
|
|
|
126
126
|
if not result["data"]:
|
|
127
127
|
click.echo(FeedbackManager.info_no_rows())
|
|
128
|
+
elif script:
|
|
129
|
+
try:
|
|
130
|
+
# Execute the LLM-generated plotext script
|
|
131
|
+
chart_output = _execute_plotext_script(script, result["data"], result["meta"])
|
|
132
|
+
click.echo(chart_output)
|
|
133
|
+
except Exception as script_error:
|
|
134
|
+
click.echo(FeedbackManager.error(message=f"There was an error rendering the chart.\n{script_error}"))
|
|
135
|
+
ctx.deps.thinking_animation.start()
|
|
136
|
+
return f"After executing the query: {query}, there was an error rendering the chart: {script_error}. Fix the script and render the chart again."
|
|
128
137
|
else:
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
click.echo(chart_output)
|
|
134
|
-
except Exception as script_error:
|
|
135
|
-
click.echo(
|
|
136
|
-
FeedbackManager.error(message=f"There was an error rendering the chart.\n{script_error}")
|
|
137
|
-
)
|
|
138
|
-
ctx.deps.thinking_animation.start()
|
|
139
|
-
return f"After executing the query: {query}, there was an error rendering the chart: {script_error}. Fix the script and render the chart again."
|
|
140
|
-
else:
|
|
141
|
-
echo_safe_humanfriendly_tables_format_pretty_table(
|
|
142
|
-
data=[d.values() for d in result["data"]], column_names=result["data"][0].keys()
|
|
143
|
-
)
|
|
144
|
-
click.echo("Showing first 10 results\n")
|
|
138
|
+
echo_safe_humanfriendly_tables_format_pretty_table(
|
|
139
|
+
data=[d.values() for d in result["data"]], column_names=result["data"][0].keys()
|
|
140
|
+
)
|
|
141
|
+
click.echo("Showing first 10 results\n")
|
|
145
142
|
|
|
146
143
|
ctx.deps.thinking_animation.start()
|
|
147
144
|
display_format = "chart" if script else "table"
|
|
@@ -151,7 +148,7 @@ def execute_query(
|
|
|
151
148
|
ctx.deps.thinking_animation.stop()
|
|
152
149
|
click.echo(FeedbackManager.error(message=error))
|
|
153
150
|
ctx.deps.thinking_animation.start()
|
|
154
|
-
if "not found" in error.lower() and cloud:
|
|
151
|
+
if "not found" in error.lower() and cloud_or_local == "cloud":
|
|
155
152
|
return f"Error executing query: {error}. Please run the query against Tinybird local instead of cloud."
|
|
156
153
|
else:
|
|
157
154
|
return f"Error executing query: {error}. Please try again."
|
|
@@ -87,7 +87,7 @@ def request_endpoint(
|
|
|
87
87
|
click.echo(FeedbackManager.error(message=error))
|
|
88
88
|
ctx.deps.thinking_animation.start()
|
|
89
89
|
not_found_errors = ["not found", "does not exist"]
|
|
90
|
-
if any(not_found_error in error.lower() for not_found_error in not_found_errors) and cloud:
|
|
90
|
+
if any(not_found_error in error.lower() for not_found_error in not_found_errors) and cloud_or_local == "cloud":
|
|
91
91
|
return f"Error executing query: {error}. Please run the query against Tinybird local instead of cloud."
|
|
92
92
|
else:
|
|
93
93
|
return f"Error executing query: {error}. Please try again."
|
|
@@ -3,6 +3,7 @@ import subprocess
|
|
|
3
3
|
import click
|
|
4
4
|
from pydantic_ai import RunContext
|
|
5
5
|
|
|
6
|
+
from tinybird.tb.modules.agent.prompts import available_commands
|
|
6
7
|
from tinybird.tb.modules.agent.utils import (
|
|
7
8
|
AgentRunCancelled,
|
|
8
9
|
SubAgentRunCancelled,
|
|
@@ -20,6 +21,14 @@ def run_command(ctx: RunContext[TinybirdAgentContext], command: str):
|
|
|
20
21
|
command (str): The command to run. Required. Examples: `tb --local sql "select 1"`, `tb --cloud datasource ls`, `tb --help`
|
|
21
22
|
"""
|
|
22
23
|
try:
|
|
24
|
+
clean_commands = [cmd.split(":")[0].replace("`", "").split("[")[0] for cmd in available_commands]
|
|
25
|
+
available_commands_to_cloud = [f"tb --cloud {command.replace('tb ', '')}" for command in clean_commands]
|
|
26
|
+
available_commands_to_local = [f"tb --local {command.replace('tb ', '')}" for command in clean_commands]
|
|
27
|
+
all_commands = clean_commands + available_commands_to_cloud + available_commands_to_local
|
|
28
|
+
|
|
29
|
+
if not any(cmd.startswith(command) for cmd in all_commands):
|
|
30
|
+
raise SubAgentRunCancelled(f"Command {command} not found in the list of available commands")
|
|
31
|
+
|
|
23
32
|
ctx.deps.thinking_animation.stop()
|
|
24
33
|
force_confirmation = " deploy" in command.lower() or " truncate" in command.lower()
|
|
25
34
|
confirmation = show_confirmation(
|
|
@@ -44,7 +44,6 @@ class TinybirdAgentContext(BaseModel):
|
|
|
44
44
|
build_project_test: Callable[..., None]
|
|
45
45
|
deploy_project: Callable[..., None]
|
|
46
46
|
deploy_check_project: Callable[[], None]
|
|
47
|
-
mock_data: Callable[..., list[dict[str, Any]]]
|
|
48
47
|
append_data_local: Callable[..., None]
|
|
49
48
|
append_data_cloud: Callable[..., None]
|
|
50
49
|
analyze_fixture: Callable[..., dict[str, Any]]
|
|
@@ -512,7 +511,7 @@ def create_terminal_box(content: str, new_content: Optional[str] = None, title:
|
|
|
512
511
|
new_line_num = int(match.group(2))
|
|
513
512
|
old_index = old_line_num - 1
|
|
514
513
|
new_index = new_line_num - 1
|
|
515
|
-
elif line.startswith("---"
|
|
514
|
+
elif line.startswith(("---", "+++")):
|
|
516
515
|
# Skip file headers
|
|
517
516
|
pass
|
|
518
517
|
elif line.startswith(" "):
|
|
@@ -581,59 +580,50 @@ def create_terminal_box(content: str, new_content: Optional[str] = None, title:
|
|
|
581
580
|
else:
|
|
582
581
|
line_num_str = f"{line_num:>4}"
|
|
583
582
|
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
if diff_marker == "-":
|
|
587
|
-
# Fill the entire content area with red background
|
|
588
|
-
content_with_bg = f"{Back.RED}{diff_marker} {content}{Style.RESET_ALL}"
|
|
589
|
-
# Calculate padding needed for the content area
|
|
590
|
-
content_area_width = available_width - 9 # 9 is reduced prefix length
|
|
591
|
-
content_padding = content_area_width - len(f"{diff_marker} {content}")
|
|
592
|
-
if content_padding > 0:
|
|
593
|
-
content_with_bg = (
|
|
594
|
-
f"{Back.RED}{diff_marker} {content}{' ' * content_padding}{Style.RESET_ALL}"
|
|
595
|
-
)
|
|
596
|
-
line = f"{vertical} {line_num_str} {content_with_bg}"
|
|
597
|
-
elif diff_marker == "+":
|
|
598
|
-
# Fill the entire content area with green background
|
|
599
|
-
content_with_bg = f"{Back.GREEN}{diff_marker} {content}{Style.RESET_ALL}"
|
|
600
|
-
# Calculate padding needed for the content area
|
|
601
|
-
content_area_width = available_width - 9 # 9 is reduced prefix length
|
|
602
|
-
content_padding = content_area_width - len(f"{diff_marker} {content}")
|
|
603
|
-
if content_padding > 0:
|
|
604
|
-
content_with_bg = (
|
|
605
|
-
f"{Back.GREEN}{diff_marker} {content}{' ' * content_padding}{Style.RESET_ALL}"
|
|
606
|
-
)
|
|
607
|
-
line = f"{vertical} {line_num_str} {content_with_bg}"
|
|
608
|
-
else:
|
|
609
|
-
line = f"{vertical} {line_num:>4} {diff_marker} {content}"
|
|
610
|
-
else:
|
|
611
|
-
line = f"{vertical} {line_num_str} {content}"
|
|
612
|
-
else:
|
|
613
|
-
# Continuation line without number - fill background starting from where symbol would be
|
|
614
|
-
if diff_marker and COLORAMA_AVAILABLE:
|
|
583
|
+
if diff_marker:
|
|
584
|
+
if COLORAMA_AVAILABLE:
|
|
615
585
|
if diff_marker == "-":
|
|
616
|
-
#
|
|
586
|
+
# Fill the entire content area with red background
|
|
587
|
+
content_with_bg = f"{Back.RED}{diff_marker} {content}{Style.RESET_ALL}"
|
|
588
|
+
# Calculate padding needed for the content area
|
|
617
589
|
content_area_width = available_width - 9 # 9 is reduced prefix length
|
|
618
|
-
content_padding = content_area_width - len(
|
|
619
|
-
content
|
|
620
|
-
) # Don't subtract spaces, they're in the background
|
|
590
|
+
content_padding = content_area_width - len(f"{diff_marker} {content}")
|
|
621
591
|
if content_padding > 0:
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
line = f"{vertical} {Back.RED} {content}{Style.RESET_ALL}"
|
|
592
|
+
content_with_bg = f"{Back.RED}{diff_marker} {content}{' ' * content_padding}{Style.RESET_ALL}"
|
|
593
|
+
line = f"{vertical} {line_num_str} {content_with_bg}"
|
|
625
594
|
elif diff_marker == "+":
|
|
626
|
-
#
|
|
595
|
+
# Fill the entire content area with green background
|
|
596
|
+
content_with_bg = f"{Back.GREEN}{diff_marker} {content}{Style.RESET_ALL}"
|
|
597
|
+
# Calculate padding needed for the content area
|
|
627
598
|
content_area_width = available_width - 9 # 9 is reduced prefix length
|
|
628
|
-
content_padding = content_area_width - len(
|
|
629
|
-
content
|
|
630
|
-
) # Don't subtract spaces, they're in the background
|
|
599
|
+
content_padding = content_area_width - len(f"{diff_marker} {content}")
|
|
631
600
|
if content_padding > 0:
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
601
|
+
content_with_bg = (
|
|
602
|
+
f"{Back.GREEN}{diff_marker} {content}{' ' * content_padding}{Style.RESET_ALL}"
|
|
603
|
+
)
|
|
604
|
+
line = f"{vertical} {line_num_str} {content_with_bg}"
|
|
635
605
|
else:
|
|
636
|
-
line = f"{vertical}
|
|
606
|
+
line = f"{vertical} {line_num:>4} {diff_marker} {content}"
|
|
607
|
+
elif diff_marker and COLORAMA_AVAILABLE:
|
|
608
|
+
# Continuation line without number - fill background starting from where symbol would be
|
|
609
|
+
if diff_marker == "-":
|
|
610
|
+
# Calculate how much space we need to fill with background
|
|
611
|
+
content_area_width = available_width - 9 # 9 is reduced prefix length
|
|
612
|
+
content_padding = content_area_width - len(content) # Don't subtract spaces, they're in the background
|
|
613
|
+
if content_padding > 0:
|
|
614
|
+
line = f"{vertical} {Back.RED} {content}{' ' * content_padding}{Style.RESET_ALL}"
|
|
615
|
+
else:
|
|
616
|
+
line = f"{vertical} {Back.RED} {content}{Style.RESET_ALL}"
|
|
617
|
+
elif diff_marker == "+":
|
|
618
|
+
# Calculate how much space we need to fill with background
|
|
619
|
+
content_area_width = available_width - 9 # 9 is reduced prefix length
|
|
620
|
+
content_padding = content_area_width - len(content) # Don't subtract spaces, they're in the background
|
|
621
|
+
if content_padding > 0:
|
|
622
|
+
line = f"{vertical} {Back.GREEN} {content}{' ' * content_padding}{Style.RESET_ALL}"
|
|
623
|
+
else:
|
|
624
|
+
line = f"{vertical} {Back.GREEN} {content}{Style.RESET_ALL}"
|
|
625
|
+
else:
|
|
626
|
+
line = f"{vertical} {content}"
|
|
637
627
|
|
|
638
628
|
# Pad to terminal width
|
|
639
629
|
# Need to account for ANSI escape sequences not taking visual space
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# This is a command file for our CLI. Please keep it clean.
|
|
2
|
+
#
|
|
3
|
+
# - If it makes sense and only when strictly necessary, you can create utility functions in this file.
|
|
4
|
+
# - But please, **do not** interleave utility functions and command definitions.
|
|
5
|
+
|
|
6
|
+
from typing import List, Optional, Tuple
|
|
7
|
+
|
|
8
|
+
import click
|
|
9
|
+
|
|
10
|
+
from tinybird.tb.modules.cli import cli
|
|
11
|
+
from tinybird.tb.modules.common import (
|
|
12
|
+
MAIN_BRANCH,
|
|
13
|
+
create_workspace_branch,
|
|
14
|
+
echo_safe_humanfriendly_tables_format_smart_table,
|
|
15
|
+
get_current_main_workspace,
|
|
16
|
+
get_current_workspace_branches,
|
|
17
|
+
get_workspace_member_email,
|
|
18
|
+
switch_to_workspace_by_user_workspace_data,
|
|
19
|
+
try_update_config_with_remote,
|
|
20
|
+
)
|
|
21
|
+
from tinybird.tb.modules.config import CLIConfig
|
|
22
|
+
from tinybird.tb.modules.exceptions import CLIBranchException, CLIException
|
|
23
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@cli.group()
|
|
27
|
+
def branch() -> None:
|
|
28
|
+
"""Branch commands. Custom branches is an experimental feature in beta."""
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@branch.command(name="ls")
|
|
33
|
+
@click.option("--sort/--no-sort", default=False, help="Sort the table rows by name")
|
|
34
|
+
def branch_ls(sort: bool) -> None:
|
|
35
|
+
"""List all the branches available in the current workspace"""
|
|
36
|
+
|
|
37
|
+
config = CLIConfig.get_project_config()
|
|
38
|
+
_ = try_update_config_with_remote(config, only_if_needed=True)
|
|
39
|
+
|
|
40
|
+
client = config.get_client()
|
|
41
|
+
|
|
42
|
+
current_main_workspace = get_current_main_workspace(config)
|
|
43
|
+
assert isinstance(current_main_workspace, dict)
|
|
44
|
+
|
|
45
|
+
if current_main_workspace["id"] != config["id"]:
|
|
46
|
+
client = config.get_client(token=current_main_workspace["token"])
|
|
47
|
+
|
|
48
|
+
response = client.branches()
|
|
49
|
+
|
|
50
|
+
columns = ["name", "id", "created_at", "owner", "current"]
|
|
51
|
+
|
|
52
|
+
table: List[Tuple[str, str, str, str, bool]] = []
|
|
53
|
+
|
|
54
|
+
for branch in response["environments"]:
|
|
55
|
+
branch_owner_email = get_workspace_member_email(branch, branch["owner"])
|
|
56
|
+
|
|
57
|
+
table.append(
|
|
58
|
+
(branch["name"], branch["id"], branch["created_at"], branch_owner_email, config["id"] == branch["id"])
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
current_branch = [row for row in table if row[4]]
|
|
62
|
+
other_branches = [row for row in table if not row[4]]
|
|
63
|
+
|
|
64
|
+
if sort:
|
|
65
|
+
other_branches.sort(key=lambda x: x[0])
|
|
66
|
+
|
|
67
|
+
sorted_table = current_branch + other_branches
|
|
68
|
+
|
|
69
|
+
click.echo(FeedbackManager.info(message="\n** Branches:"))
|
|
70
|
+
echo_safe_humanfriendly_tables_format_smart_table(sorted_table, column_names=columns)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@branch.command(name="create", short_help="Create a new branch in the current Workspace")
|
|
74
|
+
@click.argument("branch_name", required=False)
|
|
75
|
+
@click.option(
|
|
76
|
+
"--last-partition",
|
|
77
|
+
is_flag=True,
|
|
78
|
+
default=False,
|
|
79
|
+
help="Attach the last modified partition from the current workspace to the new branch",
|
|
80
|
+
)
|
|
81
|
+
@click.option(
|
|
82
|
+
"-i",
|
|
83
|
+
"--ignore-datasource",
|
|
84
|
+
"ignore_datasources",
|
|
85
|
+
type=str,
|
|
86
|
+
multiple=True,
|
|
87
|
+
help="Ignore specified data source partitions",
|
|
88
|
+
)
|
|
89
|
+
@click.option(
|
|
90
|
+
"--wait/--no-wait",
|
|
91
|
+
is_flag=True,
|
|
92
|
+
default=True,
|
|
93
|
+
help="Wait for data branch jobs to finish, showing a progress bar. Disabled by default.",
|
|
94
|
+
)
|
|
95
|
+
def create_branch(branch_name: Optional[str], last_partition: bool, ignore_datasources: List[str], wait: bool) -> None:
|
|
96
|
+
create_workspace_branch(branch_name, last_partition, False, list(ignore_datasources), wait)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@branch.command(name="rm", short_help="Removes an branch from the workspace. It can't be recovered.")
|
|
100
|
+
@click.argument("branch_name_or_id")
|
|
101
|
+
@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
|
|
102
|
+
def delete_branch(branch_name_or_id: str, yes: bool) -> None:
|
|
103
|
+
"""Remove an branch"""
|
|
104
|
+
|
|
105
|
+
config = CLIConfig.get_project_config()
|
|
106
|
+
_ = try_update_config_with_remote(config)
|
|
107
|
+
|
|
108
|
+
client = config.get_client()
|
|
109
|
+
|
|
110
|
+
if branch_name_or_id == MAIN_BRANCH:
|
|
111
|
+
raise CLIException(FeedbackManager.error_not_allowed_in_main_branch())
|
|
112
|
+
|
|
113
|
+
try:
|
|
114
|
+
workspace_branches = get_current_workspace_branches(config)
|
|
115
|
+
workspace_to_delete = next(
|
|
116
|
+
(
|
|
117
|
+
workspace
|
|
118
|
+
for workspace in workspace_branches
|
|
119
|
+
if workspace["name"] == branch_name_or_id or workspace["id"] == branch_name_or_id
|
|
120
|
+
),
|
|
121
|
+
None,
|
|
122
|
+
)
|
|
123
|
+
except Exception as e:
|
|
124
|
+
raise CLIBranchException(FeedbackManager.error_exception(error=str(e)))
|
|
125
|
+
|
|
126
|
+
if not workspace_to_delete:
|
|
127
|
+
raise CLIBranchException(FeedbackManager.error_branch(branch=branch_name_or_id))
|
|
128
|
+
|
|
129
|
+
if yes or click.confirm(FeedbackManager.warning_confirm_delete_branch(branch=workspace_to_delete["name"])):
|
|
130
|
+
need_to_switch_to_main = workspace_to_delete.get("main") and config["id"] == workspace_to_delete["id"]
|
|
131
|
+
# get origin workspace if deleting current branch
|
|
132
|
+
if need_to_switch_to_main:
|
|
133
|
+
try:
|
|
134
|
+
workspaces = (client.user_workspaces()).get("workspaces", [])
|
|
135
|
+
workspace_main = next(
|
|
136
|
+
(workspace for workspace in workspaces if workspace["id"] == workspace_to_delete["main"]), None
|
|
137
|
+
)
|
|
138
|
+
except Exception:
|
|
139
|
+
workspace_main = None
|
|
140
|
+
try:
|
|
141
|
+
client.delete_branch(workspace_to_delete["id"])
|
|
142
|
+
click.echo(FeedbackManager.success_branch_deleted(branch_name=workspace_to_delete["name"]))
|
|
143
|
+
except Exception as e:
|
|
144
|
+
raise CLIBranchException(FeedbackManager.error_exception(error=str(e)))
|
|
145
|
+
else:
|
|
146
|
+
if need_to_switch_to_main:
|
|
147
|
+
if workspace_main:
|
|
148
|
+
switch_to_workspace_by_user_workspace_data(config, workspace_main)
|
|
149
|
+
else:
|
|
150
|
+
raise CLIException(FeedbackManager.error_switching_to_main())
|
tinybird/tb/modules/build.py
CHANGED
|
@@ -3,12 +3,12 @@ import time
|
|
|
3
3
|
from copy import deepcopy
|
|
4
4
|
from functools import partial
|
|
5
5
|
from pathlib import Path
|
|
6
|
-
from typing import Any, Callable, Dict, List
|
|
6
|
+
from typing import Any, Callable, Dict, List, Optional
|
|
7
7
|
from urllib.parse import urlencode
|
|
8
8
|
|
|
9
9
|
import click
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
from tinybird import context
|
|
12
12
|
from tinybird.datafile.exceptions import ParseException
|
|
13
13
|
from tinybird.datafile.parse_datasource import parse_datasource
|
|
14
14
|
from tinybird.datafile.parse_pipe import parse_pipe
|
|
@@ -34,7 +34,11 @@ def build(ctx: click.Context, watch: bool) -> None:
|
|
|
34
34
|
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
35
35
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
36
36
|
tb_client: TinyB = ctx.ensure_object(dict)["client"]
|
|
37
|
-
|
|
37
|
+
config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
|
|
38
|
+
is_branch = bool(ctx.ensure_object(dict)["branch"])
|
|
39
|
+
|
|
40
|
+
# TODO: Explain that you can use custom branches too once they are open for everyone
|
|
41
|
+
if obj["env"] == "cloud" and not is_branch:
|
|
38
42
|
raise click.ClickException(FeedbackManager.error_build_only_supported_in_local())
|
|
39
43
|
|
|
40
44
|
if project.has_deeper_level():
|
|
@@ -47,48 +51,89 @@ def build(ctx: click.Context, watch: bool) -> None:
|
|
|
47
51
|
)
|
|
48
52
|
|
|
49
53
|
click.echo(FeedbackManager.highlight_building_project())
|
|
50
|
-
process(project=project, tb_client=tb_client, watch=False)
|
|
54
|
+
process(project=project, tb_client=tb_client, watch=False, config=config, is_branch=is_branch)
|
|
51
55
|
if watch:
|
|
52
56
|
run_watch(
|
|
53
57
|
project=project,
|
|
54
58
|
tb_client=tb_client,
|
|
55
|
-
|
|
59
|
+
config=config,
|
|
60
|
+
process=partial(
|
|
61
|
+
process,
|
|
62
|
+
project=project,
|
|
63
|
+
tb_client=tb_client,
|
|
64
|
+
watch=True,
|
|
65
|
+
config=config,
|
|
66
|
+
is_branch=is_branch,
|
|
67
|
+
),
|
|
56
68
|
)
|
|
57
69
|
|
|
58
70
|
|
|
59
71
|
@cli.command("dev", help="Build the project server side and watch for changes.")
|
|
60
72
|
@click.option("--data-origin", type=str, default="", help="Data origin: local or cloud")
|
|
61
|
-
@click.option("--ui", is_flag=True, default=
|
|
73
|
+
@click.option("--ui/--skip-ui", is_flag=True, default=True, help="Connect your local project to Tinybird UI")
|
|
62
74
|
@click.pass_context
|
|
63
75
|
def dev(ctx: click.Context, data_origin: str, ui: bool) -> None:
|
|
76
|
+
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
77
|
+
branch: Optional[str] = ctx.ensure_object(dict)["branch"]
|
|
78
|
+
is_branch = bool(branch)
|
|
79
|
+
|
|
80
|
+
if obj["env"] == "cloud" and not is_branch:
|
|
81
|
+
raise click.ClickException(FeedbackManager.error_build_only_supported_in_local())
|
|
82
|
+
|
|
64
83
|
if data_origin == "cloud":
|
|
84
|
+
click.echo(
|
|
85
|
+
FeedbackManager.warning(
|
|
86
|
+
message="--data-origin=cloud is deprecated and will be removed in a future version. Create an branch and use `tb --branch <branch_name> dev`"
|
|
87
|
+
)
|
|
88
|
+
)
|
|
65
89
|
return dev_cloud(ctx)
|
|
90
|
+
|
|
66
91
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
67
92
|
tb_client: TinyB = ctx.ensure_object(dict)["client"]
|
|
93
|
+
config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
|
|
94
|
+
|
|
68
95
|
build_status = BuildStatus()
|
|
69
96
|
if ui:
|
|
70
97
|
server_thread = threading.Thread(
|
|
71
|
-
target=start_server, args=(project, tb_client, process, build_status), daemon=True
|
|
98
|
+
target=start_server, args=(project, tb_client, process, build_status, branch), daemon=True
|
|
72
99
|
)
|
|
73
100
|
server_thread.start()
|
|
74
101
|
# Wait for the server to start
|
|
75
102
|
time.sleep(0.5)
|
|
76
103
|
|
|
77
104
|
click.echo(FeedbackManager.highlight_building_project())
|
|
78
|
-
process(
|
|
105
|
+
process(
|
|
106
|
+
project=project,
|
|
107
|
+
tb_client=tb_client,
|
|
108
|
+
watch=True,
|
|
109
|
+
config=config,
|
|
110
|
+
build_status=build_status,
|
|
111
|
+
is_branch=is_branch,
|
|
112
|
+
)
|
|
79
113
|
run_watch(
|
|
80
114
|
project=project,
|
|
81
115
|
tb_client=tb_client,
|
|
82
|
-
|
|
116
|
+
config=config,
|
|
117
|
+
branch=branch,
|
|
118
|
+
process=partial(
|
|
119
|
+
process,
|
|
120
|
+
project=project,
|
|
121
|
+
tb_client=tb_client,
|
|
122
|
+
build_status=build_status,
|
|
123
|
+
config=config,
|
|
124
|
+
is_branch=is_branch,
|
|
125
|
+
),
|
|
83
126
|
)
|
|
84
127
|
|
|
85
128
|
|
|
86
|
-
def run_watch(
|
|
87
|
-
|
|
129
|
+
def run_watch(
|
|
130
|
+
project: Project, tb_client: TinyB, process: Callable, config: dict[str, Any], branch: Optional[str] = None
|
|
131
|
+
) -> None:
|
|
132
|
+
shell = Shell(project=project, tb_client=tb_client, branch=branch)
|
|
88
133
|
click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
|
|
89
134
|
watcher_thread = threading.Thread(
|
|
90
135
|
target=watch_project,
|
|
91
|
-
args=(shell, process, project),
|
|
136
|
+
args=(shell, process, project, config),
|
|
92
137
|
daemon=True,
|
|
93
138
|
)
|
|
94
139
|
watcher_thread.start()
|
|
@@ -136,7 +181,7 @@ def dev_cloud(
|
|
|
136
181
|
context.disable_template_security_validation.set(True)
|
|
137
182
|
|
|
138
183
|
def process(filenames: List[str], watch: bool = False):
|
|
139
|
-
datafiles = [f for f in filenames if f.endswith(".datasource"
|
|
184
|
+
datafiles = [f for f in filenames if f.endswith((".datasource", ".pipe"))]
|
|
140
185
|
if len(datafiles) > 0:
|
|
141
186
|
check_filenames(filenames=datafiles)
|
|
142
187
|
folder_playground(
|