tinybird 0.0.1.dev41__tar.gz → 0.0.1.dev43__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/PKG-INFO +1 -1
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/client.py +1 -1
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/connectors.py +3 -3
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/datafile.py +11 -11
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/feedback_manager.py +1 -1
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/prompts.py +11 -1
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/sql.py +1 -1
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/sql_template_fmt.py +1 -1
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/__cli__.py +2 -2
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/build.py +34 -24
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/build_client.py +1 -1
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/cicd.py +2 -2
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/cli.py +6 -4
- tinybird-0.0.1.dev43/tinybird/tb/modules/copy.py +159 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/datafile/build.py +52 -26
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/deployment.py +86 -61
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/endpoint.py +1 -1
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/llm_utils.py +2 -2
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/pipe.py +0 -90
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/project.py +25 -1
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/shell.py +4 -4
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/watch.py +54 -5
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/cli.py +1 -1
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/common.py +1 -1
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/pipe.py +1 -1
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +1 -1
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tornado_template.py +2 -2
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird.egg-info/PKG-INFO +1 -1
- tinybird-0.0.1.dev41/tinybird/tb/modules/copy.py +0 -68
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/setup.cfg +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/__cli__.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/ch_utils/constants.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/ch_utils/engine.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/check_pypi.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/config.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/context.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/datatypes.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/git_settings.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/sql_template.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/sql_toolset.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/syncasync.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/cli.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/auth.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/common.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/config.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/create.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/datafile/build_common.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/datafile/build_datasource.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/datafile/build_pipe.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/datafile/common.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/datafile/diff.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/datafile/exceptions.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/datafile/fixture.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/datafile/format_common.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/datafile/format_datasource.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/datafile/format_pipe.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/datafile/parse_datasource.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/datafile/parse_pipe.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/datafile/pipe_checker.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/datafile/pull.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/datasource.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/exceptions.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/feedback_manager.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/fmt.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/job.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/llm.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/local.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/local_common.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/login.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/mock.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/regions.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/table.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/tag.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/telemetry.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/test.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/tinyunit/tinyunit.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/token.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/update.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/workspace.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb/modules/workspace_members.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/auth.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/branch.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/cicd.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/config.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/connection.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/datasource.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/exceptions.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/fmt.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/job.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/regions.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/tag.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/telemetry.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/test.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/workspace.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird/tb_cli_modules/workspace_members.py +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird.egg-info/SOURCES.txt +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird.egg-info/dependency_links.txt +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird.egg-info/entry_points.txt +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird.egg-info/requires.txt +0 -0
- {tinybird-0.0.1.dev41 → tinybird-0.0.1.dev43}/tinybird.egg-info/top_level.txt +0 -0
|
@@ -72,7 +72,7 @@ def parse_error_response(response: Response) -> str:
|
|
|
72
72
|
if content.get("error", None):
|
|
73
73
|
error = content["error"]
|
|
74
74
|
if content.get("errors", None):
|
|
75
|
-
error += f
|
|
75
|
+
error += f" -> errors: {content.get('errors')}"
|
|
76
76
|
else:
|
|
77
77
|
error = json.dumps(response, indent=4)
|
|
78
78
|
return error
|
|
@@ -246,7 +246,7 @@ class BigQuery(Connector):
|
|
|
246
246
|
uri='{self.gcs.gs_url()}{destination}*.csv',
|
|
247
247
|
format='CSV',
|
|
248
248
|
overwrite=true,
|
|
249
|
-
header={
|
|
249
|
+
header={"true" if with_headers else "false"},
|
|
250
250
|
field_delimiter=',') AS
|
|
251
251
|
{sql}
|
|
252
252
|
"""
|
|
@@ -319,7 +319,7 @@ class Snowflake(Connector):
|
|
|
319
319
|
|
|
320
320
|
def create_stage(self):
|
|
321
321
|
sql = f"""
|
|
322
|
-
create stage "{self.options[
|
|
322
|
+
create stage "{self.options["schema"]}".{self.stage()}
|
|
323
323
|
url='{self.gcs.gcs_url()}'
|
|
324
324
|
storage_integration = {self.storage_integration()};
|
|
325
325
|
"""
|
|
@@ -337,7 +337,7 @@ class Snowflake(Connector):
|
|
|
337
337
|
from ({sql})
|
|
338
338
|
overwrite = true
|
|
339
339
|
file_format = (TYPE=CSV COMPRESSION=NONE ESCAPE_UNENCLOSED_FIELD=NONE FIELD_DELIMITER='|' FIELD_OPTIONALLY_ENCLOSED_BY='"' null_if=())
|
|
340
|
-
header = {"true" if with_headers else "false"
|
|
340
|
+
header = {"true" if with_headers else "false"}
|
|
341
341
|
max_file_size = 2500000000;
|
|
342
342
|
"""
|
|
343
343
|
self.execute(sql)
|
|
@@ -2175,8 +2175,8 @@ class PipeCheckerRunner:
|
|
|
2175
2175
|
AND extractURLParameter(assumeNotNull(url), 'debug') <> 'query'
|
|
2176
2176
|
AND error = 0
|
|
2177
2177
|
AND not mapContains(parameters, '__tb__semver')
|
|
2178
|
-
{" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else
|
|
2179
|
-
{
|
|
2178
|
+
{" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ""}
|
|
2179
|
+
{extra_where_clause}
|
|
2180
2180
|
Limit 5000000 -- Enough to bring data while not processing all requests from highly used pipes
|
|
2181
2181
|
)
|
|
2182
2182
|
group by request_param_names, http_method
|
|
@@ -2202,7 +2202,7 @@ class PipeCheckerRunner:
|
|
|
2202
2202
|
AND extractURLParameter(assumeNotNull(url), 'debug') <> 'query'
|
|
2203
2203
|
AND error = 0
|
|
2204
2204
|
AND not mapContains(parameters, '__tb__semver')
|
|
2205
|
-
{" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else
|
|
2205
|
+
{" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ""}
|
|
2206
2206
|
{extra_where_clause}
|
|
2207
2207
|
LIMIT {limit}
|
|
2208
2208
|
)
|
|
@@ -4425,7 +4425,7 @@ async def folder_push(
|
|
|
4425
4425
|
name=(
|
|
4426
4426
|
name
|
|
4427
4427
|
if to_run[name]["version"] is None
|
|
4428
|
-
else f
|
|
4428
|
+
else f"{name}__v{to_run[name]['version']}"
|
|
4429
4429
|
)
|
|
4430
4430
|
)
|
|
4431
4431
|
)
|
|
@@ -4442,7 +4442,7 @@ async def folder_push(
|
|
|
4442
4442
|
if raise_on_exists:
|
|
4443
4443
|
raise AlreadyExistsException(
|
|
4444
4444
|
FeedbackManager.warning_name_already_exists(
|
|
4445
|
-
name=name if to_run[name]["version"] is None else f
|
|
4445
|
+
name=name if to_run[name]["version"] is None else f"{name}__v{to_run[name]['version']}"
|
|
4446
4446
|
)
|
|
4447
4447
|
)
|
|
4448
4448
|
else:
|
|
@@ -4457,7 +4457,7 @@ async def folder_push(
|
|
|
4457
4457
|
name=(
|
|
4458
4458
|
name
|
|
4459
4459
|
if to_run[name]["version"] is None
|
|
4460
|
-
else f
|
|
4460
|
+
else f"{name}__v{to_run[name]['version']}"
|
|
4461
4461
|
)
|
|
4462
4462
|
)
|
|
4463
4463
|
)
|
|
@@ -5013,7 +5013,7 @@ async def format_engine(
|
|
|
5013
5013
|
else:
|
|
5014
5014
|
if node.get("engine", None):
|
|
5015
5015
|
empty = '""'
|
|
5016
|
-
file_parts.append(f
|
|
5016
|
+
file_parts.append(f"ENGINE {node['engine']['type']}" if node.get("engine", {}).get("type") else empty)
|
|
5017
5017
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
5018
5018
|
for arg in sorted(node["engine"].get("args", [])):
|
|
5019
5019
|
elem = ", ".join([x.strip() for x in arg[1].split(",")])
|
|
@@ -5030,7 +5030,7 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
|
|
|
5030
5030
|
if node_type == PipeNodeTypes.MATERIALIZED:
|
|
5031
5031
|
file_parts.append(node_type_upper)
|
|
5032
5032
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
5033
|
-
file_parts.append(f
|
|
5033
|
+
file_parts.append(f"DATASOURCE {node['datasource']}")
|
|
5034
5034
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
5035
5035
|
await format_engine(file_parts, node)
|
|
5036
5036
|
|
|
@@ -5038,10 +5038,10 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
|
|
|
5038
5038
|
if node_type == PipeNodeTypes.COPY:
|
|
5039
5039
|
file_parts.append(node_type_upper)
|
|
5040
5040
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
5041
|
-
file_parts.append(f
|
|
5041
|
+
file_parts.append(f"TARGET_DATASOURCE {node['target_datasource']}")
|
|
5042
5042
|
if node.get("mode"):
|
|
5043
5043
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
5044
|
-
file_parts.append(f
|
|
5044
|
+
file_parts.append(f"COPY_MODE {node.get('mode')}")
|
|
5045
5045
|
|
|
5046
5046
|
if node.get(CopyParameters.COPY_SCHEDULE):
|
|
5047
5047
|
is_ondemand = node[CopyParameters.COPY_SCHEDULE].lower() == ON_DEMAND
|
|
@@ -5095,7 +5095,7 @@ async def format_node(
|
|
|
5095
5095
|
if item and not unroll_includes:
|
|
5096
5096
|
return
|
|
5097
5097
|
|
|
5098
|
-
file_parts.append(f
|
|
5098
|
+
file_parts.append(f"NODE {node['name'].strip()}")
|
|
5099
5099
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
5100
5100
|
|
|
5101
5101
|
from collections import namedtuple
|
|
@@ -951,7 +951,7 @@ Ready? """
|
|
|
951
951
|
)
|
|
952
952
|
success_datasource_alter = success_message("** The Data Source has been correctly updated.")
|
|
953
953
|
success_datasource_kafka_connected = success_message(
|
|
954
|
-
"** Data Source '{id}' created\n
|
|
954
|
+
"** Data Source '{id}' created\n** Kafka streaming connection configured successfully!"
|
|
955
955
|
)
|
|
956
956
|
success_datasource_shared = success_message(
|
|
957
957
|
"** The Data Source {datasource} has been correctly shared with {workspace}"
|
|
@@ -371,10 +371,20 @@ You are a Tinybird expert. You will be given a pipe containing different nodes w
|
|
|
371
371
|
- The test command must be a valid Tinybird command that can be run in the terminal.
|
|
372
372
|
- The test command can have as many parameters as are needed to test the pipe.
|
|
373
373
|
- The parameter within Tinybird templating syntax looks like this one {{String(my_param_name, default_value)}}.
|
|
374
|
-
- If there are no parameters, you can omit parameters and generate a single test
|
|
374
|
+
- If there are no parameters, you can omit parameters and generate a single test.
|
|
375
375
|
- The format of the parameters is the following: ?param1=value1¶m2=value2¶m3=value3
|
|
376
|
+
- If some parameters are provided by the user and you need to use them, preserve in the same format as they were provided, like case sensitive.
|
|
376
377
|
</instructions>
|
|
377
378
|
|
|
379
|
+
This is an example of a test with parameters:
|
|
380
|
+
<example>
|
|
381
|
+
<test>
|
|
382
|
+
<name>kpis_date_range</name>
|
|
383
|
+
<description>Test specific date range with daily granularity</description>
|
|
384
|
+
<parameters>?date_from=2024-01-01&date_to=2024-01-10</parameters>
|
|
385
|
+
</test>
|
|
386
|
+
</example>
|
|
387
|
+
|
|
378
388
|
Follow the instructions and generate the following response with no additional text:
|
|
379
389
|
|
|
380
390
|
<response>
|
|
@@ -241,7 +241,7 @@ def format_parse_error(
|
|
|
241
241
|
message += f" found at position {adjusted_position - len(keyword)}"
|
|
242
242
|
else:
|
|
243
243
|
message += (
|
|
244
|
-
f" found {repr(table_structure[i]) if len(table_structure)>i else 'EOF'} at position {adjusted_position}"
|
|
244
|
+
f" found {repr(table_structure[i]) if len(table_structure) > i else 'EOF'} at position {adjusted_position}"
|
|
245
245
|
)
|
|
246
246
|
return message
|
|
247
247
|
|
|
@@ -100,7 +100,7 @@ def _format_jinja_node(self, node: Node, max_length: int) -> bool:
|
|
|
100
100
|
parts = tag.code.split("\n")
|
|
101
101
|
prefix = INDENT * (node.depth[0] + node.depth[1])
|
|
102
102
|
if len(parts) > 1:
|
|
103
|
-
tag.code = "\n".join([f
|
|
103
|
+
tag.code = "\n".join([f"{prefix if i != 0 else ''}{part}" for i, part in enumerate(parts)])
|
|
104
104
|
|
|
105
105
|
node.value = str(tag)
|
|
106
106
|
|
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '0.0.1.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '0.0.1.dev43'
|
|
8
|
+
__revision__ = 'ecb1311'
|
|
@@ -31,18 +31,16 @@ def build(ctx: click.Context, watch: bool) -> None:
|
|
|
31
31
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
32
32
|
tb_client = asyncio.run(get_tinybird_local_client(str(project.path)))
|
|
33
33
|
click.echo(FeedbackManager.highlight(message="\n» Building project..."))
|
|
34
|
-
|
|
35
34
|
time_start = time.time()
|
|
36
35
|
|
|
37
|
-
def process(file_changed: Optional[str] = None) -> None:
|
|
36
|
+
def process(file_changed: Optional[str] = None, diff: Optional[str] = None) -> None:
|
|
38
37
|
if file_changed and file_changed.endswith(".ndjson"):
|
|
39
|
-
rebuild_fixture(project, file_changed)
|
|
38
|
+
rebuild_fixture(project, tb_client, file_changed)
|
|
40
39
|
else:
|
|
41
40
|
build_project(project, tb_client)
|
|
42
|
-
new_tb_client = asyncio.run(get_tinybird_local_client(str(project.path)))
|
|
43
41
|
try:
|
|
44
42
|
if file_changed:
|
|
45
|
-
build_and_print_resource(
|
|
43
|
+
build_and_print_resource(tb_client, file_changed, diff)
|
|
46
44
|
except Exception:
|
|
47
45
|
pass
|
|
48
46
|
|
|
@@ -52,7 +50,7 @@ def build(ctx: click.Context, watch: bool) -> None:
|
|
|
52
50
|
click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s"))
|
|
53
51
|
|
|
54
52
|
if watch:
|
|
55
|
-
shell = Shell(project=project)
|
|
53
|
+
shell = Shell(project=project, tb_client=tb_client)
|
|
56
54
|
click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
|
|
57
55
|
watcher_thread = threading.Thread(
|
|
58
56
|
target=watch_project,
|
|
@@ -80,6 +78,9 @@ def build_project(project: Project, tb_client: TinyB) -> None:
|
|
|
80
78
|
project_path = project.path
|
|
81
79
|
project_files = project.get_project_files()
|
|
82
80
|
|
|
81
|
+
if not project_files:
|
|
82
|
+
return
|
|
83
|
+
|
|
83
84
|
for file_path in project_files:
|
|
84
85
|
relative_path = str(Path(file_path).relative_to(project_path))
|
|
85
86
|
fd = open(file_path, "rb")
|
|
@@ -88,7 +89,7 @@ def build_project(project: Project, tb_client: TinyB) -> None:
|
|
|
88
89
|
files.append((MULTIPART_BOUNDARY_DATA_PROJECT, (relative_path, fd.read().decode("utf-8"), content_type)))
|
|
89
90
|
HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
|
|
90
91
|
|
|
91
|
-
r = requests.post(TINYBIRD_API_URL, files=files, headers=HEADERS
|
|
92
|
+
r = requests.post(TINYBIRD_API_URL, files=files, headers=HEADERS)
|
|
92
93
|
try:
|
|
93
94
|
result = r.json()
|
|
94
95
|
except Exception as e:
|
|
@@ -103,19 +104,21 @@ def build_project(project: Project, tb_client: TinyB) -> None:
|
|
|
103
104
|
datasources = result.get("datasources", [])
|
|
104
105
|
pipes = result.get("pipes", [])
|
|
105
106
|
for ds in datasources:
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
107
|
+
ds_path_str: Optional[str] = next(
|
|
108
|
+
(p for p in project_files if p.endswith(ds.get("name") + ".datasource")), None
|
|
109
|
+
)
|
|
110
|
+
if ds_path_str:
|
|
111
|
+
ds_path = Path(ds_path_str)
|
|
112
|
+
ds_path_str = ds_path_str.replace(f"{project.folder}/", "")
|
|
113
|
+
click.echo(FeedbackManager.info(message=f"✓ {ds_path_str} created"))
|
|
110
114
|
for pipe in pipes:
|
|
111
115
|
pipe_name = pipe.get("name")
|
|
112
|
-
|
|
113
|
-
if
|
|
114
|
-
|
|
115
|
-
click.echo(FeedbackManager.info(message=f"✓ {
|
|
116
|
+
pipe_path_str: Optional[str] = next((p for p in project_files if p.endswith(pipe_name + ".pipe")), None)
|
|
117
|
+
if pipe_path_str:
|
|
118
|
+
pipe_path_str = pipe_path_str.replace(f"{project.folder}/", "")
|
|
119
|
+
click.echo(FeedbackManager.info(message=f"✓ {pipe_path_str} created"))
|
|
116
120
|
|
|
117
121
|
try:
|
|
118
|
-
new_tb_client = asyncio.run(get_tinybird_local_client(str(project.path)))
|
|
119
122
|
for filename in project_files:
|
|
120
123
|
if filename.endswith(".datasource"):
|
|
121
124
|
ds_path = Path(filename)
|
|
@@ -128,7 +131,7 @@ def build_project(project: Project, tb_client: TinyB) -> None:
|
|
|
128
131
|
fixture_path = fixture_folder / f"{ds_name}.ndjson"
|
|
129
132
|
|
|
130
133
|
if fixture_path.exists():
|
|
131
|
-
append_fixture(
|
|
134
|
+
append_fixture(tb_client, ds_name, str(fixture_path))
|
|
132
135
|
except Exception:
|
|
133
136
|
pass
|
|
134
137
|
|
|
@@ -166,9 +169,8 @@ def append_fixture(
|
|
|
166
169
|
)
|
|
167
170
|
|
|
168
171
|
|
|
169
|
-
def rebuild_fixture(project: Project, fixture: str) -> None:
|
|
172
|
+
def rebuild_fixture(project: Project, tb_client: TinyB, fixture: str) -> None:
|
|
170
173
|
try:
|
|
171
|
-
tb_client = asyncio.run(get_tinybird_local_client(str(project.path)))
|
|
172
174
|
fixture_path = Path(fixture)
|
|
173
175
|
datasources_path = Path(project.folder) / "datasources"
|
|
174
176
|
ds_name = fixture_path.stem
|
|
@@ -188,9 +190,17 @@ def rebuild_fixture(project: Project, fixture: str) -> None:
|
|
|
188
190
|
click.echo(FeedbackManager.error_exception(error=e))
|
|
189
191
|
|
|
190
192
|
|
|
191
|
-
def build_and_print_resource(tb_client: TinyB, filename: str):
|
|
193
|
+
def build_and_print_resource(tb_client: TinyB, filename: str, diff: Optional[str] = None):
|
|
194
|
+
table_name = diff
|
|
192
195
|
resource_path = Path(filename)
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
196
|
+
resource_name = resource_path.stem
|
|
197
|
+
|
|
198
|
+
pipeline = resource_name if filename.endswith(".pipe") else None
|
|
199
|
+
|
|
200
|
+
if not table_name:
|
|
201
|
+
table_name = resource_name
|
|
202
|
+
|
|
203
|
+
sql = f"SELECT * FROM {table_name} FORMAT JSON"
|
|
204
|
+
|
|
205
|
+
res = asyncio.run(tb_client.query(sql, pipeline=pipeline))
|
|
206
|
+
print_table_formatted(res, table_name)
|
|
@@ -166,7 +166,7 @@ def build_client(
|
|
|
166
166
|
build_ok = asyncio.run(build_once(filenames))
|
|
167
167
|
|
|
168
168
|
if watch:
|
|
169
|
-
shell = Shell(project=project,
|
|
169
|
+
shell = Shell(project=project, tb_client=tb_client)
|
|
170
170
|
click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
|
|
171
171
|
watcher_thread = threading.Thread(
|
|
172
172
|
target=watch_files, args=(filenames, process, shell, project, build_ok), daemon=True
|
|
@@ -39,7 +39,7 @@ jobs:
|
|
|
39
39
|
working-directory: '{{ data_project_dir }}'
|
|
40
40
|
services:
|
|
41
41
|
tinybird:
|
|
42
|
-
image: tinybirdco/tinybird-local:
|
|
42
|
+
image: tinybirdco/tinybird-local:beta
|
|
43
43
|
ports:
|
|
44
44
|
- 80:80
|
|
45
45
|
steps:
|
|
@@ -83,7 +83,7 @@ tinybird_ci_workflow:
|
|
|
83
83
|
- tb build
|
|
84
84
|
- tb test run
|
|
85
85
|
services:
|
|
86
|
-
- name: tinybirdco/tinybird-local:
|
|
86
|
+
- name: tinybirdco/tinybird-local:beta
|
|
87
87
|
alias: tinybird-local
|
|
88
88
|
"""
|
|
89
89
|
|
|
@@ -66,15 +66,17 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
|
|
|
66
66
|
@click.option("--host", help="Use custom host, defaults to TB_HOST envvar, then to https://api.tinybird.co")
|
|
67
67
|
@click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
|
|
68
68
|
@click.option("--prod/--local", is_flag=True, default=False, help="Run against production or local")
|
|
69
|
-
@click.option("--folder", type=str,
|
|
69
|
+
@click.option("--folder", type=str, help="Folder where files will be placed")
|
|
70
70
|
@click.version_option(version=VERSION)
|
|
71
71
|
@click.pass_context
|
|
72
72
|
@coro
|
|
73
|
-
async def cli(
|
|
73
|
+
async def cli(
|
|
74
|
+
ctx: Context, debug: bool, token: str, host: str, show_tokens: bool, prod: bool, folder: Optional[str]
|
|
75
|
+
) -> None:
|
|
74
76
|
"""
|
|
75
77
|
Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
|
|
76
78
|
"""
|
|
77
|
-
project = Project(folder=folder)
|
|
79
|
+
project = Project(folder=folder or os.getcwd())
|
|
78
80
|
# We need to unpatch for our tests not to break
|
|
79
81
|
if show_tokens or not prod or ctx.invoked_subcommand == "build":
|
|
80
82
|
__unpatch_click_output()
|
|
@@ -87,7 +89,7 @@ async def cli(ctx: Context, debug: bool, token: str, host: str, show_tokens: boo
|
|
|
87
89
|
if debug:
|
|
88
90
|
logging.basicConfig(level=logging.DEBUG)
|
|
89
91
|
|
|
90
|
-
config_temp = CLIConfig.get_project_config(project.path)
|
|
92
|
+
config_temp = CLIConfig.get_project_config(str(project.path))
|
|
91
93
|
if token:
|
|
92
94
|
config_temp.set_token(token)
|
|
93
95
|
if host:
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
# This is a command file for our CLI. Please keep it clean.
|
|
2
|
+
#
|
|
3
|
+
# - If it makes sense and only when strictly necessary, you can create utility functions in this file.
|
|
4
|
+
# - But please, **do not** interleave utility functions and command definitions.
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import re
|
|
8
|
+
from typing import Optional, Tuple
|
|
9
|
+
|
|
10
|
+
import click
|
|
11
|
+
from click import Context
|
|
12
|
+
|
|
13
|
+
from tinybird.client import AuthNoTokenException, TinyB
|
|
14
|
+
from tinybird.tb.modules.cli import cli
|
|
15
|
+
from tinybird.tb.modules.common import coro, echo_safe_humanfriendly_tables_format_smart_table, wait_job
|
|
16
|
+
from tinybird.tb.modules.datafile.common import get_name_version
|
|
17
|
+
from tinybird.tb.modules.exceptions import CLIPipeException
|
|
18
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@cli.group(hidden=True)
|
|
22
|
+
@click.pass_context
|
|
23
|
+
def copy(ctx):
|
|
24
|
+
"""Copy pipe commands"""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@copy.command(name="ls")
|
|
28
|
+
@click.option("--match", default=None, help="Retrieve any resourcing matching the pattern. eg --match _test")
|
|
29
|
+
@click.option(
|
|
30
|
+
"--format",
|
|
31
|
+
"format_",
|
|
32
|
+
type=click.Choice(["json"], case_sensitive=False),
|
|
33
|
+
default=None,
|
|
34
|
+
help="Force a type of the output",
|
|
35
|
+
)
|
|
36
|
+
@click.pass_context
|
|
37
|
+
@coro
|
|
38
|
+
async def copy_ls(ctx: Context, match: str, format_: str):
|
|
39
|
+
"""List copy pipes"""
|
|
40
|
+
|
|
41
|
+
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
42
|
+
pipes = await client.pipes(dependencies=False, node_attrs="name", attrs="name,updated_at,type")
|
|
43
|
+
copies = [p for p in pipes if p.get("type") == "copy"]
|
|
44
|
+
copies = sorted(copies, key=lambda p: p["updated_at"])
|
|
45
|
+
columns = ["name", "updated at", "nodes"]
|
|
46
|
+
table_human_readable = []
|
|
47
|
+
table_machine_readable = []
|
|
48
|
+
pattern = re.compile(match) if match else None
|
|
49
|
+
for t in copies:
|
|
50
|
+
tk = get_name_version(t["name"])
|
|
51
|
+
if pattern and not pattern.search(tk["name"]):
|
|
52
|
+
continue
|
|
53
|
+
table_human_readable.append((tk["name"], t["updated_at"][:-7], len(t["nodes"])))
|
|
54
|
+
table_machine_readable.append(
|
|
55
|
+
{
|
|
56
|
+
"name": tk["name"],
|
|
57
|
+
"updated at": t["updated_at"][:-7],
|
|
58
|
+
"nodes": len(t["nodes"]),
|
|
59
|
+
}
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
if not format_:
|
|
63
|
+
click.echo(FeedbackManager.info_pipes())
|
|
64
|
+
echo_safe_humanfriendly_tables_format_smart_table(table_human_readable, column_names=columns)
|
|
65
|
+
click.echo("\n")
|
|
66
|
+
elif format_ == "json":
|
|
67
|
+
click.echo(json.dumps({"pipes": table_machine_readable}, indent=2))
|
|
68
|
+
else:
|
|
69
|
+
raise CLIPipeException(FeedbackManager.error_pipe_ls_type())
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@copy.command(name="run", short_help="Run an on-demand copy job")
|
|
73
|
+
@click.argument("pipe_name_or_id")
|
|
74
|
+
@click.option("--wait", is_flag=True, default=False, help="Wait for the copy job to finish")
|
|
75
|
+
@click.option(
|
|
76
|
+
"--mode", type=click.Choice(["append", "replace"], case_sensitive=True), default=None, help="Copy strategy"
|
|
77
|
+
)
|
|
78
|
+
@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
|
|
79
|
+
@click.option(
|
|
80
|
+
"--param",
|
|
81
|
+
nargs=1,
|
|
82
|
+
type=str,
|
|
83
|
+
multiple=True,
|
|
84
|
+
default=None,
|
|
85
|
+
help="Key and value of the params you want the Copy pipe to be called with. For example: tb pipe copy run <my_copy_pipe> --param foo=bar",
|
|
86
|
+
)
|
|
87
|
+
@click.pass_context
|
|
88
|
+
@coro
|
|
89
|
+
async def copy_run(
|
|
90
|
+
ctx: click.Context, pipe_name_or_id: str, wait: bool, mode: str, yes: bool, param: Optional[Tuple[str]]
|
|
91
|
+
):
|
|
92
|
+
"""Run an on-demand copy job"""
|
|
93
|
+
|
|
94
|
+
params = dict(key_value.split("=") for key_value in param) if param else {}
|
|
95
|
+
|
|
96
|
+
if yes or click.confirm(FeedbackManager.warning_confirm_copy_pipe(pipe=pipe_name_or_id)):
|
|
97
|
+
click.echo(FeedbackManager.info_copy_job_running(pipe=pipe_name_or_id))
|
|
98
|
+
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
99
|
+
|
|
100
|
+
try:
|
|
101
|
+
response = await client.pipe_run_copy(pipe_name_or_id, params, mode)
|
|
102
|
+
|
|
103
|
+
job_id = response["job"]["job_id"]
|
|
104
|
+
job_url = response["job"]["job_url"]
|
|
105
|
+
target_datasource_id = response["tags"]["copy_target_datasource"]
|
|
106
|
+
target_datasource = await client.get_datasource(target_datasource_id)
|
|
107
|
+
target_datasource_name = target_datasource["name"]
|
|
108
|
+
click.echo(
|
|
109
|
+
FeedbackManager.success_copy_job_created(target_datasource=target_datasource_name, job_url=job_url)
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
if wait:
|
|
113
|
+
await wait_job(client, job_id, job_url, "** Copying data")
|
|
114
|
+
click.echo(FeedbackManager.success_data_copied_to_ds(target_datasource=target_datasource_name))
|
|
115
|
+
|
|
116
|
+
except AuthNoTokenException:
|
|
117
|
+
raise
|
|
118
|
+
except Exception as e:
|
|
119
|
+
raise CLIPipeException(FeedbackManager.error_creating_copy_job(error=e))
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@copy.command(name="resume", short_help="Resume a paused copy pipe")
|
|
123
|
+
@click.argument("pipe_name_or_id")
|
|
124
|
+
@click.pass_context
|
|
125
|
+
@coro
|
|
126
|
+
async def copy_resume(ctx: click.Context, pipe_name_or_id: str):
|
|
127
|
+
"""Resume a paused copy pipe"""
|
|
128
|
+
|
|
129
|
+
click.echo(FeedbackManager.info_copy_pipe_resuming(pipe=pipe_name_or_id))
|
|
130
|
+
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
131
|
+
|
|
132
|
+
try:
|
|
133
|
+
await client.pipe_resume_copy(pipe_name_or_id)
|
|
134
|
+
click.echo(FeedbackManager.success_copy_pipe_resumed(pipe=pipe_name_or_id))
|
|
135
|
+
|
|
136
|
+
except AuthNoTokenException:
|
|
137
|
+
raise
|
|
138
|
+
except Exception as e:
|
|
139
|
+
raise CLIPipeException(FeedbackManager.error_resuming_copy_pipe(error=e))
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@copy.command(name="pause", short_help="Pause a running copy pipe")
|
|
143
|
+
@click.argument("pipe_name_or_id")
|
|
144
|
+
@click.pass_context
|
|
145
|
+
@coro
|
|
146
|
+
async def copy_pause(ctx: click.Context, pipe_name_or_id: str):
|
|
147
|
+
"""Pause a running copy pipe"""
|
|
148
|
+
|
|
149
|
+
click.echo(FeedbackManager.info_copy_pipe_pausing(pipe=pipe_name_or_id))
|
|
150
|
+
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
151
|
+
|
|
152
|
+
try:
|
|
153
|
+
await client.pipe_pause_copy(pipe_name_or_id)
|
|
154
|
+
click.echo(FeedbackManager.success_copy_pipe_paused(pipe=pipe_name_or_id))
|
|
155
|
+
|
|
156
|
+
except AuthNoTokenException:
|
|
157
|
+
raise
|
|
158
|
+
except Exception as e:
|
|
159
|
+
raise CLIPipeException(FeedbackManager.error_pausing_copy_pipe(error=e))
|