tinybird 0.0.1.dev43__tar.gz → 0.0.1.dev46__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/PKG-INFO +1 -1
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/client.py +17 -1
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/prompts.py +135 -15
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/__cli__.py +2 -2
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/cli.py +1 -1
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/build.py +28 -20
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/cli.py +18 -62
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/common.py +3 -2
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/copy.py +1 -1
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/create.py +134 -59
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/build.py +12 -221
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/common.py +1 -1
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/format_datasource.py +1 -1
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/format_pipe.py +4 -4
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/pipe_checker.py +3 -3
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datasource.py +1 -1
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/deployment.py +1 -1
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/endpoint.py +89 -2
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/feedback_manager.py +5 -1
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/local_common.py +10 -7
- tinybird-0.0.1.dev46/tinybird/tb/modules/materialization.py +146 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/mock.py +56 -16
- tinybird-0.0.1.dev46/tinybird/tb/modules/pipe.py +72 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/project.py +10 -4
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/shell.py +3 -3
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/test.py +73 -38
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/tinyunit/tinyunit.py +1 -1
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/update.py +1 -1
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/workspace.py +2 -1
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird.egg-info/PKG-INFO +1 -1
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird.egg-info/SOURCES.txt +1 -1
- tinybird-0.0.1.dev43/tinybird/tb/modules/build_client.py +0 -199
- tinybird-0.0.1.dev43/tinybird/tb/modules/pipe.py +0 -390
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/setup.cfg +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/__cli__.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/ch_utils/constants.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/ch_utils/engine.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/check_pypi.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/config.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/connectors.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/context.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/datafile.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/datatypes.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/feedback_manager.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/git_settings.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/sql.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/sql_template.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/sql_template_fmt.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/sql_toolset.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/syncasync.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/auth.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/cicd.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/config.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/build_common.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/build_datasource.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/build_pipe.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/diff.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/exceptions.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/fixture.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/format_common.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/parse_datasource.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/parse_pipe.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/pull.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/exceptions.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/fmt.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/job.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/llm.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/llm_utils.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/local.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/login.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/regions.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/table.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/tag.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/telemetry.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/token.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/watch.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/workspace_members.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/auth.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/branch.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/cicd.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/cli.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/common.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/config.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/connection.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/datasource.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/exceptions.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/fmt.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/job.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/pipe.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/regions.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/tag.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/telemetry.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/test.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/workspace.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/workspace_members.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tornado_template.py +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird.egg-info/dependency_links.txt +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird.egg-info/entry_points.txt +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird.egg-info/requires.txt +0 -0
- {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import json
|
|
3
3
|
import logging
|
|
4
|
+
import os
|
|
4
5
|
import ssl
|
|
5
6
|
from pathlib import Path
|
|
6
7
|
from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Union
|
|
@@ -91,6 +92,7 @@ class TinyB:
|
|
|
91
92
|
disable_ssl_checks: bool = False,
|
|
92
93
|
send_telemetry: bool = False,
|
|
93
94
|
semver: Optional[str] = None,
|
|
95
|
+
env: Optional[str] = "production",
|
|
94
96
|
):
|
|
95
97
|
ctx = ssl.create_default_context()
|
|
96
98
|
ctx.check_hostname = False
|
|
@@ -102,6 +104,7 @@ class TinyB:
|
|
|
102
104
|
self.disable_ssl_checks = disable_ssl_checks
|
|
103
105
|
self.send_telemetry = send_telemetry
|
|
104
106
|
self.semver = semver
|
|
107
|
+
self.env = env
|
|
105
108
|
|
|
106
109
|
async def _req_raw(
|
|
107
110
|
self,
|
|
@@ -687,7 +690,20 @@ class TinyB:
|
|
|
687
690
|
return await self._req(f"/v0/jobs/{job_id}/cancel", method="POST", data=b"")
|
|
688
691
|
|
|
689
692
|
async def user_workspaces(self):
|
|
690
|
-
|
|
693
|
+
data = await self._req("/v0/user/workspaces/?with_environments=false")
|
|
694
|
+
# TODO: this is repeated in local_common.py but I'm avoiding circular imports
|
|
695
|
+
local_port = int(os.getenv("TB_LOCAL_PORT", 80))
|
|
696
|
+
local_host = f"http://localhost:{local_port}"
|
|
697
|
+
if local_host != self.host:
|
|
698
|
+
return data
|
|
699
|
+
|
|
700
|
+
local_workspaces = [
|
|
701
|
+
x
|
|
702
|
+
for x in data["workspaces"]
|
|
703
|
+
if x["name"].startswith("Tinybird_Local_") and not x["name"].startswith("Tinybird_Local_Build_")
|
|
704
|
+
]
|
|
705
|
+
|
|
706
|
+
return {**data, "workspaces": local_workspaces}
|
|
691
707
|
|
|
692
708
|
async def user_workspaces_and_branches(self):
|
|
693
709
|
return await self._req("/v0/user/workspaces/?with_environments=true")
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
|
|
1
3
|
general_functions = [
|
|
2
4
|
"BLAKE3",
|
|
3
5
|
"CAST",
|
|
@@ -368,8 +370,7 @@ You are a Tinybird expert. You will be given a pipe containing different nodes w
|
|
|
368
370
|
|
|
369
371
|
<instructions>
|
|
370
372
|
- Every test name must be unique.
|
|
371
|
-
- The test
|
|
372
|
-
- The test command can have as many parameters as are needed to test the pipe.
|
|
373
|
+
- The test can have as many parameters as are needed to test the pipe.
|
|
373
374
|
- The parameter within Tinybird templating syntax looks like this one {{String(my_param_name, default_value)}}.
|
|
374
375
|
- If there are no parameters, you can omit parameters and generate a single test.
|
|
375
376
|
- The format of the parameters is the following: ?param1=value1¶m2=value2¶m3=value3
|
|
@@ -397,9 +398,19 @@ Follow the instructions and generate the following response with no additional t
|
|
|
397
398
|
"""
|
|
398
399
|
|
|
399
400
|
|
|
400
|
-
def create_prompt(existing_resources: str) -> str:
|
|
401
|
+
def create_prompt(existing_resources: str, feedback: str = "", history: str = "") -> str:
|
|
402
|
+
feedback_history = ""
|
|
403
|
+
if feedback and history:
|
|
404
|
+
feedback_history = f"""In case the <feedback> and <history> tags are present and not empty,
|
|
405
|
+
it means there was a previous attempt to generate the resources and the user provided feedback and history about previous responses.
|
|
406
|
+
Use the following feedback and history to regenerate the response:
|
|
407
|
+
Feedback to improve the response:
|
|
408
|
+
{feedback}
|
|
409
|
+
History of previous results:
|
|
410
|
+
{history}"""
|
|
411
|
+
|
|
401
412
|
return """
|
|
402
|
-
You are a Tinybird expert. You will be given a prompt to generate Tinybird resources: datasources and/or pipes.
|
|
413
|
+
You are a Tinybird expert. You will be given a prompt to generate new or update existing Tinybird resources: datasources and/or pipes.
|
|
403
414
|
<existing_resources>{existing_resources}</existing_resources>
|
|
404
415
|
{datasource_instructions}
|
|
405
416
|
{pipe_instructions}
|
|
@@ -408,6 +419,9 @@ You are a Tinybird expert. You will be given a prompt to generate Tinybird resou
|
|
|
408
419
|
{pipe_example}
|
|
409
420
|
{copy_pipe_instructions}
|
|
410
421
|
{materialized_pipe_instructions}
|
|
422
|
+
|
|
423
|
+
{feedback_history}
|
|
424
|
+
|
|
411
425
|
Use the following format to generate the response and do not wrap it in any other text, including the <response> tag.
|
|
412
426
|
<response>
|
|
413
427
|
<resource>
|
|
@@ -426,10 +440,21 @@ Use the following format to generate the response and do not wrap it in any othe
|
|
|
426
440
|
pipe_example=pipe_example,
|
|
427
441
|
copy_pipe_instructions=copy_pipe_instructions,
|
|
428
442
|
materialized_pipe_instructions=materialized_pipe_instructions,
|
|
443
|
+
feedback_history=feedback_history,
|
|
429
444
|
)
|
|
430
445
|
|
|
431
446
|
|
|
432
|
-
def mock_prompt(rows: int) -> str:
|
|
447
|
+
def mock_prompt(rows: int, feedback: str = "", history: str = "") -> str:
|
|
448
|
+
feedback_history = ""
|
|
449
|
+
if feedback and history:
|
|
450
|
+
feedback_history = f"""In case the <feedback> and <history> tags are present and not empty,
|
|
451
|
+
it means there was a previous attempt to generate the resources and the user provided feedback and history about previous responses.
|
|
452
|
+
Use the following feedback and history to regenerate the response:
|
|
453
|
+
Feedback to improve the response:
|
|
454
|
+
{feedback}
|
|
455
|
+
History of previous results:
|
|
456
|
+
{history}"""
|
|
457
|
+
|
|
433
458
|
return f"""
|
|
434
459
|
Given the schema for a Tinybird datasource, return a can you create a clickhouse sql query to generate some random data that matches that schema.
|
|
435
460
|
|
|
@@ -570,6 +595,9 @@ Follow the instructions and generate the following response with no additional t
|
|
|
570
595
|
<response>
|
|
571
596
|
<sql>[raw sql query here]</sql>
|
|
572
597
|
</response>
|
|
598
|
+
|
|
599
|
+
{feedback_history}
|
|
600
|
+
|
|
573
601
|
"""
|
|
574
602
|
|
|
575
603
|
|
|
@@ -732,6 +760,10 @@ sql_instructions = """
|
|
|
732
760
|
AND {{{{DateTime(end_date)}}}}
|
|
733
761
|
{{%end%}}
|
|
734
762
|
</valid_condition_without_now>
|
|
763
|
+
- Parameters must not be quoted.
|
|
764
|
+
- When you use defined function with a paremeter inside, do NOT add quotes around the parameter:
|
|
765
|
+
<invalid_defined_function_with_parameter>{{% if defined('my_param') %}}</invalid_defined_function_with_parameter>
|
|
766
|
+
<valid_defined_function_without_parameter>{{% if defined(my_param) %}}</valid_defined_function_without_parameter>
|
|
735
767
|
- Use datasource names as table names when doing SELECT statements.
|
|
736
768
|
- Do not use pipe names as table names.
|
|
737
769
|
- The available datasource names to use in the SQL are the ones present in the existing_resources section or the ones you will create.
|
|
@@ -749,7 +781,6 @@ sql_instructions = """
|
|
|
749
781
|
- When aliasing a column, use first the column name and then the alias.
|
|
750
782
|
- General functions and aggregate functions are case sensitive.
|
|
751
783
|
- Character insensitive functions are case insensitive.
|
|
752
|
-
- When you use defined function with a paremeter inside, do NOT add quotes around the parameter.
|
|
753
784
|
- Parameters are never quoted in any case.
|
|
754
785
|
</sql_instructions>
|
|
755
786
|
""".format(
|
|
@@ -792,22 +823,111 @@ Use the following format to generate the response and do not wrap it in any othe
|
|
|
792
823
|
)
|
|
793
824
|
|
|
794
825
|
|
|
795
|
-
|
|
826
|
+
def rules_prompt(source: Optional[str] = None) -> str:
|
|
827
|
+
base_command = source or "tb"
|
|
828
|
+
return """
|
|
796
829
|
You are an expert in SQL and Tinybird. Follow these instructions when working with .datasource and .pipe files:
|
|
797
830
|
|
|
831
|
+
<command_calling>
|
|
832
|
+
You have commands at your disposal to develop a tinybird project:
|
|
833
|
+
- {base_command} build: to build the project locally and check it works.
|
|
834
|
+
- {base_command} deployment create --wait --auto: to create a deployment and promote it automatically
|
|
835
|
+
- {base_command} test run: to run existing tests
|
|
836
|
+
- {base_command} --build endpoint url <pipe_name>: to get the url of an endpoint, token included.
|
|
837
|
+
- {base_command} --build endpoint data <pipe_name>: to get the data of an endpoint. You can pass parameters to the endpoint like this: {base_command} --build endpoint data <pipe_name> --param1 value1 --param2 value2
|
|
838
|
+
- {base_command} --build token ls: to list all the tokens
|
|
839
|
+
There are other commands that you can use, but these are the most common ones. Run `{base_command} -h` to see all the commands if needed.
|
|
840
|
+
When you need to check resources or data in the Tinybird environment that you updated with the build command, add always the --build flag before the command. Example: {base_command} --build datasource ls
|
|
841
|
+
</command_calling>
|
|
842
|
+
<development_instructions>
|
|
843
|
+
- When asking to create a tinybird data project, if the needed folders are not already created, use the following structure:
|
|
844
|
+
├── copies
|
|
845
|
+
├── datasources
|
|
846
|
+
├── endpoints
|
|
847
|
+
├── fixtures
|
|
848
|
+
├── materializations
|
|
849
|
+
├── sinks
|
|
850
|
+
└── tests
|
|
851
|
+
- The local development server will be available at http://localhost:80. Even if some response uses another base url, use always http://localhost:80.
|
|
852
|
+
- After every change in your .datasource, .pipe or .ndjson files, run `{base_command} build` to build the project locally.
|
|
853
|
+
- When you need to ingest data locally in a datasource, create a .ndjson file with the same name of the datasource and the data you want and run `{base_command} build` so the data is ingested.
|
|
854
|
+
- The format of the generated api endpoint urls is: http://localhost:80/v0/pipe/<pipe_name>.json?token=<token>
|
|
855
|
+
- Before running the tests, remember to have the project built with `{base_command} build` with the latest changes.
|
|
856
|
+
</development_instructions>
|
|
857
|
+
|
|
858
|
+
<datasource_file_instructions>
|
|
859
|
+
Follow these instructions when creating or updating .datasource files:
|
|
798
860
|
{datasource_instructions}
|
|
861
|
+
</datasource_file_instructions>
|
|
862
|
+
|
|
863
|
+
<pipe_file_instructions>
|
|
864
|
+
Follow these instructions when creating or updating .pipe files:
|
|
799
865
|
{pipe_instructions}
|
|
800
866
|
{sql_instructions}
|
|
801
867
|
{datasource_example}
|
|
802
868
|
{pipe_example}
|
|
803
869
|
{copy_pipe_instructions}
|
|
804
870
|
{materialized_pipe_instructions}
|
|
871
|
+
</pipe_file_instructions>
|
|
872
|
+
<test_file_instructions>
|
|
873
|
+
Follow these instructions when creating or updating .yaml files for tests:
|
|
874
|
+
{test_instructions}
|
|
875
|
+
</test_file_instructions>
|
|
805
876
|
""".format(
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
877
|
+
base_command=base_command,
|
|
878
|
+
datasource_instructions=datasource_instructions,
|
|
879
|
+
pipe_instructions=pipe_instructions,
|
|
880
|
+
sql_instructions=sql_instructions,
|
|
881
|
+
datasource_example=datasource_example,
|
|
882
|
+
pipe_example=pipe_example,
|
|
883
|
+
copy_pipe_instructions=copy_pipe_instructions,
|
|
884
|
+
materialized_pipe_instructions=materialized_pipe_instructions,
|
|
885
|
+
test_instructions=test_instructions,
|
|
886
|
+
)
|
|
887
|
+
|
|
888
|
+
|
|
889
|
+
test_instructions = """
|
|
890
|
+
- The test file name must match the name of the pipe it is testing.
|
|
891
|
+
- Every scenario name must be unique inside the test file.
|
|
892
|
+
- When looking for the parameters available, you will find them in the pipes in the following format: {{{{String(my_param_name, default_value)}}}}.
|
|
893
|
+
- If there are no parameters, you can omit parameters and generate a single test.
|
|
894
|
+
- The format of the parameters is the following: param1=value1¶m2=value2¶m3=value3
|
|
895
|
+
- If some parameters are provided by the user and you need to use them, preserve in the same format as they were provided, like case sensitive
|
|
896
|
+
- Test as many scenarios as possible.
|
|
897
|
+
- The format of the test file is the following:
|
|
898
|
+
<test_file_format>
|
|
899
|
+
- name: kpis_single_day
|
|
900
|
+
description: Test hourly granularity for a single day
|
|
901
|
+
parameters: date_from=2024-01-01&date_to=2024-01-01
|
|
902
|
+
expected_result: |
|
|
903
|
+
{"date":"2024-01-01 00:00:00","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0}
|
|
904
|
+
{"date":"2024-01-01 01:00:00","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0}
|
|
905
|
+
|
|
906
|
+
- name: kpis_date_range
|
|
907
|
+
description: Test daily granularity for a date range
|
|
908
|
+
parameters: date_from=2024-01-01&date_to=2024-01-31
|
|
909
|
+
expected_result: |
|
|
910
|
+
{"date":"2024-01-01","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0}
|
|
911
|
+
{"date":"2024-01-02","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0}
|
|
912
|
+
|
|
913
|
+
- name: kpis_default_range
|
|
914
|
+
description: Test default behavior without date parameters (last 7 days)
|
|
915
|
+
parameters: ''
|
|
916
|
+
expected_result: |
|
|
917
|
+
{"date":"2025-01-10","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0}
|
|
918
|
+
{"date":"2025-01-11","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0}
|
|
919
|
+
|
|
920
|
+
- name: kpis_fixed_time
|
|
921
|
+
description: Test with fixed timestamp for consistent testing
|
|
922
|
+
parameters: fixed_time=2024-01-15T12:00:00
|
|
923
|
+
expected_result: ''
|
|
924
|
+
|
|
925
|
+
- name: kpis_single_day
|
|
926
|
+
description: Test single day with hourly granularity
|
|
927
|
+
parameters: date_from=2024-01-01&date_to=2024-01-01
|
|
928
|
+
expected_result: |
|
|
929
|
+
{"date":"2024-01-01 00:00:00","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0}
|
|
930
|
+
{"date":"2024-01-01 01:00:00","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0}
|
|
931
|
+
|
|
932
|
+
</test_file_format>
|
|
933
|
+
"""
|
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '0.0.1.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '0.0.1.dev46'
|
|
8
|
+
__revision__ = '7b9942e'
|
|
@@ -6,7 +6,6 @@ if sys.platform == "win32":
|
|
|
6
6
|
|
|
7
7
|
import tinybird.tb.modules.auth
|
|
8
8
|
import tinybird.tb.modules.build
|
|
9
|
-
import tinybird.tb.modules.build_client
|
|
10
9
|
import tinybird.tb.modules.cli
|
|
11
10
|
import tinybird.tb.modules.common
|
|
12
11
|
import tinybird.tb.modules.copy
|
|
@@ -18,6 +17,7 @@ import tinybird.tb.modules.fmt
|
|
|
18
17
|
import tinybird.tb.modules.job
|
|
19
18
|
import tinybird.tb.modules.local
|
|
20
19
|
import tinybird.tb.modules.login
|
|
20
|
+
import tinybird.tb.modules.materialization
|
|
21
21
|
import tinybird.tb.modules.mock
|
|
22
22
|
import tinybird.tb.modules.pipe
|
|
23
23
|
import tinybird.tb.modules.tag
|
|
@@ -12,6 +12,7 @@ import requests
|
|
|
12
12
|
from tinybird.client import TinyB
|
|
13
13
|
from tinybird.tb.modules.cli import cli
|
|
14
14
|
from tinybird.tb.modules.common import push_data
|
|
15
|
+
from tinybird.tb.modules.datafile.build import folder_build
|
|
15
16
|
from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
|
|
16
17
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
17
18
|
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
@@ -27,19 +28,19 @@ def build(ctx: click.Context, watch: bool) -> None:
|
|
|
27
28
|
"""
|
|
28
29
|
Validate and build the project server side.
|
|
29
30
|
"""
|
|
30
|
-
|
|
31
31
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
32
|
-
tb_client = asyncio.run(get_tinybird_local_client(str(project.path)))
|
|
33
|
-
click.echo(FeedbackManager.
|
|
32
|
+
tb_client = asyncio.run(get_tinybird_local_client(str(project.path), build=True))
|
|
33
|
+
click.echo(FeedbackManager.highlight_building_project())
|
|
34
34
|
time_start = time.time()
|
|
35
35
|
|
|
36
36
|
def process(file_changed: Optional[str] = None, diff: Optional[str] = None) -> None:
|
|
37
37
|
if file_changed and file_changed.endswith(".ndjson"):
|
|
38
38
|
rebuild_fixture(project, tb_client, file_changed)
|
|
39
39
|
else:
|
|
40
|
-
build_project(project, tb_client)
|
|
40
|
+
build_project(project, tb_client, file_changed)
|
|
41
41
|
try:
|
|
42
42
|
if file_changed:
|
|
43
|
+
asyncio.run(folder_build(project, filenames=[file_changed]))
|
|
43
44
|
build_and_print_resource(tb_client, file_changed, diff)
|
|
44
45
|
except Exception:
|
|
45
46
|
pass
|
|
@@ -61,7 +62,7 @@ def build(ctx: click.Context, watch: bool) -> None:
|
|
|
61
62
|
shell.run()
|
|
62
63
|
|
|
63
64
|
|
|
64
|
-
def build_project(project: Project, tb_client: TinyB) -> None:
|
|
65
|
+
def build_project(project: Project, tb_client: TinyB, file_changed: Optional[str] = None) -> None:
|
|
65
66
|
MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
|
|
66
67
|
DATAFILE_TYPE_TO_CONTENT_TYPE = {
|
|
67
68
|
".datasource": "text/plain",
|
|
@@ -70,6 +71,7 @@ def build_project(project: Project, tb_client: TinyB) -> None:
|
|
|
70
71
|
TINYBIRD_API_URL = tb_client.host + "/v1/build"
|
|
71
72
|
logging.debug(TINYBIRD_API_URL)
|
|
72
73
|
TINYBIRD_API_KEY = tb_client.token
|
|
74
|
+
error = False
|
|
73
75
|
try:
|
|
74
76
|
files = [
|
|
75
77
|
("context://", ("cli-version", "1.0.0", "text/plain")),
|
|
@@ -103,20 +105,23 @@ def build_project(project: Project, tb_client: TinyB) -> None:
|
|
|
103
105
|
if build_result == "success":
|
|
104
106
|
datasources = result.get("datasources", [])
|
|
105
107
|
pipes = result.get("pipes", [])
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
108
|
+
if not file_changed:
|
|
109
|
+
for ds in datasources:
|
|
110
|
+
ds_path_str: Optional[str] = next(
|
|
111
|
+
(p for p in project_files if p.endswith(ds.get("name") + ".datasource")), None
|
|
112
|
+
)
|
|
113
|
+
if ds_path_str:
|
|
114
|
+
ds_path = Path(ds_path_str)
|
|
115
|
+
ds_path_str = ds_path_str.replace(f"{project.folder}/", "")
|
|
116
|
+
click.echo(FeedbackManager.info(message=f"✓ {ds_path_str} created"))
|
|
117
|
+
for pipe in pipes:
|
|
118
|
+
pipe_name = pipe.get("name")
|
|
119
|
+
pipe_path_str: Optional[str] = next(
|
|
120
|
+
(p for p in project_files if p.endswith(pipe_name + ".pipe")), None
|
|
121
|
+
)
|
|
122
|
+
if pipe_path_str:
|
|
123
|
+
pipe_path_str = pipe_path_str.replace(f"{project.folder}/", "")
|
|
124
|
+
click.echo(FeedbackManager.info(message=f"✓ {pipe_path_str} created"))
|
|
120
125
|
|
|
121
126
|
try:
|
|
122
127
|
for filename in project_files:
|
|
@@ -136,13 +141,14 @@ def build_project(project: Project, tb_client: TinyB) -> None:
|
|
|
136
141
|
pass
|
|
137
142
|
|
|
138
143
|
elif build_result == "failed":
|
|
139
|
-
click.echo(FeedbackManager.error(message="Build failed"))
|
|
140
144
|
build_errors = result.get("errors")
|
|
145
|
+
error = True
|
|
141
146
|
for build_error in build_errors:
|
|
142
147
|
filename_bit = f"{build_error.get('filename', '')}"
|
|
143
148
|
error_msg = ((filename_bit + "\n\n") if filename_bit else "") + build_error.get("error")
|
|
144
149
|
click.echo(FeedbackManager.error(message=error_msg))
|
|
145
150
|
else:
|
|
151
|
+
error = True
|
|
146
152
|
click.echo(FeedbackManager.error(message=f"Unknown build result. Error: {result.get('error')}"))
|
|
147
153
|
|
|
148
154
|
except Exception as e:
|
|
@@ -150,6 +156,8 @@ def build_project(project: Project, tb_client: TinyB) -> None:
|
|
|
150
156
|
finally:
|
|
151
157
|
for fd in fds:
|
|
152
158
|
fd.close()
|
|
159
|
+
if error:
|
|
160
|
+
raise click.ClickException(FeedbackManager.error_build_failed())
|
|
153
161
|
|
|
154
162
|
|
|
155
163
|
def append_fixture(
|
|
@@ -6,10 +6,9 @@
|
|
|
6
6
|
import json
|
|
7
7
|
import logging
|
|
8
8
|
import os
|
|
9
|
-
import pprint
|
|
10
9
|
from os import getcwd
|
|
11
10
|
from pathlib import Path
|
|
12
|
-
from typing import Any, Callable, Dict,
|
|
11
|
+
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
|
13
12
|
|
|
14
13
|
import click
|
|
15
14
|
import humanfriendly
|
|
@@ -35,13 +34,7 @@ from tinybird.tb.modules.common import (
|
|
|
35
34
|
)
|
|
36
35
|
from tinybird.tb.modules.config import CLIConfig
|
|
37
36
|
from tinybird.tb.modules.datafile.build import build_graph
|
|
38
|
-
from tinybird.tb.modules.datafile.common import Datafile, DatafileSyntaxError
|
|
39
37
|
from tinybird.tb.modules.datafile.diff import diff_command
|
|
40
|
-
from tinybird.tb.modules.datafile.exceptions import (
|
|
41
|
-
ParseException,
|
|
42
|
-
)
|
|
43
|
-
from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
|
|
44
|
-
from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
|
|
45
38
|
from tinybird.tb.modules.datafile.pull import folder_pull
|
|
46
39
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
47
40
|
from tinybird.tb.modules.llm import LLM
|
|
@@ -66,12 +59,20 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
|
|
|
66
59
|
@click.option("--host", help="Use custom host, defaults to TB_HOST envvar, then to https://api.tinybird.co")
|
|
67
60
|
@click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
|
|
68
61
|
@click.option("--prod/--local", is_flag=True, default=False, help="Run against production or local")
|
|
62
|
+
@click.option("--build", is_flag=True, default=False, help="Run against build mode")
|
|
69
63
|
@click.option("--folder", type=str, help="Folder where files will be placed")
|
|
70
64
|
@click.version_option(version=VERSION)
|
|
71
65
|
@click.pass_context
|
|
72
66
|
@coro
|
|
73
67
|
async def cli(
|
|
74
|
-
ctx: Context,
|
|
68
|
+
ctx: Context,
|
|
69
|
+
debug: bool,
|
|
70
|
+
token: str,
|
|
71
|
+
host: str,
|
|
72
|
+
show_tokens: bool,
|
|
73
|
+
prod: bool,
|
|
74
|
+
build: bool,
|
|
75
|
+
folder: Optional[str],
|
|
75
76
|
) -> None:
|
|
76
77
|
"""
|
|
77
78
|
Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
|
|
@@ -123,64 +124,19 @@ async def cli(
|
|
|
123
124
|
|
|
124
125
|
logging.debug("debug enabled")
|
|
125
126
|
|
|
126
|
-
skip_client = ctx.invoked_subcommand in ["auth", "check", "login", "
|
|
127
|
-
client = await create_ctx_client(config, prod, skip_client, project)
|
|
127
|
+
skip_client = ctx.invoked_subcommand in ["auth", "check", "login", "local", "build"]
|
|
128
|
+
client = await create_ctx_client(config, prod, build, skip_client, project)
|
|
128
129
|
|
|
129
130
|
if client:
|
|
131
|
+
if not build:
|
|
132
|
+
target = config.get("name", "production") if prod else "Tinybird local"
|
|
133
|
+
click.echo(FeedbackManager.gray(message=f"Running against {target}\n"))
|
|
130
134
|
ctx.ensure_object(dict)["client"] = client
|
|
131
135
|
|
|
132
136
|
ctx.ensure_object(dict)["project"] = project
|
|
133
137
|
|
|
134
138
|
|
|
135
|
-
@cli.command(
|
|
136
|
-
@click.argument("filenames", type=click.Path(exists=True), nargs=-1, default=None)
|
|
137
|
-
@click.option("--debug", is_flag=True, default=False, help="Print internal representation")
|
|
138
|
-
@click.pass_context
|
|
139
|
-
def check(ctx: Context, filenames: List[str], debug: bool) -> None:
|
|
140
|
-
"""Check file syntax."""
|
|
141
|
-
|
|
142
|
-
if not filenames:
|
|
143
|
-
project: Project = ctx.ensure_object(dict)["project"]
|
|
144
|
-
filenames = project.get_project_files()
|
|
145
|
-
|
|
146
|
-
def process(filenames: Iterable):
|
|
147
|
-
parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
|
|
148
|
-
incl_suffix = ".incl"
|
|
149
|
-
try:
|
|
150
|
-
for filename in filenames:
|
|
151
|
-
if os.path.isdir(filename):
|
|
152
|
-
process(filenames=filename)
|
|
153
|
-
|
|
154
|
-
click.echo(FeedbackManager.info_processing_file(filename=filename))
|
|
155
|
-
|
|
156
|
-
file_suffix = Path(filename).suffix
|
|
157
|
-
if file_suffix == incl_suffix:
|
|
158
|
-
click.echo(FeedbackManager.info_ignoring_incl_file(filename=filename))
|
|
159
|
-
continue
|
|
160
|
-
|
|
161
|
-
doc: Datafile
|
|
162
|
-
parser = parser_matrix.get(file_suffix)
|
|
163
|
-
if not parser:
|
|
164
|
-
raise ParseException(FeedbackManager.error_unsupported_datafile(extension=file_suffix))
|
|
165
|
-
|
|
166
|
-
doc = parser(filename)
|
|
167
|
-
|
|
168
|
-
click.echo(FeedbackManager.success_processing_file(filename=filename))
|
|
169
|
-
if debug:
|
|
170
|
-
pp = pprint.PrettyPrinter()
|
|
171
|
-
for x in doc.nodes:
|
|
172
|
-
pp.pprint(x)
|
|
173
|
-
|
|
174
|
-
except DatafileSyntaxError as e:
|
|
175
|
-
# TODO(eclbg): add the filename to the error message
|
|
176
|
-
raise CLIException(str(e))
|
|
177
|
-
except ParseException as e:
|
|
178
|
-
raise CLIException(FeedbackManager.error_exception(error=e))
|
|
179
|
-
|
|
180
|
-
process(filenames=filenames)
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
@cli.command(hidden=True)
|
|
139
|
+
@cli.command()
|
|
184
140
|
@click.option(
|
|
185
141
|
"--folder", default=None, type=click.Path(exists=True, file_okay=False), help="Folder where files will be placed"
|
|
186
142
|
)
|
|
@@ -489,11 +445,11 @@ def __unpatch_click_output():
|
|
|
489
445
|
click.secho = __old_click_secho
|
|
490
446
|
|
|
491
447
|
|
|
492
|
-
async def create_ctx_client(config: Dict[str, Any], prod: bool, skip_client: bool, project: Project):
|
|
448
|
+
async def create_ctx_client(config: Dict[str, Any], prod: bool, build: bool, skip_client: bool, project: Project):
|
|
493
449
|
if skip_client:
|
|
494
450
|
return None
|
|
495
451
|
|
|
496
452
|
if prod:
|
|
497
453
|
return _get_tb_client(config.get("token", None), config["host"])
|
|
498
454
|
|
|
499
|
-
return await get_tinybird_local_client(str(project.path))
|
|
455
|
+
return await get_tinybird_local_client(str(project.path), build=build)
|
|
@@ -163,7 +163,7 @@ def generate_datafile(
|
|
|
163
163
|
force: Optional[bool] = False,
|
|
164
164
|
_format: Optional[str] = "csv",
|
|
165
165
|
folder: Optional[str] = None,
|
|
166
|
-
):
|
|
166
|
+
) -> Path:
|
|
167
167
|
p = Path(filename)
|
|
168
168
|
base = Path("datasources")
|
|
169
169
|
if folder:
|
|
@@ -190,6 +190,7 @@ def generate_datafile(
|
|
|
190
190
|
fixture_file.write(data[: data.rfind(newline)])
|
|
191
191
|
else:
|
|
192
192
|
click.echo(FeedbackManager.error_file_already_exists(file=f))
|
|
193
|
+
return f
|
|
193
194
|
|
|
194
195
|
|
|
195
196
|
async def get_current_workspace(config: CLIConfig) -> Optional[Dict[str, Any]]:
|
|
@@ -1361,7 +1362,7 @@ async def try_update_config_with_remote(
|
|
|
1361
1362
|
def ask_for_admin_token_interactively(ui_host: str, default_token: Optional[str]) -> str:
|
|
1362
1363
|
return (
|
|
1363
1364
|
click.prompt(
|
|
1364
|
-
f
|
|
1365
|
+
f'\nCopy the "admin your@email" token from {ui_host}/tokens and paste it here {"OR press enter to use the token from .tinyb file" if default_token else ""}',
|
|
1365
1366
|
hide_input=True,
|
|
1366
1367
|
show_default=False,
|
|
1367
1368
|
default=default_token,
|