tinybird 4.6.5.dev0__py3-none-any.whl → 4.6.6.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tinybird/ch_utils/engine.py +0 -25
- tinybird/datafile/common.py +0 -15
- tinybird/datatypes.py +0 -4
- tinybird/feedback_manager.py +5 -0
- tinybird/prompts.py +2 -272
- tinybird/service_datasources.py +1 -18
- tinybird/sql.py +0 -24
- tinybird/sql_template.py +0 -29
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/client.py +30 -2
- tinybird/tb/config.py +0 -6
- tinybird/tb/modules/build.py +1 -59
- tinybird/tb/modules/cli.py +0 -4
- tinybird/tb/modules/common.py +0 -74
- tinybird/tb/modules/config.py +0 -8
- tinybird/tb/modules/connection_s3.py +4 -8
- tinybird/tb/modules/datafile/build.py +5 -249
- tinybird/tb/modules/datafile/diff.py +2 -2
- tinybird/tb/modules/datafile/fixture.py +0 -19
- tinybird/tb/modules/datafile/playground.py +4 -4
- tinybird/tb/modules/datafile/pull.py +9 -1
- tinybird/tb/modules/llm_utils.py +0 -8
- tinybird/tb/modules/local_common.py +0 -1
- tinybird/tb/modules/local_logs.py +0 -4
- tinybird/tb/modules/login_common.py +3 -2
- tinybird/tb/modules/test_common.py +0 -5
- tinybird/tb/modules/tinyunit/tinyunit.py +0 -9
- tinybird/tb_cli_modules/common.py +0 -16
- tinybird/tb_cli_modules/config.py +0 -8
- {tinybird-4.6.5.dev0.dist-info → tinybird-4.6.6.dev0.dist-info}/METADATA +6 -1
- {tinybird-4.6.5.dev0.dist-info → tinybird-4.6.6.dev0.dist-info}/RECORD +34 -34
- {tinybird-4.6.5.dev0.dist-info → tinybird-4.6.6.dev0.dist-info}/WHEEL +0 -0
- {tinybird-4.6.5.dev0.dist-info → tinybird-4.6.6.dev0.dist-info}/entry_points.txt +0 -0
- {tinybird-4.6.5.dev0.dist-info → tinybird-4.6.6.dev0.dist-info}/top_level.txt +0 -0
tinybird/ch_utils/engine.py
CHANGED
|
@@ -354,31 +354,6 @@ def sorting_key_is_valid(columns: List[Dict[str, Any]], value: Optional[str]) ->
|
|
|
354
354
|
return value
|
|
355
355
|
|
|
356
356
|
|
|
357
|
-
def case_insensitive_check(valid_values: List[str]) -> Callable[[List[Dict[str, Any]], str], Optional[str]]:
|
|
358
|
-
"""
|
|
359
|
-
>>> valid_values = ['ANY', 'ALL']
|
|
360
|
-
>>> checker = case_insensitive_check(valid_values)
|
|
361
|
-
>>> checker([],'ALL')
|
|
362
|
-
|
|
363
|
-
>>> valid_values = ['ANY', 'ALL']
|
|
364
|
-
>>> checker = case_insensitive_check(valid_values)
|
|
365
|
-
>>> checker([],'any')
|
|
366
|
-
|
|
367
|
-
>>> valid_values = ['ANY', 'ALL']
|
|
368
|
-
>>> checker = case_insensitive_check(valid_values)
|
|
369
|
-
>>> checker([],'foo')
|
|
370
|
-
Traceback (most recent call last):
|
|
371
|
-
...
|
|
372
|
-
ValueError: valid values are ANY, ALL
|
|
373
|
-
"""
|
|
374
|
-
|
|
375
|
-
def checker(columns: List[Dict[str, Any]], value: str):
|
|
376
|
-
if value.upper() not in valid_values:
|
|
377
|
-
raise ValueError(f"valid values are {', '.join(valid_values)}")
|
|
378
|
-
|
|
379
|
-
return checker
|
|
380
|
-
|
|
381
|
-
|
|
382
357
|
# [PARTITION BY expr]
|
|
383
358
|
# [ORDER BY expr]
|
|
384
359
|
# [PRIMARY KEY expr]
|
tinybird/datafile/common.py
CHANGED
|
@@ -96,10 +96,6 @@ class IndexesSyntaxError(DatafileSyntaxError):
|
|
|
96
96
|
super().__init__(message=message, lineno=lineno, pos=pos, hint=hint)
|
|
97
97
|
|
|
98
98
|
|
|
99
|
-
class MalformedColumnError(Exception):
|
|
100
|
-
pass
|
|
101
|
-
|
|
102
|
-
|
|
103
99
|
class PipeTypes:
|
|
104
100
|
MATERIALIZED = "materialized"
|
|
105
101
|
ENDPOINT = "endpoint"
|
|
@@ -2369,17 +2365,6 @@ def get_project_filenames(folder: str, with_vendor=False) -> List[str]:
|
|
|
2369
2365
|
return filenames
|
|
2370
2366
|
|
|
2371
2367
|
|
|
2372
|
-
def get_project_fixtures(folder: str) -> List[str]:
|
|
2373
|
-
folders: List[str] = [
|
|
2374
|
-
f"{folder}/fixtures/*.ndjson",
|
|
2375
|
-
f"{folder}/fixtures/*.csv",
|
|
2376
|
-
]
|
|
2377
|
-
filenames: List[str] = []
|
|
2378
|
-
for x in folders:
|
|
2379
|
-
filenames += glob.glob(x)
|
|
2380
|
-
return filenames
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
2368
|
def has_internal_datafiles(folder: str) -> bool:
|
|
2384
2369
|
folder = folder or "."
|
|
2385
2370
|
filenames = get_project_filenames(folder)
|
tinybird/datatypes.py
CHANGED
|
@@ -108,10 +108,6 @@ def is_type_date(type_to_check: str) -> bool:
|
|
|
108
108
|
return date_type_pattern.match(type_to_check) is not None
|
|
109
109
|
|
|
110
110
|
|
|
111
|
-
def string_test(x: str) -> bool:
|
|
112
|
-
return True
|
|
113
|
-
|
|
114
|
-
|
|
115
111
|
def date_test(x: str) -> bool:
|
|
116
112
|
return date_pattern.match(x) is not None
|
|
117
113
|
|
tinybird/feedback_manager.py
CHANGED
|
@@ -879,6 +879,11 @@ class FeedbackManager:
|
|
|
879
879
|
success_delete_rows_datasource = success_message(
|
|
880
880
|
"** Data Source '{datasource}' rows deleted matching condition \"{delete_condition}\""
|
|
881
881
|
)
|
|
882
|
+
success_lightweight_delete_rows_datasource = success_message(
|
|
883
|
+
"** Data Source '{datasource}' rows deleted matching condition \"{delete_condition}\""
|
|
884
|
+
"\n Rows affected: {rows_affected}"
|
|
885
|
+
"\n Partitions scanned: {partitions_scanned} (done: {partitions_done}, in progress: {partitions_in_progress})"
|
|
886
|
+
)
|
|
882
887
|
success_dry_run_delete_rows_datasource = success_message(
|
|
883
888
|
"** [DRY RUN] Data Source '{datasource}' rows '{rows}' matching condition \"{delete_condition}\" to be deleted"
|
|
884
889
|
)
|
tinybird/prompts.py
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
from datetime import date
|
|
2
|
-
from typing import Optional
|
|
3
2
|
|
|
4
3
|
general_functions = [
|
|
5
4
|
"BLAKE3",
|
|
@@ -400,64 +399,6 @@ Follow the instructions and generate the following response with no additional t
|
|
|
400
399
|
"""
|
|
401
400
|
|
|
402
401
|
|
|
403
|
-
def create_prompt(existing_resources: str, feedback: str = "", history: str = "") -> str:
|
|
404
|
-
feedback_history = ""
|
|
405
|
-
if feedback and history:
|
|
406
|
-
feedback_history = f"""In case the <feedback> and <history> tags are present and not empty,
|
|
407
|
-
it means there was a previous attempt to generate the resources and the user provided feedback and history about previous responses.
|
|
408
|
-
Use the following feedback and history to regenerate the response:
|
|
409
|
-
Feedback to improve the response:
|
|
410
|
-
{feedback}
|
|
411
|
-
History of previous results:
|
|
412
|
-
{history}"""
|
|
413
|
-
|
|
414
|
-
return """
|
|
415
|
-
You are a Tinybird expert. You will be given a prompt to generate new or update existing Tinybird resources: datasources and/or pipes.
|
|
416
|
-
<existing_resources>{existing_resources}</existing_resources>
|
|
417
|
-
{datasource_instructions}
|
|
418
|
-
{pipe_instructions}
|
|
419
|
-
{sql_instructions}
|
|
420
|
-
{datasource_example}
|
|
421
|
-
{pipe_example}
|
|
422
|
-
{copy_pipe_instructions}
|
|
423
|
-
{materialized_pipe_instructions}
|
|
424
|
-
{sink_pipe_instructions}
|
|
425
|
-
{connection_instructions}
|
|
426
|
-
{kafka_connection_example}
|
|
427
|
-
{gcs_connection_example}
|
|
428
|
-
{gcs_hmac_connection_example}
|
|
429
|
-
{s3_connection_example}
|
|
430
|
-
|
|
431
|
-
{feedback_history}
|
|
432
|
-
|
|
433
|
-
Use the following format to generate the response and do not wrap it in any other text, including the <response> tag.
|
|
434
|
-
<response>
|
|
435
|
-
<resource>
|
|
436
|
-
<type>[datasource or pipe or connection]</type>
|
|
437
|
-
<name>[resource name here]</name>
|
|
438
|
-
<content>[resource content here]</content>
|
|
439
|
-
</resource>
|
|
440
|
-
</response>
|
|
441
|
-
|
|
442
|
-
""".format(
|
|
443
|
-
existing_resources=existing_resources,
|
|
444
|
-
datasource_instructions=datasource_instructions,
|
|
445
|
-
pipe_instructions=pipe_instructions,
|
|
446
|
-
sql_instructions=sql_instructions,
|
|
447
|
-
datasource_example=datasource_example,
|
|
448
|
-
pipe_example=pipe_example,
|
|
449
|
-
copy_pipe_instructions=copy_pipe_instructions,
|
|
450
|
-
materialized_pipe_instructions=materialized_pipe_instructions,
|
|
451
|
-
sink_pipe_instructions=sink_pipe_instructions,
|
|
452
|
-
connection_instructions=connection_instructions,
|
|
453
|
-
kafka_connection_example=kafka_connection_example,
|
|
454
|
-
gcs_connection_example=gcs_connection_example,
|
|
455
|
-
gcs_hmac_connection_example=gcs_hmac_connection_example,
|
|
456
|
-
s3_connection_example=s3_connection_example,
|
|
457
|
-
feedback_history=feedback_history,
|
|
458
|
-
)
|
|
459
|
-
|
|
460
|
-
|
|
461
402
|
def mock_prompt(rows: int, feedback: str = "") -> str:
|
|
462
403
|
today = date.today().isoformat()
|
|
463
404
|
|
|
@@ -824,7 +765,7 @@ Follow these instructions when creating or updating any type of .pipe file:
|
|
|
824
765
|
</pipe_file_instructions>
|
|
825
766
|
"""
|
|
826
767
|
|
|
827
|
-
sql_instructions = """
|
|
768
|
+
sql_instructions = f"""
|
|
828
769
|
<sql_instructions>
|
|
829
770
|
- The SQL query must be a valid ClickHouse SQL query that mixes ClickHouse syntax and Tinybird templating syntax (Tornado templating language under the hood).
|
|
830
771
|
- SQL queries with parameters must start with "%" character and a newline on top of every query to be able to use the parameters. Examples:
|
|
@@ -882,186 +823,7 @@ sql_instructions = """
|
|
|
882
823
|
- Use the following syntax in the SQL section for the iceberg table function: iceberg('s3://bucket/path/to/table', {{{{tb_secret('aws_access_key_id')}}}}, {{{{tb_secret('aws_secret_access_key')}}}})
|
|
883
824
|
- Use the following syntax in the SQL section for the postgres table function: postgresql('host:port', 'database', 'table', {{{{tb_secret('db_username')}}}}, {{{{tb_secret('db_password')}}}}), 'schema')
|
|
884
825
|
</sql_instructions>
|
|
885
|
-
"""
|
|
886
|
-
general_functions=general_functions,
|
|
887
|
-
general_functions_insensitive=general_functions_insensitive,
|
|
888
|
-
aggregate_functions=aggregate_functions,
|
|
889
|
-
)
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
def rules_prompt(source: Optional[str] = None) -> str:
|
|
893
|
-
base_command = source or "tb"
|
|
894
|
-
return """
|
|
895
|
-
You are an expert in SQL and Tinybird. Follow these instructions when working with .datasource and .pipe files:
|
|
896
|
-
|
|
897
|
-
<command_calling>
|
|
898
|
-
You have commands at your disposal to develop a tinybird project:
|
|
899
|
-
- {base_command} build: to build the project locally and check it works.
|
|
900
|
-
- {base_command} deployment create --wait --auto: to create a deployment and promote it automatically
|
|
901
|
-
- {base_command} test run: to run existing tests
|
|
902
|
-
- {base_command} endpoint url <pipe_name>: to get the url of an endpoint, token included.
|
|
903
|
-
- {base_command} endpoint data <pipe_name>: to get the data of an endpoint. You can pass parameters to the endpoint like this: {base_command} endpoint data <pipe_name> --param1 value1 --param2 value2
|
|
904
|
-
- {base_command} token ls: to list all the tokens
|
|
905
|
-
There are other commands that you can use, but these are the most common ones. Run `{base_command} -h` to see all the commands if needed.
|
|
906
|
-
When you need to work with resources or data in cloud, add always the --cloud flag before the command. Example: {base_command} --cloud datasource ls
|
|
907
|
-
</command_calling>
|
|
908
|
-
<development_instructions>
|
|
909
|
-
- When asking to create a tinybird data project, if the needed folders are not already created, use the following structure:
|
|
910
|
-
├── connections
|
|
911
|
-
├── copies
|
|
912
|
-
├── sinks
|
|
913
|
-
├── datasources
|
|
914
|
-
├── endpoints
|
|
915
|
-
├── fixtures
|
|
916
|
-
├── materializations
|
|
917
|
-
├── pipes
|
|
918
|
-
└── tests
|
|
919
|
-
- The local development server will be available at http://localhost:7181. Even if some response uses another base url, use always http://localhost:7181.
|
|
920
|
-
- After every change in your .datasource, .pipe or .ndjson files, run `{base_command} build` to build the project locally.
|
|
921
|
-
- When you need to ingest data locally in a datasource, create a .ndjson file with the same name of the datasource and the data you want and run `{base_command} build` so the data is ingested.
|
|
922
|
-
- The format of the generated api endpoint urls is: http://localhost:7181/v0/pipe/<pipe_name>.json?token=<token>
|
|
923
|
-
- Before running the tests, remember to have the project built with `{base_command} build` with the latest changes.
|
|
924
|
-
</development_instructions>
|
|
925
|
-
When asking for ingesting data, adding data or appending data do the following depending on the environment you want to work with:
|
|
926
|
-
<ingest_data_instructions>
|
|
927
|
-
- When building locally, create a .ndjson file with the data you want to ingest and do `{base_command} build` to ingest the data in the build env.
|
|
928
|
-
- We call `cloud` the production environment.
|
|
929
|
-
- When appending data in cloud, use `{base_command} --cloud datasource append <datasource_name> <file_name>`
|
|
930
|
-
- When you have a response that says “there are rows in quarantine”, do `{base_command} [--cloud] datasource data <datasource_name>_quarantine` to understand what is the problem.
|
|
931
|
-
</ingest_data_instructions>
|
|
932
|
-
<datasource_file_instructions>
|
|
933
|
-
Follow these instructions when creating or updating .datasource files:
|
|
934
|
-
{datasource_instructions}
|
|
935
|
-
</datasource_file_instructions>
|
|
936
|
-
|
|
937
|
-
<pipe_file_instructions>
|
|
938
|
-
Follow these instructions when creating or updating .pipe files:
|
|
939
|
-
{pipe_instructions}
|
|
940
|
-
{sql_instructions}
|
|
941
|
-
{datasource_example}
|
|
942
|
-
{pipe_example}
|
|
943
|
-
{copy_pipe_instructions}
|
|
944
|
-
{materialized_pipe_instructions}
|
|
945
|
-
{sink_pipe_instructions}
|
|
946
|
-
{connection_instructions}
|
|
947
|
-
{kafka_connection_example}
|
|
948
|
-
{gcs_connection_example}
|
|
949
|
-
{gcs_hmac_connection_example}
|
|
950
|
-
{s3_connection_example}
|
|
951
|
-
</pipe_file_instructions>
|
|
952
|
-
<test_file_instructions>
|
|
953
|
-
Follow these instructions when creating or updating .yaml files for tests:
|
|
954
|
-
{test_instructions}
|
|
955
|
-
</test_file_instructions>
|
|
956
|
-
<deployment_instruction>
|
|
957
|
-
Follow these instructions when evolving a datasource schema:
|
|
958
|
-
{deployment_instructions}
|
|
959
|
-
</deployment_instruction>
|
|
960
|
-
""".format(
|
|
961
|
-
base_command=base_command,
|
|
962
|
-
datasource_instructions=datasource_instructions,
|
|
963
|
-
pipe_instructions=pipe_instructions,
|
|
964
|
-
sql_instructions=sql_instructions,
|
|
965
|
-
datasource_example=datasource_example,
|
|
966
|
-
pipe_example=pipe_example,
|
|
967
|
-
copy_pipe_instructions=copy_pipe_instructions,
|
|
968
|
-
materialized_pipe_instructions=materialized_pipe_instructions,
|
|
969
|
-
sink_pipe_instructions=sink_pipe_instructions,
|
|
970
|
-
test_instructions=test_instructions,
|
|
971
|
-
deployment_instructions=deployment_instructions,
|
|
972
|
-
connection_instructions=connection_instructions,
|
|
973
|
-
kafka_connection_example=kafka_connection_example,
|
|
974
|
-
gcs_connection_example=gcs_connection_example,
|
|
975
|
-
gcs_hmac_connection_example=gcs_hmac_connection_example,
|
|
976
|
-
s3_connection_example=s3_connection_example,
|
|
977
|
-
)
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
def claude_rules_prompt(source: Optional[str] = None) -> str:
|
|
981
|
-
base_command = source or "tb"
|
|
982
|
-
return """
|
|
983
|
-
# Tinybird CLI rules
|
|
984
|
-
|
|
985
|
-
## Commands
|
|
986
|
-
You have commands at your disposal to develop a tinybird project:
|
|
987
|
-
- {base_command} build: to build the project locally and check it works.
|
|
988
|
-
- {base_command} deployment create --wait --auto: to create a deployment and promote it automatically
|
|
989
|
-
- {base_command} test run: to run existing tests
|
|
990
|
-
- {base_command} endpoint url <pipe_name>: to get the url of an endpoint, token included.
|
|
991
|
-
- {base_command} endpoint data <pipe_name>: to get the data of an endpoint. You can pass parameters to the endpoint like this: {base_command} endpoint data <pipe_name> --param1 value1 --param2 value2
|
|
992
|
-
- {base_command} token ls: to list all the tokens
|
|
993
|
-
There are other commands that you can use, but these are the most common ones. Run `{base_command} -h` to see all the commands if needed.
|
|
994
|
-
When you need to work with resources or data in cloud, add always the --cloud flag before the command. Example: {base_command} --cloud datasource ls
|
|
995
|
-
|
|
996
|
-
## Development instructions
|
|
997
|
-
- When asking to create a tinybird data project, if the needed folders are not already created, use the following structure:
|
|
998
|
-
├── connections
|
|
999
|
-
├── copies
|
|
1000
|
-
├── sinks
|
|
1001
|
-
├── datasources
|
|
1002
|
-
├── endpoints
|
|
1003
|
-
├── fixtures
|
|
1004
|
-
├── materializations
|
|
1005
|
-
├── pipes
|
|
1006
|
-
└── tests
|
|
1007
|
-
- The local development server will be available at http://localhost:7181. Even if some response uses another base url, use always http://localhost:7181.
|
|
1008
|
-
- After every change in your .datasource, .pipe or .ndjson files, run `{base_command} build` to build the project locally.
|
|
1009
|
-
- When you need to ingest data locally in a datasource, create a .ndjson file with the same name of the datasource and the data you want and run `{base_command} build` so the data is ingested.
|
|
1010
|
-
- The format of the generated api endpoint urls is: http://localhost:7181/v0/pipe/<pipe_name>.json?token=<token>
|
|
1011
|
-
- Before running the tests, remember to have the project built with `{base_command} build` with the latest changes.
|
|
1012
|
-
</development_instructions>
|
|
1013
|
-
When asking for ingesting data, adding data or appending data do the following depending on the environment you want to work with:
|
|
1014
|
-
|
|
1015
|
-
## Ingestion instructions
|
|
1016
|
-
- When building locally, create a .ndjson file with the data you want to ingest and do `{base_command} build` to ingest the data in the build env.
|
|
1017
|
-
- We call `cloud` the production environment.
|
|
1018
|
-
- When appending data in cloud, use `{base_command} --cloud datasource append <datasource_name> <file_name>`
|
|
1019
|
-
- When you have a response that says “there are rows in quarantine”, do `{base_command} [--cloud] datasource data <datasource_name>_quarantine` to understand what is the problem.
|
|
1020
|
-
|
|
1021
|
-
## .datasource file instructions
|
|
1022
|
-
Follow these instructions when creating or updating .datasource files:
|
|
1023
|
-
{datasource_instructions}
|
|
1024
|
-
|
|
1025
|
-
## .pipe file instructions
|
|
1026
|
-
Follow these instructions when creating or updating .pipe files:
|
|
1027
|
-
{pipe_instructions}
|
|
1028
|
-
{sql_instructions}
|
|
1029
|
-
{datasource_example}
|
|
1030
|
-
{pipe_example}
|
|
1031
|
-
{copy_pipe_instructions}
|
|
1032
|
-
{materialized_pipe_instructions}
|
|
1033
|
-
{sink_pipe_instructions}
|
|
1034
|
-
{connection_instructions}
|
|
1035
|
-
{kafka_connection_example}
|
|
1036
|
-
{gcs_connection_example}
|
|
1037
|
-
{gcs_hmac_connection_example}
|
|
1038
|
-
{s3_connection_example}
|
|
1039
|
-
|
|
1040
|
-
## .test file instructions
|
|
1041
|
-
Follow these instructions when creating or updating .yaml files for tests:
|
|
1042
|
-
{test_instructions}
|
|
1043
|
-
|
|
1044
|
-
## Deployment instructions
|
|
1045
|
-
Follow these instructions when evolving a datasource schema:
|
|
1046
|
-
{deployment_instructions}
|
|
1047
|
-
""".format(
|
|
1048
|
-
base_command=base_command,
|
|
1049
|
-
datasource_instructions=datasource_instructions,
|
|
1050
|
-
pipe_instructions=pipe_instructions,
|
|
1051
|
-
sql_instructions=sql_instructions,
|
|
1052
|
-
datasource_example=datasource_example,
|
|
1053
|
-
pipe_example=pipe_example,
|
|
1054
|
-
copy_pipe_instructions=copy_pipe_instructions,
|
|
1055
|
-
materialized_pipe_instructions=materialized_pipe_instructions,
|
|
1056
|
-
sink_pipe_instructions=sink_pipe_instructions,
|
|
1057
|
-
test_instructions=test_instructions,
|
|
1058
|
-
deployment_instructions=deployment_instructions,
|
|
1059
|
-
connection_instructions=connection_instructions,
|
|
1060
|
-
kafka_connection_example=kafka_connection_example,
|
|
1061
|
-
gcs_connection_example=gcs_connection_example,
|
|
1062
|
-
gcs_hmac_connection_example=gcs_hmac_connection_example,
|
|
1063
|
-
s3_connection_example=s3_connection_example,
|
|
1064
|
-
)
|
|
826
|
+
"""
|
|
1065
827
|
|
|
1066
828
|
|
|
1067
829
|
test_instructions = """
|
|
@@ -1132,38 +894,6 @@ FORWARD_QUERY >
|
|
|
1132
894
|
"""
|
|
1133
895
|
|
|
1134
896
|
|
|
1135
|
-
def readme_prompt(readme: str, host: str, token: str, resources_xml: str) -> str:
|
|
1136
|
-
return f"""
|
|
1137
|
-
You are an expert in SQL and Tinybird. Follow these instructions to generate a new README.md file for a tinybird project:
|
|
1138
|
-
Current README.md file:
|
|
1139
|
-
<current_resources_xml>
|
|
1140
|
-
{resources_xml}
|
|
1141
|
-
</current_resources_xml>
|
|
1142
|
-
<readme>{readme}</readme>
|
|
1143
|
-
<readme_instructions>
|
|
1144
|
-
- If it is not present in the current readme, generate a new ## Tinybird section with the following content:
|
|
1145
|
-
- ### Overview section:
|
|
1146
|
-
- Explaining the purpose of the project.
|
|
1147
|
-
- ### Data sources section:
|
|
1148
|
-
- Explaining the purpose of each datasource.
|
|
1149
|
-
- Add a snippet of how to ingest data into each datasource like the following (where the payload example matches the datasource schema respecting non-nullable types):
|
|
1150
|
-
curl -X POST "{host}/v0/events?name=events" \
|
|
1151
|
-
-H "Authorization: Bearer {token}" \
|
|
1152
|
-
-d '{{"date":"2025-01-31","id":"123","user_id":"abc"}}'
|
|
1153
|
-
- ### Endpoints section:
|
|
1154
|
-
- Explaining the purpose of each endpoint.
|
|
1155
|
-
- Add a snippet of how to use each endpoint like the following:
|
|
1156
|
-
curl -X GET "{host}/v0/pipes/pipe_name.json?token={token}"
|
|
1157
|
-
- DateTime parameters must be formatted as YYYY-MM-DD HH:MM:SS, or else will fail.
|
|
1158
|
-
- Do not include any other extra info related to Tinybird just maintain the existing one in the <readme> tag.
|
|
1159
|
-
- Make sure the API host is correct, {host}.
|
|
1160
|
-
- It is mandatory to return a <resource> tag with type "readme" in the response.
|
|
1161
|
-
- The response must follow the following format:
|
|
1162
|
-
<readme>[readme content here]</readme>
|
|
1163
|
-
</readme_instructions>
|
|
1164
|
-
"""
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
897
|
def quarantine_prompt(datasource_definition: str) -> str:
|
|
1168
898
|
return f"""
|
|
1169
899
|
- You are an expert in Tinybird.
|
tinybird/service_datasources.py
CHANGED
|
@@ -6,7 +6,7 @@ for both Tinybird and Organization scopes.
|
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
from functools import lru_cache
|
|
9
|
-
from typing import Any, Dict, List
|
|
9
|
+
from typing import Any, Dict, List
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
|
|
@@ -1126,20 +1126,3 @@ def get_service_datasources() -> List[Dict[str, Any]]:
|
|
|
1126
1126
|
List[Dict[str, Any]]: A combined list of all service datasource definitions.
|
|
1127
1127
|
"""
|
|
1128
1128
|
return get_tinybird_service_datasources() + get_organization_service_datasources()
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
def get_service_datasource_by_name(name: str) -> Optional[Dict[str, Any]]:
|
|
1132
|
-
"""
|
|
1133
|
-
Get a specific service datasource by name. Name should include the type (e.g. tinybird.datasources_ops_log)
|
|
1134
|
-
|
|
1135
|
-
Args:
|
|
1136
|
-
name: The name of the service datasource to retrieve.
|
|
1137
|
-
|
|
1138
|
-
Returns:
|
|
1139
|
-
Optional[Dict[str, Any]]: The service datasource definition or None if not found.
|
|
1140
|
-
"""
|
|
1141
|
-
service_datasources = get_service_datasources()
|
|
1142
|
-
for ds in service_datasources:
|
|
1143
|
-
if ds["name"] == name:
|
|
1144
|
-
return ds
|
|
1145
|
-
return None
|
tinybird/sql.py
CHANGED
|
@@ -911,30 +911,6 @@ def engine_replicated_to_local(engine: str) -> str:
|
|
|
911
911
|
return _RE_REPLICATED_MT.sub(_replace, engine.strip())
|
|
912
912
|
|
|
913
913
|
|
|
914
|
-
def engine_patch_replicated_engine(engine: str, engine_full: Optional[str], new_table_name: str) -> Optional[str]:
|
|
915
|
-
"""
|
|
916
|
-
>>> engine_patch_replicated_engine("ReplicatedMergeTree", "ReplicatedMergeTree('/clickhouse/tables/1-1/table_name', 'replica') PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192", 'table_name_staging')
|
|
917
|
-
"ReplicatedMergeTree('/clickhouse/tables/1-1/table_name_staging', 'replica') PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192"
|
|
918
|
-
>>> engine_patch_replicated_engine("ReplicatedMergeTree", "ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/sales_product_rank_rt_replicated_2', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY (purchase_location, sku_rank_lc, date)", 'sales_product_rank_rt_replicated_2_staging')
|
|
919
|
-
"ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/sales_product_rank_rt_replicated_2_staging', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY (purchase_location, sku_rank_lc, date)"
|
|
920
|
-
>>> engine_patch_replicated_engine("ReplicatedMergeTree", None, 't_000') is None
|
|
921
|
-
True
|
|
922
|
-
>>> engine_patch_replicated_engine("Log", "Log()", 't_000')
|
|
923
|
-
'Log()'
|
|
924
|
-
>>> engine_patch_replicated_engine("MergeTree", "MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024", 't_000')
|
|
925
|
-
'MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024'
|
|
926
|
-
"""
|
|
927
|
-
if not engine_full:
|
|
928
|
-
return None
|
|
929
|
-
if engine.lower().startswith("Replicated".lower()):
|
|
930
|
-
parts = _RE_REPLICATED_ENGINE_SPLIT.split(engine_full)
|
|
931
|
-
paths = parts[2].split("/")
|
|
932
|
-
paths[-1] = new_table_name
|
|
933
|
-
zoo_path = "/".join(paths)
|
|
934
|
-
return "".join([*parts[:2], zoo_path, *parts[3:]])
|
|
935
|
-
return engine_full
|
|
936
|
-
|
|
937
|
-
|
|
938
914
|
if __name__ == "__main__":
|
|
939
915
|
print( # noqa: T201
|
|
940
916
|
_parse_table_structure(
|
tinybird/sql_template.py
CHANGED
|
@@ -1542,35 +1542,6 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
|
|
|
1542
1542
|
raise e
|
|
1543
1543
|
|
|
1544
1544
|
|
|
1545
|
-
class CodeWriter:
|
|
1546
|
-
def __init__(self, file, template):
|
|
1547
|
-
self.file = file
|
|
1548
|
-
self.current_template = template
|
|
1549
|
-
self.apply_counter = 0
|
|
1550
|
-
self._indent = 0
|
|
1551
|
-
|
|
1552
|
-
def indent_size(self):
|
|
1553
|
-
return self._indent
|
|
1554
|
-
|
|
1555
|
-
def indent(self):
|
|
1556
|
-
class Indenter:
|
|
1557
|
-
def __enter__(_):
|
|
1558
|
-
self._indent += 1
|
|
1559
|
-
return self
|
|
1560
|
-
|
|
1561
|
-
def __exit__(_, *args):
|
|
1562
|
-
assert self._indent > 0
|
|
1563
|
-
self._indent -= 1
|
|
1564
|
-
|
|
1565
|
-
return Indenter()
|
|
1566
|
-
|
|
1567
|
-
def write_line(self, line, line_number, indent=None):
|
|
1568
|
-
if indent is None:
|
|
1569
|
-
indent = self._indent
|
|
1570
|
-
line_comment = " # %s:%d" % ("<generated>", line_number)
|
|
1571
|
-
print(" " * indent + line + line_comment, file=self.file)
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
1545
|
def get_var_names(t: Template):
|
|
1575
1546
|
"""
|
|
1576
1547
|
Extract variable names from a template.
|
tinybird/tb/__cli__.py
CHANGED
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/forward/commands'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '4.6.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '4.6.6.dev0'
|
|
8
|
+
__revision__ = '939e465'
|
tinybird/tb/client.py
CHANGED
|
@@ -3,6 +3,7 @@ import logging
|
|
|
3
3
|
import os
|
|
4
4
|
import ssl
|
|
5
5
|
import time
|
|
6
|
+
from dataclasses import dataclass
|
|
6
7
|
from pathlib import Path
|
|
7
8
|
from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Tuple, Union
|
|
8
9
|
from urllib.parse import parse_qsl, quote, urlencode, urlsplit
|
|
@@ -20,6 +21,19 @@ LAST_PARTITION = "last_partition"
|
|
|
20
21
|
ALL_PARTITIONS = "all_partitions"
|
|
21
22
|
|
|
22
23
|
|
|
24
|
+
@dataclass(frozen=True)
|
|
25
|
+
class CLIWarning:
|
|
26
|
+
message: str
|
|
27
|
+
code: Optional[str] = None
|
|
28
|
+
docs_url: Optional[str] = None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class CLIResourceFile:
|
|
33
|
+
content: str
|
|
34
|
+
warnings: list[CLIWarning]
|
|
35
|
+
|
|
36
|
+
|
|
23
37
|
class AuthException(Exception):
|
|
24
38
|
pass
|
|
25
39
|
|
|
@@ -376,8 +390,22 @@ class TinyB:
|
|
|
376
390
|
def pipe_file(self, pipe: str):
|
|
377
391
|
return self._req(f"/v1/pipes/{pipe}.pipe")
|
|
378
392
|
|
|
379
|
-
def
|
|
380
|
-
return
|
|
393
|
+
def _parse_cli_warnings(self, warnings: list[dict[str, Any]]) -> list[CLIWarning]:
|
|
394
|
+
return [
|
|
395
|
+
CLIWarning(
|
|
396
|
+
message=warning["message"],
|
|
397
|
+
code=warning.get("code"),
|
|
398
|
+
docs_url=warning.get("docs_url"),
|
|
399
|
+
)
|
|
400
|
+
for warning in warnings
|
|
401
|
+
]
|
|
402
|
+
|
|
403
|
+
def connection_file(self, connection: str) -> CLIResourceFile:
|
|
404
|
+
response = self._req(f"/v0/connectors/{connection}.connection?include_metadata=true")
|
|
405
|
+
return CLIResourceFile(
|
|
406
|
+
content=response["content"],
|
|
407
|
+
warnings=self._parse_cli_warnings(response.get("warnings", [])),
|
|
408
|
+
)
|
|
381
409
|
|
|
382
410
|
def datasource_file(self, datasource: str):
|
|
383
411
|
try:
|
tinybird/tb/config.py
CHANGED
|
@@ -91,12 +91,6 @@ def get_config(
|
|
|
91
91
|
return config
|
|
92
92
|
|
|
93
93
|
|
|
94
|
-
def write_config(config: Dict[str, Any], dest_file: str = ".tinyb") -> None:
|
|
95
|
-
config_file = Path(getcwd()) / dest_file
|
|
96
|
-
with open(config_file, "w") as file:
|
|
97
|
-
file.write(json.dumps(config, indent=4, sort_keys=True))
|
|
98
|
-
|
|
99
|
-
|
|
100
94
|
def get_display_cloud_host(api_host: str) -> str:
|
|
101
95
|
is_local = "localhost" in api_host
|
|
102
96
|
if is_local:
|
tinybird/tb/modules/build.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import time
|
|
2
1
|
from copy import deepcopy
|
|
3
2
|
from functools import partial
|
|
4
3
|
from pathlib import Path
|
|
@@ -7,7 +6,6 @@ from urllib.parse import urlencode
|
|
|
7
6
|
|
|
8
7
|
import click
|
|
9
8
|
|
|
10
|
-
from tinybird import context
|
|
11
9
|
from tinybird.datafile.exceptions import ParseException
|
|
12
10
|
from tinybird.datafile.parse_datasource import parse_datasource
|
|
13
11
|
from tinybird.datafile.parse_pipe import parse_pipe
|
|
@@ -16,11 +14,10 @@ from tinybird.tb.config import CLOUD_HOSTS
|
|
|
16
14
|
from tinybird.tb.modules.build_common import process
|
|
17
15
|
from tinybird.tb.modules.cli import cli, get_current_git_branch
|
|
18
16
|
from tinybird.tb.modules.config import CLIConfig
|
|
19
|
-
from tinybird.tb.modules.datafile.playground import folder_playground
|
|
20
17
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
21
18
|
from tinybird.tb.modules.project import Project
|
|
22
19
|
from tinybird.tb.modules.query_output import print_table_formatted
|
|
23
|
-
from tinybird.tb.modules.watch import
|
|
20
|
+
from tinybird.tb.modules.watch import watch_project
|
|
24
21
|
|
|
25
22
|
|
|
26
23
|
def _get_dashboard_url(config: Dict[str, Any], branch_name: str, is_local: bool) -> Optional[str]:
|
|
@@ -179,10 +176,6 @@ def is_vendor(f: Path) -> bool:
|
|
|
179
176
|
return f.parts[0] == "vendor"
|
|
180
177
|
|
|
181
178
|
|
|
182
|
-
def get_vendor_workspace(f: Path) -> str:
|
|
183
|
-
return f.parts[1]
|
|
184
|
-
|
|
185
|
-
|
|
186
179
|
def is_endpoint(f: Path) -> bool:
|
|
187
180
|
return f.suffix == ".pipe" and not is_vendor(f) and f.parts[0] == "endpoints"
|
|
188
181
|
|
|
@@ -207,57 +200,6 @@ def check_filenames(filenames: List[str]):
|
|
|
207
200
|
parser(filename)
|
|
208
201
|
|
|
209
202
|
|
|
210
|
-
def dev_cloud(
|
|
211
|
-
ctx: click.Context,
|
|
212
|
-
) -> None:
|
|
213
|
-
project: Project = ctx.ensure_object(dict)["project"]
|
|
214
|
-
config = CLIConfig.get_project_config()
|
|
215
|
-
tb_client: TinyB = config.get_client()
|
|
216
|
-
context.disable_template_security_validation.set(True)
|
|
217
|
-
|
|
218
|
-
def process(filenames: List[str], watch: bool = False):
|
|
219
|
-
datafiles = [f for f in filenames if f.endswith((".datasource", ".pipe"))]
|
|
220
|
-
if len(datafiles) > 0:
|
|
221
|
-
check_filenames(filenames=datafiles)
|
|
222
|
-
folder_playground(
|
|
223
|
-
project, config, tb_client, filenames=datafiles, is_internal=False, current_ws=None, local_ws=None
|
|
224
|
-
)
|
|
225
|
-
if len(filenames) > 0 and watch:
|
|
226
|
-
filename = filenames[0]
|
|
227
|
-
build_and_print_resource(config, tb_client, filename)
|
|
228
|
-
|
|
229
|
-
datafiles = project.get_project_files()
|
|
230
|
-
filenames = datafiles
|
|
231
|
-
|
|
232
|
-
def build_once(filenames: List[str]):
|
|
233
|
-
ok = False
|
|
234
|
-
try:
|
|
235
|
-
click.echo(FeedbackManager.highlight(message="» Building project...\n"))
|
|
236
|
-
time_start = time.time()
|
|
237
|
-
process(filenames=filenames, watch=False)
|
|
238
|
-
time_end = time.time()
|
|
239
|
-
elapsed_time = time_end - time_start
|
|
240
|
-
|
|
241
|
-
click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s"))
|
|
242
|
-
ok = True
|
|
243
|
-
except Exception as e:
|
|
244
|
-
error_path = Path(".tb_error.txt")
|
|
245
|
-
if error_path.exists():
|
|
246
|
-
content = error_path.read_text()
|
|
247
|
-
content += f"\n\n{str(e)}"
|
|
248
|
-
error_path.write_text(content)
|
|
249
|
-
else:
|
|
250
|
-
error_path.write_text(str(e))
|
|
251
|
-
click.echo(FeedbackManager.error_exception(error=e))
|
|
252
|
-
ok = False
|
|
253
|
-
return ok
|
|
254
|
-
|
|
255
|
-
build_ok = build_once(filenames)
|
|
256
|
-
|
|
257
|
-
click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
|
|
258
|
-
watch_files(filenames=filenames, process=process, project=project, build_ok=build_ok)
|
|
259
|
-
|
|
260
|
-
|
|
261
203
|
def build_and_print_resource(config: CLIConfig, tb_client: TinyB, filename: str):
|
|
262
204
|
resource_path = Path(filename)
|
|
263
205
|
name = resource_path.stem
|
tinybird/tb/modules/cli.py
CHANGED
|
@@ -206,10 +206,6 @@ def get_tinybird_branch_name_from_git_branch(branch_name: Optional[str]) -> Opti
|
|
|
206
206
|
return sanitized or None
|
|
207
207
|
|
|
208
208
|
|
|
209
|
-
def get_tinybird_branch_name() -> Optional[str]:
|
|
210
|
-
return get_tinybird_branch_name_from_git_branch(get_current_git_branch())
|
|
211
|
-
|
|
212
|
-
|
|
213
209
|
def is_tinybird_local_running(timeout_seconds: float = 0.2) -> bool:
|
|
214
210
|
try:
|
|
215
211
|
with socket.create_connection((TB_LOCAL_HOST, TB_LOCAL_PORT), timeout=timeout_seconds):
|