tinybird 0.0.1.dev43__tar.gz → 0.0.1.dev46__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of tinybird might be problematic. Click here for more details.

Files changed (104) hide show
  1. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/PKG-INFO +1 -1
  2. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/client.py +17 -1
  3. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/prompts.py +135 -15
  4. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/__cli__.py +2 -2
  5. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/cli.py +1 -1
  6. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/build.py +28 -20
  7. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/cli.py +18 -62
  8. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/common.py +3 -2
  9. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/copy.py +1 -1
  10. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/create.py +134 -59
  11. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/build.py +12 -221
  12. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/common.py +1 -1
  13. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/format_datasource.py +1 -1
  14. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/format_pipe.py +4 -4
  15. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/pipe_checker.py +3 -3
  16. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datasource.py +1 -1
  17. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/deployment.py +1 -1
  18. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/endpoint.py +89 -2
  19. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/feedback_manager.py +5 -1
  20. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/local_common.py +10 -7
  21. tinybird-0.0.1.dev46/tinybird/tb/modules/materialization.py +146 -0
  22. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/mock.py +56 -16
  23. tinybird-0.0.1.dev46/tinybird/tb/modules/pipe.py +72 -0
  24. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/project.py +10 -4
  25. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/shell.py +3 -3
  26. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/test.py +73 -38
  27. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/tinyunit/tinyunit.py +1 -1
  28. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/update.py +1 -1
  29. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/workspace.py +2 -1
  30. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird.egg-info/PKG-INFO +1 -1
  31. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird.egg-info/SOURCES.txt +1 -1
  32. tinybird-0.0.1.dev43/tinybird/tb/modules/build_client.py +0 -199
  33. tinybird-0.0.1.dev43/tinybird/tb/modules/pipe.py +0 -390
  34. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/setup.cfg +0 -0
  35. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/__cli__.py +0 -0
  36. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/ch_utils/constants.py +0 -0
  37. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/ch_utils/engine.py +0 -0
  38. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/check_pypi.py +0 -0
  39. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/config.py +0 -0
  40. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/connectors.py +0 -0
  41. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/context.py +0 -0
  42. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/datafile.py +0 -0
  43. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/datatypes.py +0 -0
  44. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/feedback_manager.py +0 -0
  45. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/git_settings.py +0 -0
  46. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/sql.py +0 -0
  47. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/sql_template.py +0 -0
  48. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/sql_template_fmt.py +0 -0
  49. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/sql_toolset.py +0 -0
  50. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/syncasync.py +0 -0
  51. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/auth.py +0 -0
  52. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/cicd.py +0 -0
  53. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/config.py +0 -0
  54. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/build_common.py +0 -0
  55. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/build_datasource.py +0 -0
  56. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/build_pipe.py +0 -0
  57. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/diff.py +0 -0
  58. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/exceptions.py +0 -0
  59. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/fixture.py +0 -0
  60. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/format_common.py +0 -0
  61. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/parse_datasource.py +0 -0
  62. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/parse_pipe.py +0 -0
  63. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/datafile/pull.py +0 -0
  64. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/exceptions.py +0 -0
  65. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/fmt.py +0 -0
  66. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/job.py +0 -0
  67. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/llm.py +0 -0
  68. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/llm_utils.py +0 -0
  69. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/local.py +0 -0
  70. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/login.py +0 -0
  71. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/regions.py +0 -0
  72. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/table.py +0 -0
  73. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/tag.py +0 -0
  74. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/telemetry.py +0 -0
  75. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -0
  76. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/token.py +0 -0
  77. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/watch.py +0 -0
  78. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb/modules/workspace_members.py +0 -0
  79. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli.py +0 -0
  80. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/auth.py +0 -0
  81. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/branch.py +0 -0
  82. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/cicd.py +0 -0
  83. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/cli.py +0 -0
  84. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/common.py +0 -0
  85. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/config.py +0 -0
  86. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/connection.py +0 -0
  87. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/datasource.py +0 -0
  88. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/exceptions.py +0 -0
  89. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/fmt.py +0 -0
  90. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/job.py +0 -0
  91. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/pipe.py +0 -0
  92. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/regions.py +0 -0
  93. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/tag.py +0 -0
  94. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/telemetry.py +0 -0
  95. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/test.py +0 -0
  96. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
  97. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
  98. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/workspace.py +0 -0
  99. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tb_cli_modules/workspace_members.py +0 -0
  100. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird/tornado_template.py +0 -0
  101. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird.egg-info/dependency_links.txt +0 -0
  102. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird.egg-info/entry_points.txt +0 -0
  103. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird.egg-info/requires.txt +0 -0
  104. {tinybird-0.0.1.dev43 → tinybird-0.0.1.dev46}/tinybird.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: tinybird
3
- Version: 0.0.1.dev43
3
+ Version: 0.0.1.dev46
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/cli/introduction.html
6
6
  Author: Tinybird
@@ -1,6 +1,7 @@
1
1
  import asyncio
2
2
  import json
3
3
  import logging
4
+ import os
4
5
  import ssl
5
6
  from pathlib import Path
6
7
  from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Union
@@ -91,6 +92,7 @@ class TinyB:
91
92
  disable_ssl_checks: bool = False,
92
93
  send_telemetry: bool = False,
93
94
  semver: Optional[str] = None,
95
+ env: Optional[str] = "production",
94
96
  ):
95
97
  ctx = ssl.create_default_context()
96
98
  ctx.check_hostname = False
@@ -102,6 +104,7 @@ class TinyB:
102
104
  self.disable_ssl_checks = disable_ssl_checks
103
105
  self.send_telemetry = send_telemetry
104
106
  self.semver = semver
107
+ self.env = env
105
108
 
106
109
  async def _req_raw(
107
110
  self,
@@ -687,7 +690,20 @@ class TinyB:
687
690
  return await self._req(f"/v0/jobs/{job_id}/cancel", method="POST", data=b"")
688
691
 
689
692
  async def user_workspaces(self):
690
- return await self._req("/v0/user/workspaces/?with_environments=false")
693
+ data = await self._req("/v0/user/workspaces/?with_environments=false")
694
+ # TODO: this is repeated in local_common.py but I'm avoiding circular imports
695
+ local_port = int(os.getenv("TB_LOCAL_PORT", 80))
696
+ local_host = f"http://localhost:{local_port}"
697
+ if local_host != self.host:
698
+ return data
699
+
700
+ local_workspaces = [
701
+ x
702
+ for x in data["workspaces"]
703
+ if x["name"].startswith("Tinybird_Local_") and not x["name"].startswith("Tinybird_Local_Build_")
704
+ ]
705
+
706
+ return {**data, "workspaces": local_workspaces}
691
707
 
692
708
  async def user_workspaces_and_branches(self):
693
709
  return await self._req("/v0/user/workspaces/?with_environments=true")
@@ -1,3 +1,5 @@
1
+ from typing import Optional
2
+
1
3
  general_functions = [
2
4
  "BLAKE3",
3
5
  "CAST",
@@ -368,8 +370,7 @@ You are a Tinybird expert. You will be given a pipe containing different nodes w
368
370
 
369
371
  <instructions>
370
372
  - Every test name must be unique.
371
- - The test command must be a valid Tinybird command that can be run in the terminal.
372
- - The test command can have as many parameters as are needed to test the pipe.
373
+ - The test can have as many parameters as are needed to test the pipe.
373
374
  - The parameter within Tinybird templating syntax looks like this one {{String(my_param_name, default_value)}}.
374
375
  - If there are no parameters, you can omit parameters and generate a single test.
375
376
  - The format of the parameters is the following: ?param1=value1&param2=value2&param3=value3
@@ -397,9 +398,19 @@ Follow the instructions and generate the following response with no additional t
397
398
  """
398
399
 
399
400
 
400
- def create_prompt(existing_resources: str) -> str:
401
+ def create_prompt(existing_resources: str, feedback: str = "", history: str = "") -> str:
402
+ feedback_history = ""
403
+ if feedback and history:
404
+ feedback_history = f"""In case the <feedback> and <history> tags are present and not empty,
405
+ it means there was a previous attempt to generate the resources and the user provided feedback and history about previous responses.
406
+ Use the following feedback and history to regenerate the response:
407
+ Feedback to improve the response:
408
+ {feedback}
409
+ History of previous results:
410
+ {history}"""
411
+
401
412
  return """
402
- You are a Tinybird expert. You will be given a prompt to generate Tinybird resources: datasources and/or pipes.
413
+ You are a Tinybird expert. You will be given a prompt to generate new or update existing Tinybird resources: datasources and/or pipes.
403
414
  <existing_resources>{existing_resources}</existing_resources>
404
415
  {datasource_instructions}
405
416
  {pipe_instructions}
@@ -408,6 +419,9 @@ You are a Tinybird expert. You will be given a prompt to generate Tinybird resou
408
419
  {pipe_example}
409
420
  {copy_pipe_instructions}
410
421
  {materialized_pipe_instructions}
422
+
423
+ {feedback_history}
424
+
411
425
  Use the following format to generate the response and do not wrap it in any other text, including the <response> tag.
412
426
  <response>
413
427
  <resource>
@@ -426,10 +440,21 @@ Use the following format to generate the response and do not wrap it in any othe
426
440
  pipe_example=pipe_example,
427
441
  copy_pipe_instructions=copy_pipe_instructions,
428
442
  materialized_pipe_instructions=materialized_pipe_instructions,
443
+ feedback_history=feedback_history,
429
444
  )
430
445
 
431
446
 
432
- def mock_prompt(rows: int) -> str:
447
+ def mock_prompt(rows: int, feedback: str = "", history: str = "") -> str:
448
+ feedback_history = ""
449
+ if feedback and history:
450
+ feedback_history = f"""In case the <feedback> and <history> tags are present and not empty,
451
+ it means there was a previous attempt to generate the resources and the user provided feedback and history about previous responses.
452
+ Use the following feedback and history to regenerate the response:
453
+ Feedback to improve the response:
454
+ {feedback}
455
+ History of previous results:
456
+ {history}"""
457
+
433
458
  return f"""
434
459
  Given the schema for a Tinybird datasource, return a can you create a clickhouse sql query to generate some random data that matches that schema.
435
460
 
@@ -570,6 +595,9 @@ Follow the instructions and generate the following response with no additional t
570
595
  <response>
571
596
  <sql>[raw sql query here]</sql>
572
597
  </response>
598
+
599
+ {feedback_history}
600
+
573
601
  """
574
602
 
575
603
 
@@ -732,6 +760,10 @@ sql_instructions = """
732
760
  AND {{{{DateTime(end_date)}}}}
733
761
  {{%end%}}
734
762
  </valid_condition_without_now>
763
+ - Parameters must not be quoted.
764
+ - When you use defined function with a paremeter inside, do NOT add quotes around the parameter:
765
+ <invalid_defined_function_with_parameter>{{% if defined('my_param') %}}</invalid_defined_function_with_parameter>
766
+ <valid_defined_function_without_parameter>{{% if defined(my_param) %}}</valid_defined_function_without_parameter>
735
767
  - Use datasource names as table names when doing SELECT statements.
736
768
  - Do not use pipe names as table names.
737
769
  - The available datasource names to use in the SQL are the ones present in the existing_resources section or the ones you will create.
@@ -749,7 +781,6 @@ sql_instructions = """
749
781
  - When aliasing a column, use first the column name and then the alias.
750
782
  - General functions and aggregate functions are case sensitive.
751
783
  - Character insensitive functions are case insensitive.
752
- - When you use defined function with a paremeter inside, do NOT add quotes around the parameter.
753
784
  - Parameters are never quoted in any case.
754
785
  </sql_instructions>
755
786
  """.format(
@@ -792,22 +823,111 @@ Use the following format to generate the response and do not wrap it in any othe
792
823
  )
793
824
 
794
825
 
795
- cursorrules_prompt = """
826
+ def rules_prompt(source: Optional[str] = None) -> str:
827
+ base_command = source or "tb"
828
+ return """
796
829
  You are an expert in SQL and Tinybird. Follow these instructions when working with .datasource and .pipe files:
797
830
 
831
+ <command_calling>
832
+ You have commands at your disposal to develop a tinybird project:
833
+ - {base_command} build: to build the project locally and check it works.
834
+ - {base_command} deployment create --wait --auto: to create a deployment and promote it automatically
835
+ - {base_command} test run: to run existing tests
836
+ - {base_command} --build endpoint url <pipe_name>: to get the url of an endpoint, token included.
837
+ - {base_command} --build endpoint data <pipe_name>: to get the data of an endpoint. You can pass parameters to the endpoint like this: {base_command} --build endpoint data <pipe_name> --param1 value1 --param2 value2
838
+ - {base_command} --build token ls: to list all the tokens
839
+ There are other commands that you can use, but these are the most common ones. Run `{base_command} -h` to see all the commands if needed.
840
+ When you need to check resources or data in the Tinybird environment that you updated with the build command, add always the --build flag before the command. Example: {base_command} --build datasource ls
841
+ </command_calling>
842
+ <development_instructions>
843
+ - When asking to create a tinybird data project, if the needed folders are not already created, use the following structure:
844
+ ├── copies
845
+ ├── datasources
846
+ ├── endpoints
847
+ ├── fixtures
848
+ ├── materializations
849
+ ├── sinks
850
+ └── tests
851
+ - The local development server will be available at http://localhost:80. Even if some response uses another base url, use always http://localhost:80.
852
+ - After every change in your .datasource, .pipe or .ndjson files, run `{base_command} build` to build the project locally.
853
+ - When you need to ingest data locally in a datasource, create a .ndjson file with the same name of the datasource and the data you want and run `{base_command} build` so the data is ingested.
854
+ - The format of the generated api endpoint urls is: http://localhost:80/v0/pipe/<pipe_name>.json?token=<token>
855
+ - Before running the tests, remember to have the project built with `{base_command} build` with the latest changes.
856
+ </development_instructions>
857
+
858
+ <datasource_file_instructions>
859
+ Follow these instructions when creating or updating .datasource files:
798
860
  {datasource_instructions}
861
+ </datasource_file_instructions>
862
+
863
+ <pipe_file_instructions>
864
+ Follow these instructions when creating or updating .pipe files:
799
865
  {pipe_instructions}
800
866
  {sql_instructions}
801
867
  {datasource_example}
802
868
  {pipe_example}
803
869
  {copy_pipe_instructions}
804
870
  {materialized_pipe_instructions}
871
+ </pipe_file_instructions>
872
+ <test_file_instructions>
873
+ Follow these instructions when creating or updating .yaml files for tests:
874
+ {test_instructions}
875
+ </test_file_instructions>
805
876
  """.format(
806
- datasource_instructions=datasource_instructions,
807
- pipe_instructions=pipe_instructions,
808
- sql_instructions=sql_instructions,
809
- datasource_example=datasource_example,
810
- pipe_example=pipe_example,
811
- copy_pipe_instructions=copy_pipe_instructions,
812
- materialized_pipe_instructions=materialized_pipe_instructions,
813
- )
877
+ base_command=base_command,
878
+ datasource_instructions=datasource_instructions,
879
+ pipe_instructions=pipe_instructions,
880
+ sql_instructions=sql_instructions,
881
+ datasource_example=datasource_example,
882
+ pipe_example=pipe_example,
883
+ copy_pipe_instructions=copy_pipe_instructions,
884
+ materialized_pipe_instructions=materialized_pipe_instructions,
885
+ test_instructions=test_instructions,
886
+ )
887
+
888
+
889
+ test_instructions = """
890
+ - The test file name must match the name of the pipe it is testing.
891
+ - Every scenario name must be unique inside the test file.
892
+ - When looking for the parameters available, you will find them in the pipes in the following format: {{{{String(my_param_name, default_value)}}}}.
893
+ - If there are no parameters, you can omit parameters and generate a single test.
894
+ - The format of the parameters is the following: param1=value1&param2=value2&param3=value3
895
+ - If some parameters are provided by the user and you need to use them, preserve in the same format as they were provided, like case sensitive
896
+ - Test as many scenarios as possible.
897
+ - The format of the test file is the following:
898
+ <test_file_format>
899
+ - name: kpis_single_day
900
+ description: Test hourly granularity for a single day
901
+ parameters: date_from=2024-01-01&date_to=2024-01-01
902
+ expected_result: |
903
+ {"date":"2024-01-01 00:00:00","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0}
904
+ {"date":"2024-01-01 01:00:00","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0}
905
+
906
+ - name: kpis_date_range
907
+ description: Test daily granularity for a date range
908
+ parameters: date_from=2024-01-01&date_to=2024-01-31
909
+ expected_result: |
910
+ {"date":"2024-01-01","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0}
911
+ {"date":"2024-01-02","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0}
912
+
913
+ - name: kpis_default_range
914
+ description: Test default behavior without date parameters (last 7 days)
915
+ parameters: ''
916
+ expected_result: |
917
+ {"date":"2025-01-10","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0}
918
+ {"date":"2025-01-11","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0}
919
+
920
+ - name: kpis_fixed_time
921
+ description: Test with fixed timestamp for consistent testing
922
+ parameters: fixed_time=2024-01-15T12:00:00
923
+ expected_result: ''
924
+
925
+ - name: kpis_single_day
926
+ description: Test single day with hourly granularity
927
+ parameters: date_from=2024-01-01&date_to=2024-01-01
928
+ expected_result: |
929
+ {"date":"2024-01-01 00:00:00","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0}
930
+ {"date":"2024-01-01 01:00:00","visits":0,"pageviews":0,"bounce_rate":null,"avg_session_sec":0}
931
+
932
+ </test_file_format>
933
+ """
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
4
4
  __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
5
5
  __author__ = 'Tinybird'
6
6
  __author_email__ = 'support@tinybird.co'
7
- __version__ = '0.0.1.dev43'
8
- __revision__ = 'ecb1311'
7
+ __version__ = '0.0.1.dev46'
8
+ __revision__ = '7b9942e'
@@ -6,7 +6,6 @@ if sys.platform == "win32":
6
6
 
7
7
  import tinybird.tb.modules.auth
8
8
  import tinybird.tb.modules.build
9
- import tinybird.tb.modules.build_client
10
9
  import tinybird.tb.modules.cli
11
10
  import tinybird.tb.modules.common
12
11
  import tinybird.tb.modules.copy
@@ -18,6 +17,7 @@ import tinybird.tb.modules.fmt
18
17
  import tinybird.tb.modules.job
19
18
  import tinybird.tb.modules.local
20
19
  import tinybird.tb.modules.login
20
+ import tinybird.tb.modules.materialization
21
21
  import tinybird.tb.modules.mock
22
22
  import tinybird.tb.modules.pipe
23
23
  import tinybird.tb.modules.tag
@@ -12,6 +12,7 @@ import requests
12
12
  from tinybird.client import TinyB
13
13
  from tinybird.tb.modules.cli import cli
14
14
  from tinybird.tb.modules.common import push_data
15
+ from tinybird.tb.modules.datafile.build import folder_build
15
16
  from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
16
17
  from tinybird.tb.modules.feedback_manager import FeedbackManager
17
18
  from tinybird.tb.modules.local_common import get_tinybird_local_client
@@ -27,19 +28,19 @@ def build(ctx: click.Context, watch: bool) -> None:
27
28
  """
28
29
  Validate and build the project server side.
29
30
  """
30
-
31
31
  project: Project = ctx.ensure_object(dict)["project"]
32
- tb_client = asyncio.run(get_tinybird_local_client(str(project.path)))
33
- click.echo(FeedbackManager.highlight(message="\n» Building project..."))
32
+ tb_client = asyncio.run(get_tinybird_local_client(str(project.path), build=True))
33
+ click.echo(FeedbackManager.highlight_building_project())
34
34
  time_start = time.time()
35
35
 
36
36
  def process(file_changed: Optional[str] = None, diff: Optional[str] = None) -> None:
37
37
  if file_changed and file_changed.endswith(".ndjson"):
38
38
  rebuild_fixture(project, tb_client, file_changed)
39
39
  else:
40
- build_project(project, tb_client)
40
+ build_project(project, tb_client, file_changed)
41
41
  try:
42
42
  if file_changed:
43
+ asyncio.run(folder_build(project, filenames=[file_changed]))
43
44
  build_and_print_resource(tb_client, file_changed, diff)
44
45
  except Exception:
45
46
  pass
@@ -61,7 +62,7 @@ def build(ctx: click.Context, watch: bool) -> None:
61
62
  shell.run()
62
63
 
63
64
 
64
- def build_project(project: Project, tb_client: TinyB) -> None:
65
+ def build_project(project: Project, tb_client: TinyB, file_changed: Optional[str] = None) -> None:
65
66
  MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
66
67
  DATAFILE_TYPE_TO_CONTENT_TYPE = {
67
68
  ".datasource": "text/plain",
@@ -70,6 +71,7 @@ def build_project(project: Project, tb_client: TinyB) -> None:
70
71
  TINYBIRD_API_URL = tb_client.host + "/v1/build"
71
72
  logging.debug(TINYBIRD_API_URL)
72
73
  TINYBIRD_API_KEY = tb_client.token
74
+ error = False
73
75
  try:
74
76
  files = [
75
77
  ("context://", ("cli-version", "1.0.0", "text/plain")),
@@ -103,20 +105,23 @@ def build_project(project: Project, tb_client: TinyB) -> None:
103
105
  if build_result == "success":
104
106
  datasources = result.get("datasources", [])
105
107
  pipes = result.get("pipes", [])
106
- for ds in datasources:
107
- ds_path_str: Optional[str] = next(
108
- (p for p in project_files if p.endswith(ds.get("name") + ".datasource")), None
109
- )
110
- if ds_path_str:
111
- ds_path = Path(ds_path_str)
112
- ds_path_str = ds_path_str.replace(f"{project.folder}/", "")
113
- click.echo(FeedbackManager.info(message=f"{ds_path_str} created"))
114
- for pipe in pipes:
115
- pipe_name = pipe.get("name")
116
- pipe_path_str: Optional[str] = next((p for p in project_files if p.endswith(pipe_name + ".pipe")), None)
117
- if pipe_path_str:
118
- pipe_path_str = pipe_path_str.replace(f"{project.folder}/", "")
119
- click.echo(FeedbackManager.info(message=f"✓ {pipe_path_str} created"))
108
+ if not file_changed:
109
+ for ds in datasources:
110
+ ds_path_str: Optional[str] = next(
111
+ (p for p in project_files if p.endswith(ds.get("name") + ".datasource")), None
112
+ )
113
+ if ds_path_str:
114
+ ds_path = Path(ds_path_str)
115
+ ds_path_str = ds_path_str.replace(f"{project.folder}/", "")
116
+ click.echo(FeedbackManager.info(message=f"✓ {ds_path_str} created"))
117
+ for pipe in pipes:
118
+ pipe_name = pipe.get("name")
119
+ pipe_path_str: Optional[str] = next(
120
+ (p for p in project_files if p.endswith(pipe_name + ".pipe")), None
121
+ )
122
+ if pipe_path_str:
123
+ pipe_path_str = pipe_path_str.replace(f"{project.folder}/", "")
124
+ click.echo(FeedbackManager.info(message=f"✓ {pipe_path_str} created"))
120
125
 
121
126
  try:
122
127
  for filename in project_files:
@@ -136,13 +141,14 @@ def build_project(project: Project, tb_client: TinyB) -> None:
136
141
  pass
137
142
 
138
143
  elif build_result == "failed":
139
- click.echo(FeedbackManager.error(message="Build failed"))
140
144
  build_errors = result.get("errors")
145
+ error = True
141
146
  for build_error in build_errors:
142
147
  filename_bit = f"{build_error.get('filename', '')}"
143
148
  error_msg = ((filename_bit + "\n\n") if filename_bit else "") + build_error.get("error")
144
149
  click.echo(FeedbackManager.error(message=error_msg))
145
150
  else:
151
+ error = True
146
152
  click.echo(FeedbackManager.error(message=f"Unknown build result. Error: {result.get('error')}"))
147
153
 
148
154
  except Exception as e:
@@ -150,6 +156,8 @@ def build_project(project: Project, tb_client: TinyB) -> None:
150
156
  finally:
151
157
  for fd in fds:
152
158
  fd.close()
159
+ if error:
160
+ raise click.ClickException(FeedbackManager.error_build_failed())
153
161
 
154
162
 
155
163
  def append_fixture(
@@ -6,10 +6,9 @@
6
6
  import json
7
7
  import logging
8
8
  import os
9
- import pprint
10
9
  from os import getcwd
11
10
  from pathlib import Path
12
- from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
11
+ from typing import Any, Callable, Dict, List, Optional, Tuple, Union
13
12
 
14
13
  import click
15
14
  import humanfriendly
@@ -35,13 +34,7 @@ from tinybird.tb.modules.common import (
35
34
  )
36
35
  from tinybird.tb.modules.config import CLIConfig
37
36
  from tinybird.tb.modules.datafile.build import build_graph
38
- from tinybird.tb.modules.datafile.common import Datafile, DatafileSyntaxError
39
37
  from tinybird.tb.modules.datafile.diff import diff_command
40
- from tinybird.tb.modules.datafile.exceptions import (
41
- ParseException,
42
- )
43
- from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
44
- from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
45
38
  from tinybird.tb.modules.datafile.pull import folder_pull
46
39
  from tinybird.tb.modules.feedback_manager import FeedbackManager
47
40
  from tinybird.tb.modules.llm import LLM
@@ -66,12 +59,20 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
66
59
  @click.option("--host", help="Use custom host, defaults to TB_HOST envvar, then to https://api.tinybird.co")
67
60
  @click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
68
61
  @click.option("--prod/--local", is_flag=True, default=False, help="Run against production or local")
62
+ @click.option("--build", is_flag=True, default=False, help="Run against build mode")
69
63
  @click.option("--folder", type=str, help="Folder where files will be placed")
70
64
  @click.version_option(version=VERSION)
71
65
  @click.pass_context
72
66
  @coro
73
67
  async def cli(
74
- ctx: Context, debug: bool, token: str, host: str, show_tokens: bool, prod: bool, folder: Optional[str]
68
+ ctx: Context,
69
+ debug: bool,
70
+ token: str,
71
+ host: str,
72
+ show_tokens: bool,
73
+ prod: bool,
74
+ build: bool,
75
+ folder: Optional[str],
75
76
  ) -> None:
76
77
  """
77
78
  Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
@@ -123,64 +124,19 @@ async def cli(
123
124
 
124
125
  logging.debug("debug enabled")
125
126
 
126
- skip_client = ctx.invoked_subcommand in ["auth", "check", "login", "workspace", "local", "build"]
127
- client = await create_ctx_client(config, prod, skip_client, project)
127
+ skip_client = ctx.invoked_subcommand in ["auth", "check", "login", "local", "build"]
128
+ client = await create_ctx_client(config, prod, build, skip_client, project)
128
129
 
129
130
  if client:
131
+ if not build:
132
+ target = config.get("name", "production") if prod else "Tinybird local"
133
+ click.echo(FeedbackManager.gray(message=f"Running against {target}\n"))
130
134
  ctx.ensure_object(dict)["client"] = client
131
135
 
132
136
  ctx.ensure_object(dict)["project"] = project
133
137
 
134
138
 
135
- @cli.command(hidden=True)
136
- @click.argument("filenames", type=click.Path(exists=True), nargs=-1, default=None)
137
- @click.option("--debug", is_flag=True, default=False, help="Print internal representation")
138
- @click.pass_context
139
- def check(ctx: Context, filenames: List[str], debug: bool) -> None:
140
- """Check file syntax."""
141
-
142
- if not filenames:
143
- project: Project = ctx.ensure_object(dict)["project"]
144
- filenames = project.get_project_files()
145
-
146
- def process(filenames: Iterable):
147
- parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
148
- incl_suffix = ".incl"
149
- try:
150
- for filename in filenames:
151
- if os.path.isdir(filename):
152
- process(filenames=filename)
153
-
154
- click.echo(FeedbackManager.info_processing_file(filename=filename))
155
-
156
- file_suffix = Path(filename).suffix
157
- if file_suffix == incl_suffix:
158
- click.echo(FeedbackManager.info_ignoring_incl_file(filename=filename))
159
- continue
160
-
161
- doc: Datafile
162
- parser = parser_matrix.get(file_suffix)
163
- if not parser:
164
- raise ParseException(FeedbackManager.error_unsupported_datafile(extension=file_suffix))
165
-
166
- doc = parser(filename)
167
-
168
- click.echo(FeedbackManager.success_processing_file(filename=filename))
169
- if debug:
170
- pp = pprint.PrettyPrinter()
171
- for x in doc.nodes:
172
- pp.pprint(x)
173
-
174
- except DatafileSyntaxError as e:
175
- # TODO(eclbg): add the filename to the error message
176
- raise CLIException(str(e))
177
- except ParseException as e:
178
- raise CLIException(FeedbackManager.error_exception(error=e))
179
-
180
- process(filenames=filenames)
181
-
182
-
183
- @cli.command(hidden=True)
139
+ @cli.command()
184
140
  @click.option(
185
141
  "--folder", default=None, type=click.Path(exists=True, file_okay=False), help="Folder where files will be placed"
186
142
  )
@@ -489,11 +445,11 @@ def __unpatch_click_output():
489
445
  click.secho = __old_click_secho
490
446
 
491
447
 
492
- async def create_ctx_client(config: Dict[str, Any], prod: bool, skip_client: bool, project: Project):
448
+ async def create_ctx_client(config: Dict[str, Any], prod: bool, build: bool, skip_client: bool, project: Project):
493
449
  if skip_client:
494
450
  return None
495
451
 
496
452
  if prod:
497
453
  return _get_tb_client(config.get("token", None), config["host"])
498
454
 
499
- return await get_tinybird_local_client(str(project.path))
455
+ return await get_tinybird_local_client(str(project.path), build=build)
@@ -163,7 +163,7 @@ def generate_datafile(
163
163
  force: Optional[bool] = False,
164
164
  _format: Optional[str] = "csv",
165
165
  folder: Optional[str] = None,
166
- ):
166
+ ) -> Path:
167
167
  p = Path(filename)
168
168
  base = Path("datasources")
169
169
  if folder:
@@ -190,6 +190,7 @@ def generate_datafile(
190
190
  fixture_file.write(data[: data.rfind(newline)])
191
191
  else:
192
192
  click.echo(FeedbackManager.error_file_already_exists(file=f))
193
+ return f
193
194
 
194
195
 
195
196
  async def get_current_workspace(config: CLIConfig) -> Optional[Dict[str, Any]]:
@@ -1361,7 +1362,7 @@ async def try_update_config_with_remote(
1361
1362
  def ask_for_admin_token_interactively(ui_host: str, default_token: Optional[str]) -> str:
1362
1363
  return (
1363
1364
  click.prompt(
1364
- f"\nCopy the \"admin your@email\" token from {ui_host}/tokens and paste it here { 'OR press enter to use the token from .tinyb file' if default_token else ''}",
1365
+ f'\nCopy the "admin your@email" token from {ui_host}/tokens and paste it here {"OR press enter to use the token from .tinyb file" if default_token else ""}',
1365
1366
  hide_input=True,
1366
1367
  show_default=False,
1367
1368
  default=default_token,
@@ -18,7 +18,7 @@ from tinybird.tb.modules.exceptions import CLIPipeException
18
18
  from tinybird.tb.modules.feedback_manager import FeedbackManager
19
19
 
20
20
 
21
- @cli.group(hidden=True)
21
+ @cli.group()
22
22
  @click.pass_context
23
23
  def copy(ctx):
24
24
  """Copy pipe commands"""