tinybird 0.0.1.dev222__py3-none-any.whl → 0.0.1.dev224__py3-none-any.whl

tinybird/ch_utils/constants.py

@@ -254,4 +254,5 @@ VALID_QUERY_FORMATS = (
     "Parquet",
     "JSONStrings",
     "Prometheus",
+    "Native",
 )

tinybird/prompts.py

@@ -420,9 +420,11 @@ You are a Tinybird expert. You will be given a prompt to generate new or update
 {pipe_example}
 {copy_pipe_instructions}
 {materialized_pipe_instructions}
+{sink_pipe_instructions}
 {connection_instructions}
 {kafka_connection_example}
 {gcs_connection_example}
+{gcs_hmac_connection_example}
 {s3_connection_example}
 
 {feedback_history}
@@ -445,9 +447,11 @@ Use the following format to generate the response and do not wrap it in any othe
         pipe_example=pipe_example,
         copy_pipe_instructions=copy_pipe_instructions,
         materialized_pipe_instructions=materialized_pipe_instructions,
+        sink_pipe_instructions=sink_pipe_instructions,
         connection_instructions=connection_instructions,
         kafka_connection_example=kafka_connection_example,
         gcs_connection_example=gcs_connection_example,
+        gcs_hmac_connection_example=gcs_hmac_connection_example,
         s3_connection_example=s3_connection_example,
         feedback_history=feedback_history,
     )
@@ -639,6 +643,43 @@ COPY_SCHEDULE 0 * * * *
 </copy_pipe_instructions>
 """
 
+sink_pipe_instructions = """
+<sink_pipe_instructions>
+- Do not create sink pipes by default, unless the user asks for it.
+- Sink pipes should be created in the /sinks folder.
+- In a .pipe file you can define how to export the result of a Pipe to an external system, optionally with a schedule.
+- Valid external systems are Kafka, S3, GCS.
+- Sink pipes depend on a connection, if no connection is provided, search for an existing connection that suits the request. If none, create a new connection.
+- Do not include EXPORT_SCHEDULE in the .pipe file unless is specifically requested by the user.
+- EXPORT_SCHEDULE is a cron expression that defines the schedule of the sink pipe.
+- EXPORT_SCHEDULE is optional and if not provided, the sink pipe will be executed only once.
+- EXPORT_CONNECTION_NAME is the name of the connection used to export.
+- TYPE SINK is the type of the pipe and it is mandatory for sink pipes.
+- If the sink pipe uses parameters, you must include the % character and a newline on top of every query to be able to use the parameters.
+- The content of the .pipe file must follow this format:
+DESCRIPTION Sink Pipe to export sales hour every hour using my_connection
+
+NODE daily_sales
+SQL >
+    %
+    SELECT toStartOfDay(starting_date) day, country, sum(sales) as total_sales
+    FROM teams
+    WHERE
+    day BETWEEN toStartOfDay(now()) - interval 1 day AND toStartOfDay(now())
+    and country = {{ String(country, 'US')}}
+    GROUP BY day, country
+
+TYPE sink
+EXPORT_CONNECTION_NAME "my_connection"
+EXPORT_BUCKET_URI "s3://tinybird-sinks"
+EXPORT_FILE_TEMPLATE "daily_prices"
+EXPORT_SCHEDULE "*/5 * * * *"
+EXPORT_FORMAT "csv"
+EXPORT_COMPRESSION "gz"
+EXPORT_WRITE_STRATEGY "truncate"
+</sink_pipe_instructions>
+"""
+
 materialized_pipe_instructions = """
 <materialized_pipe_instructions>
 - Do not create materialized pipes by default, unless the user asks for it.
@@ -709,6 +750,14 @@ GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{ tb_secret("PRODUCTION_GCS_SERVICE_ACCOUN
 </gcs_connection_content>
 """
 
+gcs_hmac_connection_example = """
+<gcs_hmac_connection_content>
+TYPE gcs
+GCS_HMAC_ACCESS_ID {{ tb_secret("gcs_hmac_access_id") }}
+GCS_HMAC_SECRET {{ tb_secret("gcs_hmac_secret") }}
+</gcs_hmac_connection_content>
+"""
+
 s3_connection_example = """
 <s3_connection_content>
 TYPE s3
@@ -767,7 +816,7 @@ pipe_instructions = """
     - Nodes can't have the same exact name as the Pipe they belong to.
     - Avoid more than one node per pipe unless it is really necessary or requested by the user.
     - No indentation is allowed for property names: DESCRIPTION, NODE, SQL, TYPE, etc.
-    - Allowed TYPE values are: endpoint, copy, materialized.
+    - Allowed TYPE values are: endpoint, copy, materialized, sink.
     - Add always the output node in the TYPE section or in the last node of the pipe.
 </pipe_file_instructions>
 """
@@ -858,6 +907,7 @@ When you need to work with resources or data in cloud, add always the --cloud fl
 - When asking to create a tinybird data project, if the needed folders are not already created, use the following structure:
 ├── connections
 ├── copies
+├── sinks
 ├── datasources
 ├── endpoints
 ├── fixtures
@@ -890,9 +940,11 @@ Follow these instructions when creating or updating .pipe files:
 {pipe_example}
 {copy_pipe_instructions}
 {materialized_pipe_instructions}
+{sink_pipe_instructions}
 {connection_instructions}
 {kafka_connection_example}
 {gcs_connection_example}
+{gcs_hmac_connection_example}
 {s3_connection_example}
 </pipe_file_instructions>
 <test_file_instructions>
@@ -912,11 +964,13 @@ Follow these instructions when evolving a datasource schema:
         pipe_example=pipe_example,
         copy_pipe_instructions=copy_pipe_instructions,
         materialized_pipe_instructions=materialized_pipe_instructions,
+        sink_pipe_instructions=sink_pipe_instructions,
         test_instructions=test_instructions,
         deployment_instructions=deployment_instructions,
         connection_instructions=connection_instructions,
         kafka_connection_example=kafka_connection_example,
         gcs_connection_example=gcs_connection_example,
+        gcs_hmac_connection_example=gcs_hmac_connection_example,
         s3_connection_example=s3_connection_example,
     )
 

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev222'
-__revision__ = 'd9587d9'
+__version__ = '0.0.1.dev224'
+__revision__ = 'c632cc3'

tinybird/tb/modules/create.py

@@ -197,7 +197,17 @@ async def create(
         raise CLICreateException(FeedbackManager.error(message=str(e)))
 
 
-PROJECT_PATHS = ("datasources", "endpoints", "materializations", "copies", "pipes", "fixtures", "tests", "connections")
+PROJECT_PATHS = (
+    "datasources",
+    "endpoints",
+    "materializations",
+    "copies",
+    "sinks",
+    "pipes",
+    "fixtures",
+    "tests",
+    "connections",
+)
 
 
 def validate_project_structure(project: Project) -> bool:
@@ -248,6 +258,7 @@ def create_project_structure(folder: str):
         "endpoints         →": "Expose real-time HTTP APIs of your transformed data.",
         "materializations  →": "Stream continuous updates of the result of a pipe into a new data source.",
         "copies            →": "Capture the result of a pipe at a moment in time and write it into a target data source.",
+        "sinks             →": "Export your data to external systems on a scheduled or on-demand basis.",
         "pipes             →": "Transform your data and reuse the logic in endpoints, materializations and copies.",
         "fixtures          →": "Files with sample data for your project.",
         "tests             →": "Test your pipe files with data validation tests.",

tinybird/tb/modules/token.py

@@ -165,22 +165,20 @@ def parse_fixed_params(fixed_params_list):
 def create(ctx: Context) -> None:
     """Token creation commands.
 
-    You can create two types of tokens: JWT or Static.
+    You can create two types of tokens: Static or JWT.
+
+    * Static Tokens do not have a TTL and can have any valid scope (ADMIN, TOKENS, or ORG_DATASOURCES:READ).
 
     * JWT tokens have a TTL and can only have the PIPES:READ scope.Their main use case is allow your users to call your endpoints without exposing your API key.
 
-    * Static Tokens do not have a TTL and can have any valid scope (DATASOURCES:READ, DATASOURCES:APPEND, DATASOURCES:CREATE, DATASOURCES:DROP, PIPES:CREATE, PIPES:READ, PIPES:DROP).
 
     Examples:
 
-    tb token create jwt my_jwt_token --ttl 1h --scope PIPES:READ --resource my_pipe
-
-    tb token create static my_static_token --scope PIPES:READ --resource my_pipe
+    tb token create static my_static_token --scope ADMIN
 
-    tb token create static my_static_token --scope DATASOURCES:READ --resource my_datasource
-
-    tb token create static my_static_token --scope DATASOURCES:READ --resource my_datasource --filters "column_name=value"
+    tb token create static my_static_token --scope TOKENS
 
+    tb token create jwt my_jwt_token --ttl 1h --scope PIPES:READ --resource my_pipe
     """
 
 
@@ -248,13 +246,9 @@ async def create_jwt_token(ctx: Context, name: str, ttl: timedelta, scope, resou
 
 # Valid scopes for Static Tokens
 valid_scopes = [
-    "DATASOURCES:READ",
-    "DATASOURCES:APPEND",
-    "DATASOURCES:CREATE",
-    "DATASOURCES:DROP",
-    "PIPES:CREATE",
-    "PIPES:READ",
-    "PIPES:DROP",
+    "ADMIN",
+    "TOKENS",
+    "ORG_DATASOURCES:READ",
 ]
 
 
@@ -271,8 +265,6 @@ class DynamicOptionsCommand(click.Command):
         # Options
         dynamic_options_help += "Options:\n"
         dynamic_options_help += f"  --scope [{','.join(valid_scopes)}]   Scope for the token [Required]\n"
-        dynamic_options_help += "  --resource TEXT    Resource you want to associate the scope with\n"
-        dynamic_options_help += "  --filter TEXT  SQL condition used to filter the values when calling with this token (eg. --filter=value > 0) \n"
         dynamic_options_help += "  -h, --help            Show this message and exit.\n"
 
         return dynamic_options_help

{tinybird-0.0.1.dev222.dist-info → tinybird-0.0.1.dev224.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev222
+Version: 0.0.1.dev224
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev222.dist-info → tinybird-0.0.1.dev224.dist-info}/RECORD

@@ -3,21 +3,21 @@ tinybird/context.py,sha256=FfqYfrGX_I7PKGTQo93utaKPDNVYWelg4Hsp3evX5wM,1291
 tinybird/datatypes.py,sha256=r4WCvspmrXTJHiPjjyOTiZyZl31FO3Ynkwq4LQsYm6E,11059
 tinybird/feedback_manager.py,sha256=1INQFfRfuMCb9lfB8KNf4r6qC2khW568hoHjtk-wshI,69305
 tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
-tinybird/prompts.py,sha256=XScvI-IUOJ1gdXBdENUUCu13dHSepSnCliGXlvsKS4Q,38880
+tinybird/prompts.py,sha256=D8MpYXFrnlIf6VDeoD-uy0dGyYw0rsIiC8svj5ZGCi0,41199
 tinybird/sql.py,sha256=BufnOgclQokDyihtuXesOwHBsebN6wRXIxO5wKRkOwE,48299
 tinybird/sql_template.py,sha256=WjsTBjpQLVBHGZbY2dZuhZUurFR-rbJ_KRRy5vx4Y5E,99967
 tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
 tinybird/sql_toolset.py,sha256=M2rpLYkgV2W8NnYEYPC1tJdpy4uZHXVF64NBSKLQka4,19549
 tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
 tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
-tinybird/ch_utils/constants.py,sha256=F19-_nQ7uNmayD0T9pO1Qu7bh9MFowwj0b6ATH45vwk,4083
+tinybird/ch_utils/constants.py,sha256=yEKR11gLCL-irEXXF9QwShaR0JLXiBTlaxfolcCIoqY,4097
 tinybird/ch_utils/engine.py,sha256=X4tE9OrfaUy6kO9cqVEzyI9cDcmOF3IAssRRzsTsfEQ,40781
 tinybird/datafile/common.py,sha256=9hC1APatnfFeYXyTm_s47VmXP0JUajvpkw1vsrSXo70,91664
 tinybird/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
 tinybird/datafile/parse_connection.py,sha256=tRyn2Rpr1TeWet5BXmMoQgaotbGdYep1qiTak_OqC5E,1825
 tinybird/datafile/parse_datasource.py,sha256=ssW8QeFSgglVFi3sDZj_HgkJiTJ2069v2JgqnH3CkDE,1825
 tinybird/datafile/parse_pipe.py,sha256=xf4m0Tw44QWJzHzAm7Z7FwUoUUtr7noMYjU1NiWnX0k,3880
-tinybird/tb/__cli__.py,sha256=WyFlnks0sjbhifmcA4TRyMG2ASeWEJUqQhD97AfFN0g,247
+tinybird/tb/__cli__.py,sha256=PujnGW-G8Oj4-QEaIK25CMFn5vrp0b-YhKlTKuayFmc,247
 tinybird/tb/check_pypi.py,sha256=i3l2L8IajeB7sreikR7oPlYJki9MtS3c_M4crnmbByc,760
 tinybird/tb/cli.py,sha256=0xYk2Ip4vb3nNFbxfTdG3VoIgdRvUKVbUVU_mviErPA,1107
 tinybird/tb/client.py,sha256=FKj61vY9STPW03kfVcxYuY1_csI-kP-mc1ERQfqJtg8,56505
@@ -29,7 +29,7 @@ tinybird/tb/modules/common.py,sha256=WWvbDSQmyYUy9Xyc_BqtrdRzpDdzLuE9i_pWhoA1a8k
 tinybird/tb/modules/config.py,sha256=VnzYVUo4q1RBEEMMce4_OCrKp4erhgkRPHElydVlKj0,11488
 tinybird/tb/modules/connection.py,sha256=rdEsdcP-AqyHiW3KNoETGPeTjk7Wt3It6z_SnmInGcc,18648
 tinybird/tb/modules/copy.py,sha256=zHN1d5NA-MFsgbk2kKJq2P9qA8dNOnIsIa60QpVnSwc,4458
-tinybird/tb/modules/create.py,sha256=l3Q3QG8R-CXP-gj5rIXI3uD9tPBnQmzdklgkxt_ggNY,22106
+tinybird/tb/modules/create.py,sha256=dRpzwv1GWqu1BqZtl1-AkFbap52adA-GwY-xf0qxIcw,22262
 tinybird/tb/modules/datasource.py,sha256=4a50A_qwB-3FUEUeB3ps6tUCJvn02rMUUwW-vHaMjTw,40846
 tinybird/tb/modules/deployment.py,sha256=ByXIgEvwxB49pJEKKj0EJIfORWyflCYr04k8961nBkA,28391
 tinybird/tb/modules/deprecations.py,sha256=rrszC1f_JJeJ8mUxGoCxckQTJFBCR8wREf4XXXN-PRc,4507
@@ -58,7 +58,7 @@ tinybird/tb/modules/sink.py,sha256=bIjLjog4XH42yfp944cMnaE4il7Kl7Wn7F-nLKfH2Bc,3
 tinybird/tb/modules/table.py,sha256=4XrtjM-N0zfNtxVkbvLDQQazno1EPXnxTyo7llivfXk,11035
 tinybird/tb/modules/telemetry.py,sha256=T9gtsQffWqG_4hRBaUJPzOfMkPwz7mH-R6Bn1XRYViA,11482
 tinybird/tb/modules/test.py,sha256=XakpYi0Q2gGKItpPdtRVLKzQldkvCPqzPhwwbUxyrmc,13292
-tinybird/tb/modules/token.py,sha256=2fmKwu10_M0pqs6YmJVeILR9ZQB0ejRAET86agASbKM,13488
+tinybird/tb/modules/token.py,sha256=pSlDed5RzcadJlvIKQh22Y49gt1Rfw-0rCTko6x4y3g,12879
 tinybird/tb/modules/watch.py,sha256=HhruZoUrehlxL_nFIK3BlpHp2uyzKAM9cmNXBCa4Zgs,8965
 tinybird/tb/modules/workspace.py,sha256=ngn5yGG9K9sFaUzm0W42j_vQnm3pQS2Jh38GNgbMtP0,11607
 tinybird/tb/modules/workspace_members.py,sha256=RYLpyPM1ECCasHRg3uvpckzXplX0_KgNFsSPZn_i6qk,8744
@@ -82,8 +82,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev222.dist-info/METADATA,sha256=1kqEFMibxdfCNMu3CwE3QqoxtzhctpD7EvnRaC3b-rA,1682
-tinybird-0.0.1.dev222.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev222.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev222.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev222.dist-info/RECORD,,
+tinybird-0.0.1.dev224.dist-info/METADATA,sha256=yGFt6VbHGg2Ui5NByQrsz-kNbbjYzaPNX6qhqRRukig,1682
+tinybird-0.0.1.dev224.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev224.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev224.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev224.dist-info/RECORD,,