tinybird 0.0.1.dev223__tar.gz → 0.0.1.dev224__tar.gz

{tinybird-0.0.1.dev223 → tinybird-0.0.1.dev224}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev223
+Version: 0.0.1.dev224
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev223 → tinybird-0.0.1.dev224}/tinybird/prompts.py

@@ -420,9 +420,11 @@ You are a Tinybird expert. You will be given a prompt to generate new or update
 {pipe_example}
 {copy_pipe_instructions}
 {materialized_pipe_instructions}
+{sink_pipe_instructions}
 {connection_instructions}
 {kafka_connection_example}
 {gcs_connection_example}
+{gcs_hmac_connection_example}
 {s3_connection_example}
 
 {feedback_history}
@@ -445,9 +447,11 @@ Use the following format to generate the response and do not wrap it in any othe
         pipe_example=pipe_example,
         copy_pipe_instructions=copy_pipe_instructions,
         materialized_pipe_instructions=materialized_pipe_instructions,
+        sink_pipe_instructions=sink_pipe_instructions,
         connection_instructions=connection_instructions,
         kafka_connection_example=kafka_connection_example,
         gcs_connection_example=gcs_connection_example,
+        gcs_hmac_connection_example=gcs_hmac_connection_example,
         s3_connection_example=s3_connection_example,
         feedback_history=feedback_history,
     )
@@ -639,6 +643,43 @@ COPY_SCHEDULE 0 * * * *
 </copy_pipe_instructions>
 """
 
+sink_pipe_instructions = """
+<sink_pipe_instructions>
+- Do not create sink pipes by default, unless the user asks for it.
+- Sink pipes should be created in the /sinks folder.
+- In a .pipe file you can define how to export the result of a Pipe to an external system, optionally with a schedule.
+- Valid external systems are Kafka, S3, GCS.
+- Sink pipes depend on a connection, if no connection is provided, search for an existing connection that suits the request. If none, create a new connection.
+- Do not include EXPORT_SCHEDULE in the .pipe file unless is specifically requested by the user.
+- EXPORT_SCHEDULE is a cron expression that defines the schedule of the sink pipe.
+- EXPORT_SCHEDULE is optional and if not provided, the sink pipe will be executed only once.
+- EXPORT_CONNECTION_NAME is the name of the connection used to export.
+- TYPE SINK is the type of the pipe and it is mandatory for sink pipes.
+- If the sink pipe uses parameters, you must include the % character and a newline on top of every query to be able to use the parameters.
+- The content of the .pipe file must follow this format:
+DESCRIPTION Sink Pipe to export sales hour every hour using my_connection
+
+NODE daily_sales
+SQL >
+    %
+    SELECT toStartOfDay(starting_date) day, country, sum(sales) as total_sales
+    FROM teams
+    WHERE
+    day BETWEEN toStartOfDay(now()) - interval 1 day AND toStartOfDay(now())
+    and country = {{ String(country, 'US')}}
+    GROUP BY day, country
+
+TYPE sink
+EXPORT_CONNECTION_NAME "my_connection"
+EXPORT_BUCKET_URI "s3://tinybird-sinks"
+EXPORT_FILE_TEMPLATE "daily_prices"
+EXPORT_SCHEDULE "*/5 * * * *"
+EXPORT_FORMAT "csv"
+EXPORT_COMPRESSION "gz"
+EXPORT_WRITE_STRATEGY "truncate"
+</sink_pipe_instructions>
+"""
+
 materialized_pipe_instructions = """
 <materialized_pipe_instructions>
 - Do not create materialized pipes by default, unless the user asks for it.
@@ -709,6 +750,14 @@ GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{ tb_secret("PRODUCTION_GCS_SERVICE_ACCOUN
 </gcs_connection_content>
 """
 
+gcs_hmac_connection_example = """
+<gcs_hmac_connection_content>
+TYPE gcs
+GCS_HMAC_ACCESS_ID {{ tb_secret("gcs_hmac_access_id") }}
+GCS_HMAC_SECRET {{ tb_secret("gcs_hmac_secret") }}
+</gcs_hmac_connection_content>
+"""
+
 s3_connection_example = """
 <s3_connection_content>
 TYPE s3
@@ -767,7 +816,7 @@ pipe_instructions = """
     - Nodes can't have the same exact name as the Pipe they belong to.
     - Avoid more than one node per pipe unless it is really necessary or requested by the user.
     - No indentation is allowed for property names: DESCRIPTION, NODE, SQL, TYPE, etc.
-    - Allowed TYPE values are: endpoint, copy, materialized.
+    - Allowed TYPE values are: endpoint, copy, materialized, sink.
     - Add always the output node in the TYPE section or in the last node of the pipe.
 </pipe_file_instructions>
 """
@@ -858,6 +907,7 @@ When you need to work with resources or data in cloud, add always the --cloud fl
 - When asking to create a tinybird data project, if the needed folders are not already created, use the following structure:
 ├── connections
 ├── copies
+├── sinks
 ├── datasources
 ├── endpoints
 ├── fixtures
@@ -890,9 +940,11 @@ Follow these instructions when creating or updating .pipe files:
 {pipe_example}
 {copy_pipe_instructions}
 {materialized_pipe_instructions}
+{sink_pipe_instructions}
 {connection_instructions}
 {kafka_connection_example}
 {gcs_connection_example}
+{gcs_hmac_connection_example}
 {s3_connection_example}
 </pipe_file_instructions>
 <test_file_instructions>
@@ -912,11 +964,13 @@ Follow these instructions when evolving a datasource schema:
         pipe_example=pipe_example,
         copy_pipe_instructions=copy_pipe_instructions,
         materialized_pipe_instructions=materialized_pipe_instructions,
+        sink_pipe_instructions=sink_pipe_instructions,
         test_instructions=test_instructions,
         deployment_instructions=deployment_instructions,
         connection_instructions=connection_instructions,
         kafka_connection_example=kafka_connection_example,
         gcs_connection_example=gcs_connection_example,
+        gcs_hmac_connection_example=gcs_hmac_connection_example,
         s3_connection_example=s3_connection_example,
     )
 

{tinybird-0.0.1.dev223 → tinybird-0.0.1.dev224}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev223'
-__revision__ = '3fd90ce'
+__version__ = '0.0.1.dev224'
+__revision__ = 'c632cc3'

{tinybird-0.0.1.dev223 → tinybird-0.0.1.dev224}/tinybird/tb/modules/create.py

@@ -197,7 +197,17 @@ async def create(
         raise CLICreateException(FeedbackManager.error(message=str(e)))
 
 
-PROJECT_PATHS = ("datasources", "endpoints", "materializations", "copies", "pipes", "fixtures", "tests", "connections")
+PROJECT_PATHS = (
+    "datasources",
+    "endpoints",
+    "materializations",
+    "copies",
+    "sinks",
+    "pipes",
+    "fixtures",
+    "tests",
+    "connections",
+)
 
 
 def validate_project_structure(project: Project) -> bool:
@@ -248,6 +258,7 @@ def create_project_structure(folder: str):
         "endpoints         →": "Expose real-time HTTP APIs of your transformed data.",
         "materializations  →": "Stream continuous updates of the result of a pipe into a new data source.",
         "copies            →": "Capture the result of a pipe at a moment in time and write it into a target data source.",
+        "sinks             →": "Export your data to external systems on a scheduled or on-demand basis.",
         "pipes             →": "Transform your data and reuse the logic in endpoints, materializations and copies.",
         "fixtures          →": "Files with sample data for your project.",
         "tests             →": "Test your pipe files with data validation tests.",

{tinybird-0.0.1.dev223 → tinybird-0.0.1.dev224}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev223
+Version: 0.0.1.dev224
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird