tinybird 0.0.1.dev148__py3-none-any.whl → 0.0.1.dev149__py3-none-any.whl

tinybird/prompts.py

@@ -401,7 +401,7 @@ Follow the instructions and generate the following response with no additional t
 def create_prompt(existing_resources: str, feedback: str = "", history: str = "") -> str:
     feedback_history = ""
     if feedback and history:
-        feedback_history = f"""In case the <feedback> and <history> tags are present and not empty, 
+        feedback_history = f"""In case the <feedback> and <history> tags are present and not empty,
 it means there was a previous attempt to generate the resources and the user provided feedback and history about previous responses.
 Use the following feedback and history to regenerate the response:
 Feedback to improve the response:
@@ -420,7 +420,9 @@ You are a Tinybird expert. You will be given a prompt to generate new or update
 {copy_pipe_instructions}
 {materialized_pipe_instructions}
 {connection_instructions}
-{connection_example}
+{kafka_connection_example}
+{gcs_connection_example}
+{s3_connection_example}
 
 {feedback_history}
 
@@ -443,14 +445,16 @@ Use the following format to generate the response and do not wrap it in any othe
         copy_pipe_instructions=copy_pipe_instructions,
         materialized_pipe_instructions=materialized_pipe_instructions,
         connection_instructions=connection_instructions,
-        connection_example=connection_example,
+        kafka_connection_example=kafka_connection_example,
+        gcs_connection_example=gcs_connection_example,
+        s3_connection_example=s3_connection_example,
         feedback_history=feedback_history,
     )
 
 
 def mock_prompt(rows: int, feedback: str = "") -> str:
     if feedback:
-        feedback = f"""In case the <feedback> tag is present and not empty, 
+        feedback = f"""In case the <feedback> tag is present and not empty,
 it means there was a previous attempt to generate the resources and the system provided feedback about the previous response.
 Use the following feedback to regenerate the response:
 <feedback>{feedback}</feedback>
@@ -515,7 +519,7 @@ SCHEMA >
     `customer_id` UInt64 `json:$.customer_id`,
     `order_date` DateTime `json:$.order_date`,
     `total_amount` Float64 `json:$.total_amount`,
-    `items` Array(String) `json:$.items[:]` // This is an array field 
+    `items` Array(String) `json:$.items[:]` // This is an array field
 
 ### Desired final output of the query:
 {{
@@ -680,19 +684,34 @@ connection_instructions = """
     - Content cannot be empty.
     - The connection names must be unique.
     - No indentation is allowed for property names
-    - We only support kafka connections for now
+    - We support kafka, gcs and s3 connections for now
 </connection_file_instructions>
 """
 
-connection_example = """
-<connection_content>
+kafka_connection_example = """
+<kafka_connection_content>
 TYPE kafka
 KAFKA_BOOTSTRAP_SERVERS {{ tb_secret("PRODUCTION_KAFKA_SERVERS", "localhost:9092") }}
 KAFKA_SECURITY_PROTOCOL SASL_SSL
 KAFKA_SASL_MECHANISM PLAIN
 KAFKA_KEY {{ tb_secret("PRODUCTION_KAFKA_USERNAME", "") }}
 KAFKA_SECRET {{ tb_secret("PRODUCTION_KAFKA_PASSWORD", "") }}
-</connection_content>
+</kafka_connection_content>
+"""
+
+gcs_connection_example = """
+<gcs_connection_content>
+TYPE gcs
+GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{ tb_secret("PRODUCTION_GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON", "") }}
+</gcs_connection_content>
+"""
+
+s3_connection_example = """
+<s3_connection_content>
+TYPE s3
+S3_REGION {{ tb_secret("PRODUCTION_S3_REGION", "") }}
+S3_ARN {{ tb_secret("PRODUCTION_S3_ARN", "") }}
+</s3_connection_content>
 """
 
 datasource_instructions = """
@@ -703,6 +722,7 @@ datasource_instructions = """
     - Use MergeTree engine by default.
     - Use AggregatingMergeTree engine when the datasource is the target of a materialized pipe.
     - Use always json paths to define the schema. Example: `user_id` String `json:$.user_id`,
+    - If the datasource is using an S3 or GCS connection, they need to set IMPORT_CONNECTION_NAME, IMPORT_BUCKET_URI and IMPORT_SCHEDULE (GCS @on-demand only, S3 supports @auto too)
 </datasource_file_instructions>
 """
 
@@ -775,7 +795,7 @@ sql_instructions = """
     {{%if not defined(end_date)%}}
     AND now()
     {{%else%}}
-    AND {{{{DateTime(end_date)}}}} 
+    AND {{{{DateTime(end_date)}}}}
     {{%end%}}
     </valid_condition_without_now>
     - Parameters must not be quoted.
@@ -862,7 +882,9 @@ Follow these instructions when creating or updating .pipe files:
 {copy_pipe_instructions}
 {materialized_pipe_instructions}
 {connection_instructions}
-{connection_example}
+{kafka_connection_example}
+{gcs_connection_example}
+{s3_connection_example}
 </pipe_file_instructions>
 <test_file_instructions>
 Follow these instructions when creating or updating .yaml files for tests:
@@ -884,7 +906,9 @@ Follow these instructions when evolving a datasource schema:
         test_instructions=test_instructions,
         deployment_instructions=deployment_instructions,
         connection_instructions=connection_instructions,
-        connection_example=connection_example,
+        kafka_connection_example=kafka_connection_example,
+        gcs_connection_example=gcs_connection_example,
+        s3_connection_example=s3_connection_example,
     )
 
 
@@ -950,7 +974,7 @@ SCHEMA >
     `payload` String `json:$.payload`
 
 FORWARD_QUERY >
-    select timestamp, toUUID(session_id) as session_id, action, version, payload 
+    select timestamp, toUUID(session_id) as session_id, action, version, payload
     </forward_query_example>
 </deployment_instruction>
 """
@@ -968,7 +992,7 @@ Current README.md file:
 - If it is not present in the current readme, generate a new ## Tinybird section with the following content:
     - ### Overview section:
         - Explaining the purpose of the project.
-    - ### Data sources section: 
+    - ### Data sources section:
         - Explaining the purpose of each datasource.
         - Add a snippet of how to ingest data into each datasource like the following (where the payload example matches the datasource schema respecting non-nullable types):
         curl -X POST "{host}/v0/events?name=events" \

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev148'
-__revision__ = '7b2789f'
+__version__ = '0.0.1.dev149'
+__revision__ = '2838b1f'

tinybird/tb/modules/create.py

@@ -240,6 +240,11 @@ TYPE ENDPOINT
         pipes_paths = [
             Path(folder) / "endpoints" / f for f in os.listdir(Path(folder) / "endpoints") if f.endswith(".pipe")
         ]
+        connections_paths = [
+            Path(folder) / "connections" / f
+            for f in os.listdir(Path(folder) / "connections")
+            if f.endswith(".connection")
+        ]
         resources_xml = "\n".join(
             [
                 f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
@@ -254,6 +259,14 @@ TYPE ENDPOINT
                     )
                     for pipe in pipes_paths
                 ]
+                + [
+                    (
+                        "connection",
+                        conn.stem,
+                        conn.read_text(),
+                    )
+                    for conn in connections_paths
+                ]
             ]
         )
         llm = LLM(user_token=user_token, host=tb_client.host)

tinybird/tb/modules/datafile/common.py

@@ -846,6 +846,15 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
                     lineno=line,
                     pos=pos,
                 )
+            if detected_type in ("Int", "UInt"):
+                t = detected_type
+                raise SchemaSyntaxError(
+                    message=f"Precision is mandatory for {t} types",
+                    hint=f"Hint: use one of {t}8, {t}16, {t}32, {t}64, {t}128, {t}256",
+                    lineno=line,
+                    pos=pos - len(t),
+                )
+
             try:
                 # Imported in the body to be compatible with the CLI
                 from chtoolset.query import check_compatible_types

{tinybird-0.0.1.dev148.dist-info → tinybird-0.0.1.dev149.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev148
+Version: 0.0.1.dev149
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev148.dist-info → tinybird-0.0.1.dev149.dist-info}/RECORD

@@ -3,7 +3,7 @@ tinybird/context.py,sha256=FfqYfrGX_I7PKGTQo93utaKPDNVYWelg4Hsp3evX5wM,1291
 tinybird/datatypes.py,sha256=r4WCvspmrXTJHiPjjyOTiZyZl31FO3Ynkwq4LQsYm6E,11059
 tinybird/feedback_manager.py,sha256=a_ZhFX2zcB7vRknIcmHKMdQbb0c7TqlTBQ_5hPuWh88,69267
 tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
-tinybird/prompts.py,sha256=6J-dE0gOSOaIwdEwJUhEQS1Ztllz_qZnpnrv-tDoJWA,35511
+tinybird/prompts.py,sha256=yLXzHB7_P0zjriIvI9UsfnIRwmcwLSEA1tcJMzYJqjw,36456
 tinybird/sql.py,sha256=C_B81wwv3BsqyXGhF5oTk9DcTUkrp7NwIFqSzd3Dmjc,47854
 tinybird/sql_template.py,sha256=mK0yeRFctbXTAu0VNMjIzoFBJoh9PoniAVgEatA5SG4,99832
 tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
@@ -12,7 +12,7 @@ tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
 tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
 tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
 tinybird/ch_utils/engine.py,sha256=BZuPM7MFS7vaEKK5tOMR2bwSAgJudPrJt27uVEwZmTY,40512
-tinybird/tb/__cli__.py,sha256=ND-paC6OJCufE_OWkvnADMob08wWCEmSeglAIwgaUro,252
+tinybird/tb/__cli__.py,sha256=2G-RDbE_4TcARe_SbLxYFGFpjAsTqiJonlzf8czYWCI,252
 tinybird/tb/check_pypi.py,sha256=rW4QmDRbtgKdUUwJCnBkVjmTjZSZGN-XgZhx7vMkC0w,1009
 tinybird/tb/cli.py,sha256=uPV6pvi4aYVfaiGs0DQO-eoi1g9dHlrgvhutkXkdJko,1075
 tinybird/tb/client.py,sha256=aaPKq5C77e72kR7IMv9WrvnvNki8mKMOTi9EsCp0eUc,55962
@@ -25,7 +25,7 @@ tinybird/tb/modules/common.py,sha256=WzF_zRtAl3FKqbzK6yi_IjeeGIjQNu6SKnP-PX-o3Kk
 tinybird/tb/modules/config.py,sha256=ziqW_t_mRVvWOd85VoB4vKyvgMkEfpXDf9H4v38p2xc,11422
 tinybird/tb/modules/connection.py,sha256=7oOR7x4PhBcm1ETFFCH2YJ_3oeGXjAbmx1cnZX9_L70,9014
 tinybird/tb/modules/copy.py,sha256=2Mm4FWKehOG7CoOhiF1m9UZJgJn0W1_cMolqju8ONYg,5805
-tinybird/tb/modules/create.py,sha256=lj7vugRfTo6jNb02sJ9wmV6Y0ASGEbxVAd1FrmaVr58,16943
+tinybird/tb/modules/create.py,sha256=H06HmD-4CCYVMvM6doUvvKF3wU_6bBjefkwXW4ZVhE8,17378
 tinybird/tb/modules/datasource.py,sha256=V314rkpdVxVMjsp5qcSCTqDlmp4Vu--qM07BoWh-aqs,17783
 tinybird/tb/modules/deployment.py,sha256=T0lwhAT8FwZZcRnA-JDMfPUrj5MOXcQ8blxYIK2igbc,24363
 tinybird/tb/modules/deprecations.py,sha256=kugrWKWLnUL0qHXghObmzEvNIPs5F5O-kTXtYHjaSrI,3632
@@ -59,7 +59,7 @@ tinybird/tb/modules/datafile/build.py,sha256=d_h3pRFDPFrDKGhpFx2iejY25GuB2k8yfNo
 tinybird/tb/modules/datafile/build_common.py,sha256=LU24kAQmxDJIyoIapDaYG-SU3P4FrMG9UBf8m9PgVSI,4565
 tinybird/tb/modules/datafile/build_datasource.py,sha256=nXEQ0qHdq2ai7jJTv8H2d7eeDPBYzLn8VY7zMtOYb8M,17382
 tinybird/tb/modules/datafile/build_pipe.py,sha256=6Cwjf3BKEF3-oQ9PipsQfK-Z43nSwtA4qJAUoysI7Uc,11385
-tinybird/tb/modules/datafile/common.py,sha256=NIqXpsj7u8uaeSKcUZM-85Kxg35pd08DVAdbVuHmvTU,89400
+tinybird/tb/modules/datafile/common.py,sha256=1xM8R2NVmk8uQpU910Iqp_YGM1SHYoXaLZgjBYIJl54,89771
 tinybird/tb/modules/datafile/diff.py,sha256=MTmj53RYjER4neLgWVjabn-FKVFgh8h8uYiBo55lFQg,6757
 tinybird/tb/modules/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
 tinybird/tb/modules/datafile/fixture.py,sha256=DrRWivcvo_1rn7LlVUnHcXccdgx9yVj63mzBkUwCzk8,1420
@@ -79,8 +79,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev148.dist-info/METADATA,sha256=a78Hx_MhAQknWEIA9ss3ykmRIw40G6o3cXiu2bGbXSE,1612
-tinybird-0.0.1.dev148.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev148.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev148.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev148.dist-info/RECORD,,
+tinybird-0.0.1.dev149.dist-info/METADATA,sha256=pdkjsJVql6s7oNV86BGCBFOVxFzX3VDviMMeFmiq9cI,1612
+tinybird-0.0.1.dev149.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev149.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev149.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev149.dist-info/RECORD,,