tinybird 0.0.1.dev148__tar.gz → 0.0.1.dev149__tar.gz

{tinybird-0.0.1.dev148 → tinybird-0.0.1.dev149}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev148
+Version: 0.0.1.dev149
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev148 → tinybird-0.0.1.dev149}/tinybird/prompts.py

@@ -401,7 +401,7 @@ Follow the instructions and generate the following response with no additional t
 def create_prompt(existing_resources: str, feedback: str = "", history: str = "") -> str:
     feedback_history = ""
     if feedback and history:
-        feedback_history = f"""In case the <feedback> and <history> tags are present and not empty, 
+        feedback_history = f"""In case the <feedback> and <history> tags are present and not empty,
 it means there was a previous attempt to generate the resources and the user provided feedback and history about previous responses.
 Use the following feedback and history to regenerate the response:
 Feedback to improve the response:
@@ -420,7 +420,9 @@ You are a Tinybird expert. You will be given a prompt to generate new or update
 {copy_pipe_instructions}
 {materialized_pipe_instructions}
 {connection_instructions}
-{connection_example}
+{kafka_connection_example}
+{gcs_connection_example}
+{s3_connection_example}
 
 {feedback_history}
 
@@ -443,14 +445,16 @@ Use the following format to generate the response and do not wrap it in any othe
         copy_pipe_instructions=copy_pipe_instructions,
         materialized_pipe_instructions=materialized_pipe_instructions,
         connection_instructions=connection_instructions,
-        connection_example=connection_example,
+        kafka_connection_example=kafka_connection_example,
+        gcs_connection_example=gcs_connection_example,
+        s3_connection_example=s3_connection_example,
         feedback_history=feedback_history,
     )
 
 
 def mock_prompt(rows: int, feedback: str = "") -> str:
     if feedback:
-        feedback = f"""In case the <feedback> tag is present and not empty, 
+        feedback = f"""In case the <feedback> tag is present and not empty,
 it means there was a previous attempt to generate the resources and the system provided feedback about the previous response.
 Use the following feedback to regenerate the response:
 <feedback>{feedback}</feedback>
@@ -515,7 +519,7 @@ SCHEMA >
     `customer_id` UInt64 `json:$.customer_id`,
     `order_date` DateTime `json:$.order_date`,
     `total_amount` Float64 `json:$.total_amount`,
-    `items` Array(String) `json:$.items[:]` // This is an array field 
+    `items` Array(String) `json:$.items[:]` // This is an array field
 
 ### Desired final output of the query:
 {{
@@ -680,19 +684,34 @@ connection_instructions = """
     - Content cannot be empty.
     - The connection names must be unique.
     - No indentation is allowed for property names
-    - We only support kafka connections for now
+    - We support kafka, gcs and s3 connections for now
 </connection_file_instructions>
 """
 
-connection_example = """
-<connection_content>
+kafka_connection_example = """
+<kafka_connection_content>
 TYPE kafka
 KAFKA_BOOTSTRAP_SERVERS {{ tb_secret("PRODUCTION_KAFKA_SERVERS", "localhost:9092") }}
 KAFKA_SECURITY_PROTOCOL SASL_SSL
 KAFKA_SASL_MECHANISM PLAIN
 KAFKA_KEY {{ tb_secret("PRODUCTION_KAFKA_USERNAME", "") }}
 KAFKA_SECRET {{ tb_secret("PRODUCTION_KAFKA_PASSWORD", "") }}
-</connection_content>
+</kafka_connection_content>
+"""
+
+gcs_connection_example = """
+<gcs_connection_content>
+TYPE gcs
+GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{ tb_secret("PRODUCTION_GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON", "") }}
+</gcs_connection_content>
+"""
+
+s3_connection_example = """
+<s3_connection_content>
+TYPE s3
+S3_REGION {{ tb_secret("PRODUCTION_S3_REGION", "") }}
+S3_ARN {{ tb_secret("PRODUCTION_S3_ARN", "") }}
+</s3_connection_content>
 """
 
 datasource_instructions = """
@@ -703,6 +722,7 @@ datasource_instructions = """
     - Use MergeTree engine by default.
     - Use AggregatingMergeTree engine when the datasource is the target of a materialized pipe.
     - Use always json paths to define the schema. Example: `user_id` String `json:$.user_id`,
+    - If the datasource is using an S3 or GCS connection, they need to set IMPORT_CONNECTION_NAME, IMPORT_BUCKET_URI and IMPORT_SCHEDULE (GCS @on-demand only, S3 supports @auto too)
 </datasource_file_instructions>
 """
 
@@ -775,7 +795,7 @@ sql_instructions = """
     {{%if not defined(end_date)%}}
     AND now()
     {{%else%}}
-    AND {{{{DateTime(end_date)}}}} 
+    AND {{{{DateTime(end_date)}}}}
     {{%end%}}
     </valid_condition_without_now>
     - Parameters must not be quoted.
@@ -862,7 +882,9 @@ Follow these instructions when creating or updating .pipe files:
 {copy_pipe_instructions}
 {materialized_pipe_instructions}
 {connection_instructions}
-{connection_example}
+{kafka_connection_example}
+{gcs_connection_example}
+{s3_connection_example}
 </pipe_file_instructions>
 <test_file_instructions>
 Follow these instructions when creating or updating .yaml files for tests:
@@ -884,7 +906,9 @@ Follow these instructions when evolving a datasource schema:
         test_instructions=test_instructions,
         deployment_instructions=deployment_instructions,
         connection_instructions=connection_instructions,
-        connection_example=connection_example,
+        kafka_connection_example=kafka_connection_example,
+        gcs_connection_example=gcs_connection_example,
+        s3_connection_example=s3_connection_example,
     )
 
 
@@ -950,7 +974,7 @@ SCHEMA >
     `payload` String `json:$.payload`
 
 FORWARD_QUERY >
-    select timestamp, toUUID(session_id) as session_id, action, version, payload 
+    select timestamp, toUUID(session_id) as session_id, action, version, payload
     </forward_query_example>
 </deployment_instruction>
 """
@@ -968,7 +992,7 @@ Current README.md file:
 - If it is not present in the current readme, generate a new ## Tinybird section with the following content:
     - ### Overview section:
         - Explaining the purpose of the project.
-    - ### Data sources section: 
+    - ### Data sources section:
         - Explaining the purpose of each datasource.
         - Add a snippet of how to ingest data into each datasource like the following (where the payload example matches the datasource schema respecting non-nullable types):
         curl -X POST "{host}/v0/events?name=events" \

{tinybird-0.0.1.dev148 → tinybird-0.0.1.dev149}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev148'
-__revision__ = '7b2789f'
+__version__ = '0.0.1.dev149'
+__revision__ = '2838b1f'

{tinybird-0.0.1.dev148 → tinybird-0.0.1.dev149}/tinybird/tb/modules/create.py

@@ -240,6 +240,11 @@ TYPE ENDPOINT
         pipes_paths = [
             Path(folder) / "endpoints" / f for f in os.listdir(Path(folder) / "endpoints") if f.endswith(".pipe")
         ]
+        connections_paths = [
+            Path(folder) / "connections" / f
+            for f in os.listdir(Path(folder) / "connections")
+            if f.endswith(".connection")
+        ]
         resources_xml = "\n".join(
             [
                 f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
@@ -254,6 +259,14 @@ TYPE ENDPOINT
                     )
                     for pipe in pipes_paths
                 ]
+                + [
+                    (
+                        "connection",
+                        conn.stem,
+                        conn.read_text(),
+                    )
+                    for conn in connections_paths
+                ]
             ]
         )
         llm = LLM(user_token=user_token, host=tb_client.host)

{tinybird-0.0.1.dev148 → tinybird-0.0.1.dev149}/tinybird/tb/modules/datafile/common.py

@@ -846,6 +846,15 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
                     lineno=line,
                     pos=pos,
                 )
+            if detected_type in ("Int", "UInt"):
+                t = detected_type
+                raise SchemaSyntaxError(
+                    message=f"Precision is mandatory for {t} types",
+                    hint=f"Hint: use one of {t}8, {t}16, {t}32, {t}64, {t}128, {t}256",
+                    lineno=line,
+                    pos=pos - len(t),
+                )
+
             try:
                 # Imported in the body to be compatible with the CLI
                 from chtoolset.query import check_compatible_types

{tinybird-0.0.1.dev148 → tinybird-0.0.1.dev149}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev148
+Version: 0.0.1.dev149
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird