tinybird 0.0.1.dev1__py3-none-any.whl → 0.0.1.dev3__py3-none-any.whl

tinybird/tb_cli_modules/local.py

@@ -0,0 +1,108 @@
+import time
+
+import click
+import requests
+
+import docker
+from tinybird.feedback_manager import FeedbackManager
+from tinybird.tb_cli_modules.common import CLIException
+from tinybird.tb_cli_modules.config import CLIConfig
+
+# TODO: Use the official Tinybird image once it's available 'tinybirdco/tinybird-local:latest'
+TB_IMAGE_NAME = "registry.gitlab.com/tinybird/analytics/tinybird-local-jammy-3.11:latest"
+TB_CONTAINER_NAME = "tinybird-local"
+TB_LOCAL_PORT = 80
+TB_LOCAL_HOST = f"http://localhost:{TB_LOCAL_PORT}"
+
+
+def start_tinybird_local(
+    docker_client,
+):
+    """Start the Tinybird container."""
+    containers = docker_client.containers.list(all=True, filters={"name": TB_CONTAINER_NAME})
+    if containers:
+        # Container `start` is idempotent. It's safe to call it even if the container is already running.
+        container = containers[0]
+        container.start()
+    else:
+        pull_required = False
+        try:
+            local_image = docker_client.images.get(TB_IMAGE_NAME)
+            local_image_id = local_image.attrs["RepoDigests"][0].split("@")[1]
+            remote_image = docker_client.images.get_registry_data(TB_IMAGE_NAME)
+            pull_required = local_image_id != remote_image.id
+        except Exception:
+            pull_required = True
+
+        if pull_required:
+            click.echo(
+                FeedbackManager.info(message="** Downloading latest version of Tinybird development environment...")
+            )
+            docker_client.images.pull(TB_IMAGE_NAME, platform="linux/amd64")
+
+        container = docker_client.containers.run(
+            TB_IMAGE_NAME,
+            name=TB_CONTAINER_NAME,
+            detach=True,
+            ports={"80/tcp": TB_LOCAL_PORT},
+            remove=False,
+            platform="linux/amd64",
+        )
+
+    click.echo(FeedbackManager.info(message="** Waiting for Tinybird development environment to be ready..."))
+    for attempt in range(10):
+        try:
+            run = container.exec_run("tb --no-version-warning sql 'SELECT 1 AS healthcheck' --format json").output
+            # dont parse the json as docker sometimes returns warning messages
+            # todo: rafa, make this rigth
+            if b'"healthcheck": 1' in run:
+                break
+            raise RuntimeError("Unexpected response from Tinybird")
+        except Exception:
+            if attempt == 9:  # Last attempt
+                raise CLIException("Tinybird local environment not ready yet. Please try again in a few seconds.")
+            time.sleep(5)  # Wait 5 seconds before retrying
+
+
+def get_docker_client():
+    """Check if Docker is installed and running."""
+    try:
+        client = docker.from_env()
+        client.ping()
+        return client
+    except Exception:
+        raise CLIException("Docker is not running or installed. Please ensure Docker is installed and running.")
+
+
+def stop_tinybird_local(docker_client):
+    """Stop the Tinybird container."""
+    try:
+        container = docker_client.containers.get(TB_CONTAINER_NAME)
+        container.stop()
+    except Exception:
+        pass
+
+
+def remove_tinybird_local(docker_client):
+    """Remove the Tinybird container."""
+    try:
+        container = docker_client.containers.get(TB_CONTAINER_NAME)
+        container.remove(force=True)
+    except Exception:
+        pass
+
+
+def set_up_tinybird_local(docker_client):
+    """Set up the Tinybird local environment."""
+    start_tinybird_local(docker_client)
+    return get_tinybird_local_client()
+
+
+def get_tinybird_local_client():
+    """Get a Tinybird client connected to the local environment."""
+    config = CLIConfig.get_project_config()
+    tokens = requests.get(f"{TB_LOCAL_HOST}/tokens").json()
+    token = tokens["workspace_admin_token"]
+    config.set_token(token)
+    config.set_host(TB_LOCAL_HOST)
+    return config.get_client(host=TB_LOCAL_HOST, token=token)

tinybird/tb_cli_modules/prompts.py

@@ -0,0 +1,133 @@
+create_project_prompt = """
+You are a Tinybird expert. You will be given a prompt describing a data project and you will generate all the associated datasources and pipes.
+<datasource>
+    name: The name of the datasource.
+    content: The content of the datasource datafile in the following format:
+
+    ```
+DESCRIPTION >
+    Some meaningful description of the datasource
+
+SCHEMA >
+    `<column_name_1>` <clickhouse_tinybird_compatible_data_type> `json:$.<column_name_1>`,
+    `<column_name_2>` <clickhouse_tinybird_compatible_data_type> `json:$.<column_name_2>`,
+    ...
+    `<column_name_n>` <clickhouse_tinybird_compatible_data_type> `json:$.<column_name_n>`
+
+ENGINE "MergeTree"
+ENGINE_PARTITION_KEY "<partition_key>"
+ENGINE_SORTING_KEY "<sorting_key_1, sorting_key_2, ...>"
+    ```
+</datasource>
+<pipe>
+    name: The name of the pipe.
+    content: The content of the pipe datafile in the following format:
+    ```
+DESCRIPTION >
+    Some meaningful description of the pipe
+
+NODE node_1
+SQL >
+    <sql_query_using_clickhouse_syntax_and_tinybird_templating_syntax>
+
+...
+
+NODE node_n
+SQL >
+    <sql_query_using_clickhouse_syntax_and_tinybird_templating_syntax>
+    ```
+</pipe>
+<instructions>
+    - The datasource name must be unique.
+    - The pipe name must be unique.
+    - The datasource will be the landing table for the data.
+    - Create multiple pipes to show different use cases over the same datasource.
+    - The SQL query must be a valid ClickHouse SQL query that mixes ClickHouse syntax and Tinybird templating syntax.
+    - If you need to use dynamic parameters you must start the whole sql query ALWAYS with "%" symbol on top. e.g: SQL >\n    %\n SELECT * FROM <table> WHERE <condition> LIMIT 10
+    - The Parameter functions like this one {{String(<my_param_name>,<default_value>)}} can be one of the following: String, DateTime, Date, Float32, Float64, Int, Integer, UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256
+    - Parameter names must be different from column names. Pass always the param name and a default value to the function.
+    - Nodes can't have the same exact name as the Pipe they belong to.
+</instructions>
+"""
+
+sample_data_prompt = """Your role is to generate sample data for a given data source. A data source is a table in Tinybird. You will be given a schema for the data source, and you will need to generate a set of rows that fit that schema.
+
+The schema will be provided to you at the end of this prompt in the following format:
+
+```
+SCHEMA >
+    `span_id` String `json:$.span_id`,
+    `operation_id` String `json:$.operation_name`,
+    `duration` Nullable(Float32) `json:$.duration`,
+```
+
+In the example above, `span_id` are column names, String and Float32 are column types, and `json:$.duration` is a JSON path expression that points to the duration value in the JSON data. Use this information to generate valid data.
+
+You will generate {row_count} rows of data in JSON format, newlines separated. Your reponse will only contain the {row_count} rows of data, without any other text. Do not format the data in any way, and do not place it inside any markers for a code block of any kind. Most importantly, do not use backticks on the first or last line or anywhere else in the response. This is extremely important, ensure that your response doesn't include any backticks.
+
+Remember that for a JSON to be valid, there can't be a trailing comma after the last value in an object.
+
+Make sure to include all columns in the schema, and use valid JSON. You may leave some columns empty only if the column type allows it (e.g. Nullable(String)). Use realistic values for the data, and make sure to include a variety of values in each column.
+
+For columns that represent timestamps, use values that are close to the current date and time. The current date and time is {current_datetime}"""
+
+sample_data_with_errors_prompt = """Your role is to generate incorrect sample data for a given data source. A data source is a table in Tinybird. You will be given a schema for the data source, and you will need to generate a set of rows that mostly fit that schema, but with some errors.
+
+The schema will be provided to you at the end of this prompt in the following format:
+
+```
+SCHEMA >
+    `span_id` String `json:$.span_id`,
+    `operation_id` String `json:$.operation_name`,
+    `duration` Nullable(Float32) `json:$.duration`,
+```
+
+In the example above, `span_id` are column names, String and Float32 are column types, and `json:$.duration` is a JSON path expression that points to the duration value in the JSON data. Use this information to generate valid data.
+
+You will generate {row_count} rows of data in JSON format, newlines separated. Your reponse will be a single string containing all {row_count} rows, without any other text. Make sure to include all columns in the schema, and use valid JSON. Use realistic values for the data, and make sure to include a variety of values in each column. In every set of {row_count} rows, there should be at least one row that has an error. The error could be in any column, and it could be a type error (e.g. a string where a number is expected), or it could be an error in the structure of the JSON (e.g. a value in the wrong place in the document).
+
+For columns that represent timestamps, use values that are close to the current date and time. The current date and time is {current_datetime}"""
+
+
+sample_data_sql_prompt = """
+Given the schema for a Tinybird datasource, return a can you create a clickhouse sql query to generate some random data that matches that schema.
+
+Response format MUST be just a valid clickhouse sql query.
+
+# Example input:
+
+SCHEMA >
+    experience_gained Int16 `json:$.experience_gained`,
+    level Int16 `json:$.level`,
+    monster_kills Int16 `json:$.monster_kills`,
+    player_id String `json:$.player_id`,
+    pvp_kills Int16 `json:$.pvp_kills`,
+    quest_completions Int16 `json:$.quest_completions`,
+    timestamp DateTime `json:$.timestamp`
+
+
+# Example output:
+
+SELECT
+    rand() % 1000 AS experience_gained, -- Random experience gained between 0 and 999
+    1 + rand() % 100 AS level,          -- Random level between 1 and 100
+    rand() % 500 AS monster_kills,      -- Random monster kills between 0 and 499
+    concat('player_', toString(rand() % 10000)) AS player_id, -- Random player IDs like "player_1234"
+    rand() % 50 AS pvp_kills,           -- Random PvP kills between 0 and 49
+    rand() % 200 AS quest_completions,  -- Random quest completions between 0 and 199
+    now() - rand() % 86400 AS timestamp -- Random timestamp within the last day
+FROM numbers({row_count})
+
+# Instructions:
+
+- The query MUST return a random sample of data that matches the schema.
+- The query MUST return a valid clickhouse sql query.
+- The query MUST return a sample of {row_count} rows.
+- The query MUST be valid for clickhouse and Tinybird.
+- Return JUST the sql query, without any other text or symbols. 
+- Do NOT include ```clickhouse or ```sql or any other wrapping text.
+- Do NOT use any of these functions: elementAt
+- Do NOT add a semicolon at the end of the query
+- Do NOT add any FORMAT at the end of the query, because it will be added later by Tinybird.
+
+"""

{tinybird-0.0.1.dev1.dist-info → tinybird-0.0.1.dev3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird
-Version: 0.0.1.dev1
+Version: 0.0.1.dev3
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -13,6 +13,7 @@ Requires-Dist: clickhouse-toolset (==0.33.dev0)
 Requires-Dist: colorama (==0.4.6)
 Requires-Dist: cryptography (~=41.0.0)
 Requires-Dist: croniter (==1.3.8)
+Requires-Dist: docker (==7.1.0)
 Requires-Dist: GitPython (~=3.1.32)
 Requires-Dist: humanfriendly (~=8.2)
 Requires-Dist: pydantic (~=2.8.0)
@@ -24,8 +25,10 @@ Requires-Dist: shandy-sqlfmt[jinjafmt] (==0.11.1)
 Requires-Dist: toposort (==1.10)
 Requires-Dist: tornado (~=6.0.0)
 Requires-Dist: urllib3 (<2,>=1.26.14)
+Requires-Dist: watchdog (==6.0.0)
 Requires-Dist: wheel
 Requires-Dist: packaging (<24,>=23.1)
+Requires-Dist: llm (>=0.19)
 Provides-Extra: bigquery
 Requires-Dist: gsutil (==4.58) ; extra == 'bigquery'
 Requires-Dist: google-api-python-client (==2.0.2) ; extra == 'bigquery'

{tinybird-0.0.1.dev1.dist-info → tinybird-0.0.1.dev3.dist-info}/RECORD

@@ -1,34 +1,39 @@
-tinybird/__cli__.py,sha256=jC1bu854XUtvxWMNIEZCtuD07GS-GwIOqUhT4zqkeYo,250
+tinybird/__cli__.py,sha256=bgGjpK7NkAkSj1v-IXZIG9nlQnRLi4faLhm9wfdnePo,250
 tinybird/check_pypi.py,sha256=_4NkharLyR_ELrAdit-ftqIWvOf7jZNPt3i76frlo9g,975
-tinybird/client.py,sha256=RirJVGZxLKH6yk7m_pnOaYsYoX_zTqXffdr2sk39_fk,50511
-tinybird/config.py,sha256=NXro9Itgex3QaHbC2InnqyCluc6bMqIwZIJ0xh1wdnw,6147
+tinybird/client.py,sha256=nd97gD2-8Ap8yDonBcVwk9eXDAL43hmIYdo-Pse43RE,50738
+tinybird/config.py,sha256=Z-BX9FrjgsLw1YwcCdF0IztLB97Zpc70VVPplO_pDSY,6089
 tinybird/connectors.py,sha256=lkpVSUmSuViEZBa4QjTK7YmPHUop0a5UFoTrSmlVq6k,15244
 tinybird/context.py,sha256=kutUQ0kCwparowI74_YLXx6wtTzGLRouJ6oGHVBPzBo,1291
-tinybird/datafile.py,sha256=b1aqFb2KJKlwey7S-zYOnA3qeik2pp7FQa2AeqfJX4A,229308
+tinybird/datafile.py,sha256=JpZX33Ilq71PiuDBs4TNF1xJoUMtTPLynIbHuo7Bn80,253510
 tinybird/datatypes.py,sha256=IHyhZ86ib54Vnd1pbod9y2aS8DDvDKZm1HJGlThdbuQ,10460
-tinybird/feedback_manager.py,sha256=cNUbt0Jxim02UiIdlyP12DJfXfFFzxDCfJK9XRWZ9A0,67488
+tinybird/feedback_manager.py,sha256=FT5wJ06gYPGofVgpR-XdZBxUlV9oV94rLx-vzh9ArB8,67724
 tinybird/git_settings.py,sha256=XUL9ZUj59-ZVQJDYmMEq4UpnuuOuQOHGlNcX3JgQHjQ,3954
 tinybird/sql.py,sha256=gfRKjdqEygcE1WOTeQ1QV2Jal8Jzl4RSX8fftu1KSEs,45825
 tinybird/sql_template.py,sha256=IqYRfUxDYBCoOYjqqvn--_8QXLv9FSRnJ0bInx7q1Xs,93051
 tinybird/sql_template_fmt.py,sha256=1z-PuqSZXtzso8Z_mPqUc-NxIxUrNUcVIPezNieZk-M,10196
 tinybird/sql_toolset.py,sha256=xS_yD5N_TZT5d4uPcXdeIYX4GQPz7-7wHywMGdfgqcM,13794
 tinybird/syncasync.py,sha256=fAvq0qkRgqXqXMKwbY2iJNYqLT_r6mDsh1MRpGKrdRU,27763
-tinybird/tb_cli.py,sha256=q1LGAsBVVMJsjR2HK62Pu6vpVtLzNmH8wHrEVUUdVkU,744
+tinybird/tb_cli.py,sha256=cTPJfd_wOWPCwwFz_LoMREV3YSa8pltU_y-ITP13IpY,819
 tinybird/tornado_template.py,sha256=o2HguxrL1Evnt8o3IvrsI8Zm6JtRQ3zhLJKf1XyR3SQ,41965
 tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
 tinybird/ch_utils/engine.py,sha256=OXkBhlzGjZotjD0vaT-rFIbSGV4tpiHxE8qO_ip0SyQ,40454
 tinybird/tb_cli_modules/auth.py,sha256=cqxfGgFheuTmenQ3UwPBXTqwMm8JD7uzgLfoIRXdnyQ,9038
 tinybird/tb_cli_modules/branch.py,sha256=Ik8rRVPXvhyChHBqdPVNtI_C-0gLYjUHaznNWE04Ecw,39120
+tinybird/tb_cli_modules/build.py,sha256=pF6Ts2k0qCwCi1ktrGyJo1NcdWsU9O3okDdb3SiFdZo,7615
 tinybird/tb_cli_modules/cicd.py,sha256=0lMkb6CVOFZl5HOwgY8mK4T4mgI7O8335UngLXtCc-c,13851
-tinybird/tb_cli_modules/cli.py,sha256=KNEAePyQcSiWqKMbqh1JZNIknRTbrsomI2UHvT56UuY,62042
-tinybird/tb_cli_modules/common.py,sha256=JJwkLUcTbTonf1NWe4tUeenQGMr82d2152hnSs6k6Zc,78706
+tinybird/tb_cli_modules/cli.py,sha256=a738xz4DUNep_mzHTEUExd0NAuf7cHkUAlMsEus9xaM,65072
+tinybird/tb_cli_modules/common.py,sha256=pS7kisPe4vChirSY2og5e0Cu-RNDow7X3Qdk1BEHL78,78571
 tinybird/tb_cli_modules/config.py,sha256=6NTgIdwf0X132A1j6G_YrdPep87ymZ9b5pABabKLzh4,11484
 tinybird/tb_cli_modules/connection.py,sha256=YggP34Qh3hCjD41lp1ZHVCwHPjI_-um2spvEV8F_tSU,28684
-tinybird/tb_cli_modules/datasource.py,sha256=kApfys2V_JXSzzEZzXRpZxZ8CKeGR4aiEOQRr_U8tVQ,35747
+tinybird/tb_cli_modules/create.py,sha256=wcr-GIgCz_t135UIDFt_gNxUQ0huI3oXGM5bIuC665E,7808
+tinybird/tb_cli_modules/datasource.py,sha256=BVYwPkKvdnEv2YMkxof7n7FOffyJm-QK7Pk9Ui-MC3A,35816
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/fmt.py,sha256=_xL2AyGo3us7NpmfAXtSIhn1cGAC-A4PspdhUjm58jY,3382
 tinybird/tb_cli_modules/job.py,sha256=AG69LPb9MbobA1awwJFZJvxqarDKfRlsBjw2V1zvYqc,2964
+tinybird/tb_cli_modules/llm.py,sha256=RyUBtyLJcG5DFBjkalFLgwwX-415_Uom8Hq_oKiiyBs,904
+tinybird/tb_cli_modules/local.py,sha256=uamCRYoU0MhCaPUXtvCFfbzHJGwTaPMTTHRQyw0Da3k,3870
 tinybird/tb_cli_modules/pipe.py,sha256=BCLAQ3ZuWKGAih2mupnw_Y2S5B5cNS-epF317whsSEE,30989
+tinybird/tb_cli_modules/prompts.py,sha256=Esjhet-KUM1bT6RStk2voCCVjvAOUD4hDDiKpn-AYNY,7227
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/tag.py,sha256=9YHnruPnUNp1IJUe4qcSEMg9EbquIRo--Nxcsbvkvq8,3488
 tinybird/tb_cli_modules/telemetry.py,sha256=iEGnMuCuNhvF6ln__j6X9MSTwL_0Hm-GgFHHHvhfknk,10466
@@ -38,8 +43,8 @@ tinybird/tb_cli_modules/workspace.py,sha256=N8f1dl4BYc34ucmJ6oNZc4XZMGKd8m0Fhq7o
 tinybird/tb_cli_modules/workspace_members.py,sha256=ksXsjd233y9-sNlz4Qb-meZbX4zn1B84e_bSm2i8rhg,8731
 tinybird/tb_cli_modules/tinyunit/tinyunit.py,sha256=_ydOD4WxmKy7_h-d3fG62w_0_lD0uLl9w4EbEYtwd0U,11720
 tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py,sha256=hGh1ZaXC1af7rKnX7222urkj0QJMhMWclqMy59dOqwE,1922
-tinybird-0.0.1.dev1.dist-info/METADATA,sha256=QxrxshZUJhYcvHshI7jn3KrE0YAZsbThrL-WbYkdtL0,2310
-tinybird-0.0.1.dev1.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-tinybird-0.0.1.dev1.dist-info/entry_points.txt,sha256=PKPKuPmA4IfJYnCFHHUiw-aAWZuBomFvwCklv1OyCjE,43
-tinybird-0.0.1.dev1.dist-info/top_level.txt,sha256=pgw6AzERHBcW3YTi2PW4arjxLkulk2msOz_SomfOEuc,45
-tinybird-0.0.1.dev1.dist-info/RECORD,,
+tinybird-0.0.1.dev3.dist-info/METADATA,sha256=6J__4w3BceyDNH4mybjldWRwFGgNaDLSyvugF9Se_E4,2404
+tinybird-0.0.1.dev3.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+tinybird-0.0.1.dev3.dist-info/entry_points.txt,sha256=PKPKuPmA4IfJYnCFHHUiw-aAWZuBomFvwCklv1OyCjE,43
+tinybird-0.0.1.dev3.dist-info/top_level.txt,sha256=pgw6AzERHBcW3YTi2PW4arjxLkulk2msOz_SomfOEuc,45
+tinybird-0.0.1.dev3.dist-info/RECORD,,