tinybird 0.0.1.dev304__tar.gz → 0.0.1.dev306__tar.gz

{tinybird-0.0.1.dev304 → tinybird-0.0.1.dev306}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev304
+Version: 0.0.1.dev306
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -39,6 +39,7 @@ Requires-Dist: packaging<24,>=23.1
 Requires-Dist: llm>=0.19
 Requires-Dist: thefuzz==0.22.1
 Requires-Dist: python-dotenv==1.1.0
+Requires-Dist: pyjwt[crypto]==2.9.0
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{tinybird-0.0.1.dev304 → tinybird-0.0.1.dev306}/tinybird/context.py

@@ -12,7 +12,6 @@ table_id: ContextVar[str] = ContextVar("table_id")
 hfi_frequency: ContextVar[float] = ContextVar("hfi_frequency")
 hfi_frequency_gatherer: ContextVar[float] = ContextVar("hfi_frequency_gatherer")
 use_gatherer: ContextVar[bool] = ContextVar("use_gatherer")
-allow_gatherer_fallback: ContextVar[bool] = ContextVar("allow_gatherer_fallback")
 gatherer_allow_s3_backup_on_user_errors: ContextVar[bool] = ContextVar("gatherer_allow_s3_backup_on_user_errors")
 disable_template_security_validation: ContextVar[bool] = ContextVar("disable_template_security_validation")
 origin: ContextVar[str] = ContextVar("origin")

{tinybird-0.0.1.dev304 → tinybird-0.0.1.dev306}/tinybird/prompts.py

@@ -776,6 +776,7 @@ datasource_instructions = """
     - Use always json paths to define the schema. Example: `user_id` String `json:$.user_id`,
     - Array columns are supported with a special syntax. Example: `items` Array(String) `json:$.items[:]`
     - If the datasource is using an S3 or GCS connection, they need to set IMPORT_CONNECTION_NAME, IMPORT_BUCKET_URI and IMPORT_SCHEDULE (GCS @on-demand only, S3 supports @auto too)
+    - If the datasource is using a Kafka connection, they need to set KAFKA_CONNECTION_NAME as the name of the .connection file, KAFKA_TOPIC topic_name and KAFKA_GROUP_ID as the group id for the datasource
     - Unless the user asks for them, do not include ENGINE_PARTITION_KEY and ENGINE_PRIMARY_KEY.
     - DateTime64 type without precision is not supported. Use DateTime64(3) instead.
 </datasource_file_instructions>

{tinybird-0.0.1.dev304 → tinybird-0.0.1.dev306}/tinybird/service_datasources.py

@@ -455,6 +455,79 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "feature", "type": "String"},
             ],
         },
+        {
+            "name": "tinybird.llm_usage",
+            "description": "LLM usage metrics from Tinybird AI features including token consumption, costs, and model usage for each request in the workspace.",
+            "dateColumn": "start_time",
+            "engine": {
+                "engine": "MergeTree",
+                "sorting_key": "workspace_id, start_time, user_email, request_id",
+                "partition_key": "toYYYYMM(start_time)",
+            },
+            "columns": [
+                {"name": "start_time", "type": "DateTime"},
+                {"name": "end_time", "type": "DateTime"},
+                {"name": "organization_id", "type": "String"},
+                {"name": "organization_name", "type": "String"},
+                {"name": "workspace_id", "type": "String"},
+                {"name": "workspace_name", "type": "String"},
+                {"name": "user_email", "type": "String"},
+                {"name": "request_id", "type": "String"},
+                {"name": "prompt_tokens", "type": "UInt32"},
+                {"name": "completion_tokens", "type": "UInt32"},
+                {"name": "total_tokens", "type": "UInt32"},
+                {"name": "duration", "type": "Float32"},
+                {"name": "cost", "type": "Float32"},
+                {"name": "origin", "type": "String"},
+                {"name": "feature", "type": "String"},
+            ],
+        },
+        {
+            "name": "tinybird.query_metrics",
+            "description": "Query stats metrics from your workspace.",
+            "dateColumn": "event_time",
+            "engine": {
+                "engine": "ReplacingMergeTree",
+                "sorting_key": "event_time, organization_id, query_id",
+                "partition_key": "toStartOfDay(event_time)",
+            },
+            "columns": [
+                {"name": "event_time", "type": "DateTime"},
+                {"name": "organization_id", "type": "String"},
+                {"name": "workspace_id", "type": "String"},
+                {"name": "query", "type": "String"},
+                {"name": "query_id", "type": "String"},
+                {"name": "query_type", "type": "String"},
+                {"name": "query_start_time", "type": "DateTime"},
+                {"name": "query_duration_ms", "type": "Int32"},
+                {"name": "pipe_id", "type": "String"},
+                {"name": "job_id", "type": "String"},
+                {"name": "job_kind", "type": "String"},
+                {"name": "read_rows", "type": "Int32"},
+                {"name": "read_bytes", "type": "Int32"},
+                {"name": "written_rows", "type": "Int32"},
+                {"name": "written_bytes", "type": "Int32"},
+                {"name": "memory_usage", "type": "Int32"},
+                {"name": "vcpu_time", "type": "Float32"},
+                {"name": "exception_code", "type": "Int32"},
+                {"name": "exception", "type": "String"},
+            ],
+        },
+        {
+            "name": "tinybird.vcpu_time",
+            "description": "vCPU time metrics from your workspace.",
+            "dateColumn": "minute_slot",
+            "engine": {
+                "engine": "AggregatingMergeTree",
+                "sorting_key": "organization_id, minute_slot",
+                "partition_key": "toStartOfDay(minute_slot)",
+            },
+            "columns": [
+                {"name": "minute_slot", "type": "DateTime"},
+                {"name": "organization_id", "type": "String"},
+                {"name": "vcpu_time", "type": "Float64"},
+            ],
+        },
     ]
 
 
@@ -935,6 +1008,53 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "feature", "type": "String"},
             ],
         },
+        {
+            "name": "organization.query_metrics",
+            "description": "Query stats metrics from your workspace.",
+            "dateColumn": "event_time",
+            "engine": {
+                "engine": "ReplacingMergeTree",
+                "sorting_key": "event_time, organization_id, query_id",
+                "partition_key": "toStartOfDay(event_time)",
+            },
+            "columns": [
+                {"name": "event_time", "type": "DateTime"},
+                {"name": "organization_id", "type": "String"},
+                {"name": "workspace_id", "type": "String"},
+                {"name": "query", "type": "String"},
+                {"name": "query_id", "type": "String"},
+                {"name": "query_type", "type": "String"},
+                {"name": "query_start_time", "type": "DateTime"},
+                {"name": "query_duration_ms", "type": "Int32"},
+                {"name": "pipe_id", "type": "String"},
+                {"name": "job_id", "type": "String"},
+                {"name": "job_kind", "type": "String"},
+                {"name": "read_rows", "type": "Int32"},
+                {"name": "read_bytes", "type": "Int32"},
+                {"name": "written_rows", "type": "Int32"},
+                {"name": "written_bytes", "type": "Int32"},
+                {"name": "memory_usage", "type": "Int32"},
+                {"name": "vcpu_time", "type": "Float32"},
+                {"name": "exception_code", "type": "Int32"},
+                {"name": "exception", "type": "String"},
+            ],
+        },
+        {
+            "name": "organization.vcpu_time",
+            "description": "vCPU time metrics from your workspace.",
+            "dateColumn": "minute_slot",
+            "engine": {
+                "engine": "AggregatingMergeTree",
+                "sorting_key": "organization_id, minute_slot",
+                "partition_key": "toStartOfDay(minute_slot)",
+            },
+            "columns": [
+                {"name": "minute_slot", "type": "DateTime"},
+                {"name": "organization_id", "type": "String"},
+                {"name": "workspace_id", "type": "String"},
+                {"name": "vcpu_time", "type": "Float64"},
+            ],
+        },
     ]
 
 

{tinybird-0.0.1.dev304 → tinybird-0.0.1.dev306}/tinybird/sql_template.py

@@ -2385,6 +2385,14 @@ def render_sql_template(
                 documentation="/cli/advanced-templates.html",
             )
         raise SQLTemplateException(str(e), documentation="/cli/advanced-templates.html")
+    except IndexError as e:
+        # This happens when trying to access string indices on empty strings
+        if "string index out of range" in str(e):
+            raise SQLTemplateException(
+                "String index out of range. Check that string parameters have values before accessing specific characters (e.g., param[0]). Provide default values or add length checks in your template.",
+                documentation="/cli/advanced-templates.html",
+            )
+        raise SQLTemplateException(str(e), documentation="/cli/advanced-templates.html")
     except Exception as e:
         # errors might vary here, we need to support as much as possible
         # https://gitlab.com/tinybird/analytics/-/issues/943

{tinybird-0.0.1.dev304 → tinybird-0.0.1.dev306}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev304'
-__revision__ = '9a414a1'
+__version__ = '0.0.1.dev306'
+__revision__ = 'dcef65b'

{tinybird-0.0.1.dev304 → tinybird-0.0.1.dev306}/tinybird/tb/modules/cli.py

@@ -110,7 +110,7 @@ def cli(
     prompt: Optional[str] = None,
 ) -> None:
     """
-    Use Tinybird Code to interact with your project.
+    Run just `tb` to use Tinybird Code to interact with your project.
     """
 
     # We need to unpatch for our tests not to break
@@ -211,6 +211,13 @@ def cli(
     is_prompt_mode = prompt is not None
 
     if is_agent_mode or is_prompt_mode:
+        if any(arg in sys.argv for arg in ["--cloud", "--local"]):
+            raise CLIException(
+                FeedbackManager.error(
+                    message="Tinybird Code does not support --cloud or --local flags. It will choose the correct environment based on your prompts."
+                )
+            )
+
         run_agent(config, project, dangerously_skip_permissions, prompt=prompt)
 
 

{tinybird-0.0.1.dev304 → tinybird-0.0.1.dev306}/tinybird/tb/modules/local.py

@@ -1,11 +1,15 @@
+import json
+import uuid
 from pathlib import Path
+from typing import Any
 
 import click
+import jwt
 import requests
 from docker.client import DockerClient
 
 from tinybird.tb.modules.cli import cli
-from tinybird.tb.modules.common import update_cli
+from tinybird.tb.modules.common import echo_json, update_cli
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.local_common import (
     TB_CONTAINER_NAME,
@@ -125,7 +129,21 @@ def remove() -> None:
     is_flag=True,
     help="Skip pulling the latest Tinybird Local image. Use directly your current local image.",
 )
-def start(use_aws_creds: bool, volumes_path: str, skip_new_version: bool) -> None:
+@click.option(
+    "--user-token",
+    default=None,
+    envvar="TB_LOCAL_USER_TOKEN",
+    help="User token to use for the Tinybird Local container.",
+)
+@click.option(
+    "--workspace-token",
+    default=None,
+    envvar="TB_LOCAL_WORKSPACE_TOKEN",
+    help="Workspace token to use for the Tinybird Local container.",
+)
+def start(
+    use_aws_creds: bool, volumes_path: str, skip_new_version: bool, user_token: str, workspace_token: str
+) -> None:
     """Start Tinybird Local"""
     if volumes_path is not None:
         absolute_path = Path(volumes_path).absolute()
@@ -134,7 +152,7 @@ def start(use_aws_creds: bool, volumes_path: str, skip_new_version: bool) -> Non
 
     click.echo(FeedbackManager.highlight(message="» Starting Tinybird Local..."))
     docker_client = get_docker_client()
-    start_tinybird_local(docker_client, use_aws_creds, volumes_path, skip_new_version)
+    start_tinybird_local(docker_client, use_aws_creds, volumes_path, skip_new_version, user_token, workspace_token)
     click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
 
 
@@ -183,3 +201,35 @@ def version() -> None:
     """Show Tinybird Local version"""
     response = requests.get(f"{TB_LOCAL_ADDRESS}/version")
     click.echo(FeedbackManager.success(message=f"✓ Tinybird Local version: {response.text}"))
+
+
+@local.command()
+@click.pass_context
+def generate_tokens(ctx: click.Context) -> None:
+    """Generate static tokens for initializing Tinybird Local"""
+    output = ctx.ensure_object(dict).get("output")
+    user_id = str(uuid.uuid4())
+    workspace_id = str(uuid.uuid4())
+    user_token_id = str(uuid.uuid4())
+    workspace_token_id = str(uuid.uuid4())
+    payload = {"u": user_id, "id": user_token_id, "host": None}
+    user_token = generate_token(payload)
+    payload = {"u": workspace_id, "id": workspace_token_id, "host": None}
+    workspace_token = generate_token(payload)
+
+    if output == "json":
+        echo_json({"workspace_token": workspace_token, "user_token": user_token})
+    else:
+        click.echo(FeedbackManager.gray(message="Workspace token: ") + FeedbackManager.info(message=workspace_token))
+        click.echo(FeedbackManager.gray(message="User token: ") + FeedbackManager.info(message=user_token))
+        click.echo(FeedbackManager.success(message="✓ Tinybird Local tokens generated!"))
+
+
+def generate_token(payload: dict[str, Any]) -> str:
+    algo = jwt.algorithms.get_default_algorithms()["HS256"]
+    msg = json.dumps(payload)
+    msg_base64 = jwt.utils.base64url_encode(msg.encode())
+    sign_key = algo.prepare_key("abcd")
+    signature = algo.sign(msg_base64, sign_key)
+    token = msg_base64 + b"." + jwt.utils.base64url_encode(signature)
+    return "p." + token.decode()

{tinybird-0.0.1.dev304 → tinybird-0.0.1.dev306}/tinybird/tb/modules/local_common.py

@@ -234,7 +234,9 @@ def start_tinybird_local(
     docker_client: DockerClient,
     use_aws_creds: bool,
     volumes_path: Optional[str] = None,
-    skip_new_version: bool = False,
+    skip_new_version: bool = True,
+    user_token: Optional[str] = None,
+    workspace_token: Optional[str] = None,
 ) -> None:
     """Start the Tinybird container."""
     pull_show_prompt = False
@@ -261,7 +263,13 @@ def start_tinybird_local(
         if pull_required:
             docker_client.images.pull(TB_IMAGE_NAME, platform="linux/amd64")
 
-    environment = get_use_aws_creds() if use_aws_creds else {}
+    environment = {}
+    if use_aws_creds:
+        environment.update(get_use_aws_creds())
+    if user_token:
+        environment["TB_LOCAL_USER_TOKEN"] = user_token
+    if workspace_token:
+        environment["TB_LOCAL_WORKSPACE_TOKEN"] = workspace_token
 
     container = get_existing_container_with_matching_env(docker_client, TB_CONTAINER_NAME, environment)
 

{tinybird-0.0.1.dev304 → tinybird-0.0.1.dev306}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev304
+Version: 0.0.1.dev306
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -39,6 +39,7 @@ Requires-Dist: packaging<24,>=23.1
 Requires-Dist: llm>=0.19
 Requires-Dist: thefuzz==0.22.1
 Requires-Dist: python-dotenv==1.1.0
+Requires-Dist: pyjwt[crypto]==2.9.0
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{tinybird-0.0.1.dev304 → tinybird-0.0.1.dev306}/tinybird.egg-info/requires.txt

@@ -30,3 +30,4 @@ packaging<24,>=23.1
 llm>=0.19
 thefuzz==0.22.1
 python-dotenv==1.1.0
+pyjwt[crypto]==2.9.0