tinybird 3.2.1.dev0__tar.gz → 3.3.1.dev0__tar.gz

{tinybird-3.2.1.dev0 → tinybird-3.3.1.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 3.2.1.dev0
+Version: 3.3.1.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -57,6 +57,13 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+3.3.0
+*******
+
+- `Added` ``tb datasource stop`` and ``tb datasource start`` commands to pause/resume Kafka ingestion in forward branches
+- `Added` ``POST /v0/datasources/{name}/sample`` API endpoint to import sample data from S3/GCS connections in forward branches
+- `Changed` ``--with-connections`` flag for ``tb build`` and ``tb deploy`` is now stable (previously experimental)
+
 3.0.2
 *******
 

{tinybird-3.2.1.dev0 → tinybird-3.3.1.dev0}/tinybird/service_datasources.py

@@ -483,54 +483,6 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "feature", "type": "String"},
             ],
         },
-        {
-            "name": "tinybird.query_metrics",
-            "description": "Query stats metrics from your workspace.",
-            "dateColumn": "event_time",
-            "engine": {
-                "engine": "ReplacingMergeTree",
-                "sorting_key": "event_time, organization_id, query_id",
-                "partition_key": "toStartOfDay(event_time)",
-                "ttl": "toDate(event_time) + INTERVAL 30 DAY",
-            },
-            "columns": [
-                {"name": "event_time", "type": "DateTime"},
-                {"name": "organization_id", "type": "String"},
-                {"name": "workspace_id", "type": "String"},
-                {"name": "query", "type": "String"},
-                {"name": "query_id", "type": "String"},
-                {"name": "query_type", "type": "String"},
-                {"name": "query_start_time", "type": "DateTime"},
-                {"name": "query_duration_ms", "type": "Int32"},
-                {"name": "pipe_id", "type": "String"},
-                {"name": "job_id", "type": "String"},
-                {"name": "job_kind", "type": "String"},
-                {"name": "read_rows", "type": "Int32"},
-                {"name": "read_bytes", "type": "Int32"},
-                {"name": "written_rows", "type": "Int32"},
-                {"name": "written_bytes", "type": "Int32"},
-                {"name": "memory_usage", "type": "Int32"},
-                {"name": "vcpu_time", "type": "Float32"},
-                {"name": "exception_code", "type": "Int32"},
-                {"name": "exception", "type": "String"},
-            ],
-        },
-        {
-            "name": "tinybird.vcpu_time",
-            "description": "vCPU time metrics from your workspace.",
-            "dateColumn": "second_slot",
-            "engine": {
-                "engine": "AggregatingMergeTree",
-                "sorting_key": "organization_id, second_slot",
-                "partition_key": "toStartOfDay(second_slot)",
-                "ttl": "toDate(second_slot) + INTERVAL 30 DAY",
-            },
-            "columns": [
-                {"name": "second_slot", "type": "DateTime"},
-                {"name": "organization_id", "type": "String"},
-                {"name": "vcpu_time", "type": "Float64"},
-            ],
-        },
         {
             "name": "tinybird.query_validator_log",
             "description": "Log of failed queries executions in the next available ClickHouse version and their results.",
@@ -559,6 +511,32 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "run_validation", "type": "DateTime"},
             ],
         },
+        {
+            "name": "tinybird.hfi_requests",
+            "description": "HFI requests telemetry from your workspace.",
+            "dateColumn": "event_time",
+            "engine": {
+                "engine": "SummingMergeTree",
+                "sorting_key": "workspace_id, datasource_id, event_time",
+                "partition_key": "toYYYYMM(event_time)",
+            },
+            "columns": [
+                {"name": "event_time", "type": "DateTime"},
+                {"name": "workspace_id", "type": "String"},
+                {"name": "datasource_id", "type": "String"},
+                {"name": "successful_requests", "type": "UInt64"},
+                {"name": "error_requests", "type": "UInt64"},
+                {"name": "rows_total", "type": "UInt64"},
+                {"name": "rows_successful", "type": "UInt64"},
+                {"name": "rows_quarantined", "type": "UInt64"},
+                {"name": "bytes_received", "type": "UInt64"},
+                {"name": "bytes_decompressed", "type": "UInt64"},
+                {"name": "ch_written_rows", "type": "UInt64"},
+                {"name": "ch_written_bytes", "type": "UInt64"},
+                {"name": "ch_cpu_time", "type": "Float32"},
+                {"name": "rate_limited", "type": "UInt8"},
+            ],
+        },
     ]
 
 
@@ -1055,55 +1033,6 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "feature", "type": "String"},
             ],
         },
-        {
-            "name": "organization.query_metrics",
-            "description": "Query stats metrics from your workspace.",
-            "dateColumn": "event_time",
-            "engine": {
-                "engine": "ReplacingMergeTree",
-                "sorting_key": "event_time, organization_id, query_id",
-                "partition_key": "toStartOfDay(event_time)",
-                "ttl": "toDate(event_time) + INTERVAL 30 DAY",
-            },
-            "columns": [
-                {"name": "event_time", "type": "DateTime"},
-                {"name": "organization_id", "type": "String"},
-                {"name": "workspace_id", "type": "String"},
-                {"name": "query", "type": "String"},
-                {"name": "query_id", "type": "String"},
-                {"name": "query_type", "type": "String"},
-                {"name": "query_start_time", "type": "DateTime"},
-                {"name": "query_duration_ms", "type": "Int32"},
-                {"name": "pipe_id", "type": "String"},
-                {"name": "job_id", "type": "String"},
-                {"name": "job_kind", "type": "String"},
-                {"name": "read_rows", "type": "Int32"},
-                {"name": "read_bytes", "type": "Int32"},
-                {"name": "written_rows", "type": "Int32"},
-                {"name": "written_bytes", "type": "Int32"},
-                {"name": "memory_usage", "type": "Int32"},
-                {"name": "vcpu_time", "type": "Float32"},
-                {"name": "exception_code", "type": "Int32"},
-                {"name": "exception", "type": "String"},
-            ],
-        },
-        {
-            "name": "organization.vcpu_time",
-            "description": "vCPU time metrics from your workspace.",
-            "dateColumn": "second_slot",
-            "engine": {
-                "engine": "AggregatingMergeTree",
-                "sorting_key": "organization_id, second_slot",
-                "partition_key": "toStartOfDay(second_slot)",
-                "ttl": "toDate(second_slot) + INTERVAL 30 DAY",
-            },
-            "columns": [
-                {"name": "second_slot", "type": "DateTime"},
-                {"name": "organization_id", "type": "String"},
-                {"name": "workspace_id", "type": "String"},
-                {"name": "vcpu_time", "type": "Float64"},
-            ],
-        },
     ]
 
 

{tinybird-3.2.1.dev0 → tinybird-3.3.1.dev0}/tinybird/sql_toolset.py

@@ -539,7 +539,7 @@ def replace_tables(
                     if (
                         dependent_table[2]
                         and dependent_table[2] not in _enabled_table_functions
-                        and not (dependent_table[2] in ["cluster"] and replacement[0] == VALID_REMOTE)
+                        and not (dependent_table[2] == "cluster" and replacement[0] == VALID_REMOTE)
                     ):
                         raise InvalidFunction(table_function_name=dependent_table[2])
                     if dependent_table[0] or dependent_table[1]:

{tinybird-3.2.1.dev0 → tinybird-3.3.1.dev0}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '3.2.1.dev0'
-__revision__ = '5429eb1'
+__version__ = '3.3.1.dev0'
+__revision__ = '6508f3b'

{tinybird-3.2.1.dev0 → tinybird-3.3.1.dev0}/tinybird/tb/client.py

@@ -449,6 +449,14 @@ class TinyB:
     def datasource_truncate(self, datasource_name: str):
         return self._req(f"/v0/datasources/{datasource_name}/truncate", method="POST", data="")
 
+    def datasource_stop(self, datasource_name: str) -> dict:
+        """Stop Kafka ingestion for a datasource in a forward branch."""
+        return self._req(f"/v0/datasources/{datasource_name}/stop", method="POST", data="")
+
+    def datasource_start(self, datasource_name: str) -> dict:
+        """Start Kafka ingestion for a datasource in a forward branch."""
+        return self._req(f"/v0/datasources/{datasource_name}/start", method="POST", data="")
+
     def datasource_delete_rows(self, datasource_name: str, delete_condition: str, dry_run: bool = False):
         params = {"delete_condition": delete_condition}
         if dry_run:
@@ -483,6 +491,22 @@ class TinyB:
     def datasource_sync(self, datasource_id: str):
         return self._req(f"/v0/datasources/{datasource_id}/scheduling/runs", method="POST", data="")
 
+    def datasource_sample(self, datasource_name: str, max_files: int = 1) -> Dict[str, Any]:
+        """Start a sample import job for an S3/GCS connected datasource.
+
+        Args:
+            datasource_name: Name of the datasource to import sample data into
+            max_files: Maximum number of files to import (default 1, max 10)
+
+        Returns:
+            dict with job info including id, job_id, job_url, job, status
+        """
+        return self._req(
+            f"/v0/datasources/{datasource_name}/sample",
+            method="POST",
+            data=json.dumps({"max_files": max_files}),
+        )
+
     def datasource_scheduling_state(self, datasource_id: str):
         response = self._req(f"/v0/datasources/{datasource_id}/scheduling/state", method="GET")
         return response["state"]

{tinybird-3.2.1.dev0 → tinybird-3.3.1.dev0}/tinybird/tb/modules/build.py

@@ -30,7 +30,6 @@ from tinybird.tb.modules.watch import watch_files, watch_project
     "--with-connections",
     is_flag=True,
     default=False,
-    hidden=True,
     help="Create data linkers for connection datasources (S3, Kafka, GCS) during build",
 )
 @click.pass_context
@@ -89,7 +88,6 @@ def build(ctx: click.Context, watch: bool, with_connections: bool) -> None:
 @click.option(
     "--with-connections/--no-connections",
     default=None,
-    hidden=True,
     help="Create data linkers for connection datasources (S3, Kafka, GCS). Defaults to true for branches.",
 )
 @click.pass_context

{tinybird-3.2.1.dev0 → tinybird-3.3.1.dev0}/tinybird/tb/modules/datasource.py

@@ -19,7 +19,13 @@ from click import Context
 
 from tinybird.datafile.common import get_name_version
 from tinybird.prompts import quarantine_prompt
-from tinybird.tb.client import AuthNoTokenException, DoesNotExistException, TinyB
+from tinybird.tb.client import (
+    AuthException,
+    AuthNoTokenException,
+    DoesNotExistException,
+    OperationCanNotBePerformed,
+    TinyB,
+)
 from tinybird.tb.modules.agent.utils import (
     create_terminal_box,
 )
@@ -459,6 +465,64 @@ def datasource_truncate(ctx, datasource_name, yes, cascade):
         click.echo(FeedbackManager.info(message="Operation cancelled by user"))
 
 
+@datasource.command(name="stop")
+@click.argument("datasource_name", required=True)
+@click.pass_context
+def datasource_stop(ctx: Context, datasource_name: str) -> None:
+    """Stop Kafka ingestion for a datasource.
+
+    This command is only available for Kafka-connected datasources in forward branches
+    or tinybird local environments. Once stopped, no new data will be ingested from
+    the Kafka topic until the datasource is started again.
+
+    Example: tb datasource stop my_kafka_datasource
+    """
+    client: TinyB = ctx.obj["client"]
+    try:
+        client.datasource_stop(datasource_name)
+    except AuthNoTokenException:
+        raise
+    except DoesNotExistException:
+        raise CLIDatasourceException(FeedbackManager.error_datasource_does_not_exist(datasource=datasource_name))
+    except AuthException as e:
+        raise CLIDatasourceException(FeedbackManager.error_exception(error=e))
+    except OperationCanNotBePerformed as e:
+        raise CLIDatasourceException(FeedbackManager.error_exception(error=e))
+    except Exception as e:
+        raise CLIDatasourceException(FeedbackManager.error_exception(error=e))
+
+    click.echo(FeedbackManager.success_stop_datasource(datasource=datasource_name))
+
+
+@datasource.command(name="start")
+@click.argument("datasource_name", required=True)
+@click.pass_context
+def datasource_start(ctx: Context, datasource_name: str) -> None:
+    """Start Kafka ingestion for a datasource.
+
+    This command is only available for Kafka-connected datasources in forward branches
+    or tinybird local environments. Once started, data will be ingested from the Kafka
+    topic, resuming from the last committed offset.
+
+    Example: tb datasource start my_kafka_datasource
+    """
+    client: TinyB = ctx.obj["client"]
+    try:
+        client.datasource_start(datasource_name)
+    except AuthNoTokenException:
+        raise
+    except DoesNotExistException:
+        raise CLIDatasourceException(FeedbackManager.error_datasource_does_not_exist(datasource=datasource_name))
+    except AuthException as e:
+        raise CLIDatasourceException(FeedbackManager.error_exception(error=e))
+    except OperationCanNotBePerformed as e:
+        raise CLIDatasourceException(FeedbackManager.error_exception(error=e))
+    except Exception as e:
+        raise CLIDatasourceException(FeedbackManager.error_exception(error=e))
+
+    click.echo(FeedbackManager.success_start_datasource(datasource=datasource_name))
+
+
 @datasource.command(name="delete")
 @click.argument("datasource_name")
 @click.option("--sql-condition", default=None, help="SQL WHERE condition to remove rows", hidden=True, required=True)
@@ -646,6 +710,90 @@ def datasource_sync(ctx: Context, datasource_name: str, yes: bool):
         raise CLIDatasourceException(FeedbackManager.error_syncing_datasource(datasource=datasource_name, error=str(e)))
 
 
+@datasource.command(name="sample")
+@click.argument("datasource_name")
+@click.option(
+    "--max-files",
+    default=1,
+    type=int,
+    help="Maximum number of files to import (default 1, max 10)",
+)
+@click.option(
+    "--wait",
+    is_flag=True,
+    default=False,
+    help="Wait for the import job to finish",
+)
+@click.pass_context
+def datasource_sample(ctx: Context, datasource_name: str, max_files: int, wait: bool) -> None:
+    """Import sample data from a datasource connected to an S3 or GCS bucket.
+
+    This command only works with datasources that have IMPORT_CONNECTION_NAME and
+    IMPORT_BUCKET_URI configured (i.e., datasources ingesting from S3 or GCS).
+    It imports a limited number of files from the bucket URI pattern, useful for
+    testing data pipelines in branches without importing the entire dataset.
+
+    By default, returns immediately with job info. Use --wait to block until complete.
+
+    Examples:
+        tb --branch=my_branch datasource sample my_s3_ds
+        tb --branch=my_branch datasource sample my_s3_ds --max-files 3 --wait
+    """
+    from tinybird.tb.modules.common import wait_job
+    from tinybird.tb.modules.job_common import echo_job_url
+
+    try:
+        client: TinyB = ctx.obj["client"]
+        config = ctx.obj.get("config", {})
+
+        click.echo(FeedbackManager.info(message=f"Starting sample import for {datasource_name}..."))
+
+        # Start the job
+        result = client.datasource_sample(datasource_name, max_files=max_files)
+
+        job_id = result.get("job_id") or result.get("id")
+        if not job_id:
+            raise CLIDatasourceException(
+                FeedbackManager.error_sample_import_datasource(
+                    datasource=datasource_name, error="No job ID returned from server"
+                )
+            )
+        job_url = result.get("job_url", f"/v0/jobs/{job_id}")
+
+        # Show job URL
+        echo_job_url(client.token, client.host, config.get("name", ""), job_url)
+
+        if not wait:
+            # Return immediately with job info
+            click.echo(FeedbackManager.success(message=f"Job started: {job_id}"))
+            click.echo(FeedbackManager.gray(message=f"Check status: tb job get {job_id}"))
+            return
+
+        # Wait for job completion using existing wait_job utility
+        # wait_job raises CLIException on failure, so we only reach here on success
+        job_result = wait_job(client, job_id, job_url, "Importing sample")
+
+        stats = job_result.get("stats", {})
+        found_files = stats.get("found_files", max_files)
+        click.echo(
+            FeedbackManager.success_sample_import_datasource(
+                datasource=datasource_name,
+                file=f"{found_files} file(s)",
+                rows="see job details",
+                size="see job details",
+            )
+        )
+
+    except AuthNoTokenException:
+        raise
+    except CLIDatasourceException:
+        raise
+    except Exception as e:
+        raise CLIDatasourceException(
+            FeedbackManager.error_sample_import_datasource(datasource=datasource_name, error=str(e))
+        )
+
+
 @datasource.command(name="create")
 @click.option("--name", type=str, help="Name of the data source")
 @click.option("--blank", is_flag=True, default=False, help="Create a blank data source")

{tinybird-3.2.1.dev0 → tinybird-3.3.1.dev0}/tinybird/tb/modules/feedback_manager.py

@@ -223,6 +223,7 @@ class FeedbackManager:
         "The --incremental parameter is only supported when the `--connector` parameter is passed"
     )
     error_syncing_datasource = error_message("Failed syncing Data Source {datasource}: {error}")
+    error_sample_import_datasource = error_message("Failed importing sample for Data Source {datasource}: {error}")
     error_sync_not_supported = error_message("The --sync parameter is only supported for {valid_datasources}")
     error_invalid_connector = error_message("Invalid connector parameter: Use one of {connectors}")
     error_connector_not_configured = error_message(
@@ -1037,7 +1038,12 @@ STEP 3: ADD KEY TO SERVICE ACCOUNT
     success_update_datasource = success_message("** Data Source '{datasource}' updated:\n    {params}")
     success_promoting_datasource = success_message("** Data Source '{datasource}' connection settings updated")
     success_truncate_datasource = success_message("** Data Source '{datasource}' truncated")
+    success_stop_datasource = success_message("** Data Source '{datasource}' Kafka ingestion stopped")
+    success_start_datasource = success_message("** Data Source '{datasource}' Kafka ingestion started")
     success_sync_datasource = success_message("** Data Source '{datasource}' syncing in progress")
+    success_sample_import_datasource = success_message(
+        "** Sample imported into Data Source '{datasource}'\n   File: {file}\n   Rows: {rows}\n   Size: {size}"
+    )
     success_exchange_datasources = success_message("** Data Sources '{datasource_a}' and '{datasource_b}' exchanged")
 
     success_delete_rows_datasource = success_message(

{tinybird-3.2.1.dev0 → tinybird-3.3.1.dev0}/tinybird/tb/modules/tinyunit/tinyunit.py

@@ -263,7 +263,7 @@ def test_run_summary(results: List[TestSummaryResults], only_fail: bool = False,
             total_counts[test.status] = total_counts.get(test.status, 0) + 1
 
             # Skip the PASS tests if we only want the failed ones
-            if only_fail and test.status in [PASS]:
+            if only_fail and test.status == PASS:
                 continue
 
             summary.append(

{tinybird-3.2.1.dev0 → tinybird-3.3.1.dev0}/tinybird/tb_cli_modules/tinyunit/tinyunit.py

@@ -263,7 +263,7 @@ def test_run_summary(results: List[TestSummaryResults], only_fail: bool = False,
             total_counts[test.status] = total_counts.get(test.status, 0) + 1
 
             # Skip the PASS tests if we only want the failed ones
-            if only_fail and test.status in [PASS]:
+            if only_fail and test.status == PASS:
                 continue
 
             summary.append(

{tinybird-3.2.1.dev0 → tinybird-3.3.1.dev0}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 3.2.1.dev0
+Version: 3.3.1.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -57,6 +57,13 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+3.3.0
+*******
+
+- `Added` ``tb datasource stop`` and ``tb datasource start`` commands to pause/resume Kafka ingestion in forward branches
+- `Added` ``POST /v0/datasources/{name}/sample`` API endpoint to import sample data from S3/GCS connections in forward branches
+- `Changed` ``--with-connections`` flag for ``tb build`` and ``tb deploy`` is now stable (previously experimental)
+
 3.0.2
 *******