tinybird 3.5.3.dev0__tar.gz → 3.5.4__tar.gz

{tinybird-3.5.3.dev0 → tinybird-3.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 3.5.3.dev0
+Version: 3.5.4
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -57,6 +57,16 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+3.5.4
+*******
+
+- `Fixed` Pulling vendored data sources from a Classic workspace to migrate to Forward now includes all workspaces in SHARED_WITH.
+
+3.5.3
+*******
+
+- `Fixed` Support `SASL_PLAINTEXT` as a Kafka security protocol, fixing connections to brokers that use SASL authentication without TLS (e.g. WarpStream).
+
 3.5.2
 *******
 

{tinybird-3.5.3.dev0 → tinybird-3.5.4}/tinybird/datafile/common.py

@@ -559,6 +559,8 @@ class Datafile:
                 raise DatafileValidationError(
                     f"Some Kafka params have been provided, but the following required ones are missing: {missing}"
                 )
+            if "kafka_group_id" in node and not str(node["kafka_group_id"]).strip():
+                raise DatafileValidationError("KAFKA_GROUP_ID cannot be empty")
             # Validate S3 params
             if any(param in node for param in BLOB_STORAGE_PARAMS):
                 if missing := [param for param in REQUIRED_BLOB_STORAGE_PARAMS if param not in node]:

{tinybird-3.5.3.dev0 → tinybird-3.5.4}/tinybird/service_datasources.py

@@ -456,33 +456,6 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "feature", "type": "String"},
             ],
         },
-        {
-            "name": "tinybird.llm_usage",
-            "description": "LLM usage metrics from Tinybird AI features including token consumption, costs, and model usage for each request in the workspace.",
-            "dateColumn": "start_time",
-            "engine": {
-                "engine": "MergeTree",
-                "sorting_key": "workspace_id, start_time, user_email, request_id",
-                "partition_key": "toYYYYMM(start_time)",
-            },
-            "columns": [
-                {"name": "start_time", "type": "DateTime"},
-                {"name": "end_time", "type": "DateTime"},
-                {"name": "organization_id", "type": "String"},
-                {"name": "organization_name", "type": "String"},
-                {"name": "workspace_id", "type": "String"},
-                {"name": "workspace_name", "type": "String"},
-                {"name": "user_email", "type": "String"},
-                {"name": "request_id", "type": "String"},
-                {"name": "prompt_tokens", "type": "UInt32"},
-                {"name": "completion_tokens", "type": "UInt32"},
-                {"name": "total_tokens", "type": "UInt32"},
-                {"name": "duration", "type": "Float32"},
-                {"name": "cost", "type": "Float32"},
-                {"name": "origin", "type": "String"},
-                {"name": "feature", "type": "String"},
-            ],
-        },
         {
             "name": "tinybird.query_validator_log",
             "description": "Log of failed queries executions in the next available ClickHouse version and their results.",
@@ -537,6 +510,26 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "rate_limited", "type": "UInt8"},
             ],
         },
+        {
+            "name": "tinybird.realtime_ingestion_mv",
+            "description": "Per-second adjusted MB/s throughput per workspace for realtime ingestion (HFI and Kafka).",
+            "dateColumn": "second_slot",
+            "engine": {
+                "engine": "AggregatingMergeTree",
+                "sorting_key": "workspace_id, datasource_id, type, second_slot",
+                "partition_key": "toYYYYMM(second_slot)",
+            },
+            "columns": [
+                {"name": "second_slot", "type": "DateTime"},
+                {"name": "type", "type": "LowCardinality(String)"},
+                {"name": "organization_id", "type": "String"},
+                {"name": "workspace_id", "type": "String"},
+                {"name": "datasource_id", "type": "String"},
+                {"name": "total_mbps", "type": "SimpleAggregateFunction(sum, Float64)"},
+                {"name": "mbs_ingestion_block", "type": "SimpleAggregateFunction(min, Float64)"},
+                {"name": "flush_time", "type": "SimpleAggregateFunction(min, Float64)"},
+            ],
+        },
     ]
 
 
@@ -1033,6 +1026,78 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "feature", "type": "String"},
             ],
         },
+        {
+            "name": "organization.backups",
+            "description": "Contains backup information for dedicated clusters in the organization.",
+            "dateColumn": "end_time",
+            "engine": {
+                "engine": "ReplacingMergeTree",
+                "sorting_key": "organization_id, cluster, end_time",
+                "partition_key": "toYYYYMM(end_time)",
+            },
+            "columns": [
+                {"name": "organization_id", "type": "String"},
+                {"name": "cluster", "type": "LowCardinality(String)"},
+                {"name": "backup_id", "type": "String"},
+                {"name": "query_id", "type": "String"},
+                {"name": "start_time", "type": "DateTime"},
+                {"name": "end_time", "type": "DateTime"},
+                {"name": "status", "type": "LowCardinality(String)"},
+                {"name": "total_size", "type": "UInt64"},
+                {"name": "compressed_size", "type": "UInt64"},
+                {"name": "uncompressed_size", "type": "UInt64"},
+                {"name": "is_full_backup", "type": "Bool"},
+            ],
+        },
+        {
+            "name": "organization.private_link",
+            "description": "Private Link costs pre-joined with organization info for observability. Filtered for AmazonVPC/VpcEndpoint and AWSELB products.",
+            "dateColumn": "timestamp",
+            "engine": {
+                "engine": "MergeTree",
+                "sorting_key": "organization_id, timestamp",
+                "partition_key": "toYYYYMM(timestamp)",
+            },
+            "columns": [
+                {"name": "timestamp", "type": "DateTime"},
+                {"name": "region", "type": "String"},
+                {"name": "organization_id", "type": "String"},
+                {"name": "organization_name", "type": "String"},
+                {"name": "usage_start_date", "type": "Nullable(DateTime)"},
+                {"name": "usage_end_date", "type": "Nullable(DateTime)"},
+                {"name": "product_code", "type": "Nullable(String)"},
+                {"name": "usage_type", "type": "Nullable(String)"},
+                {"name": "operation", "type": "Nullable(String)"},
+                {"name": "usage_amount", "type": "Nullable(Float32)"},
+                {"name": "cost", "type": "Nullable(Float32)"},
+                {"name": "blended_cost", "type": "Nullable(Float32)"},
+                {"name": "currency", "type": "Nullable(String)"},
+                {"name": "product_name", "type": "Nullable(String)"},
+                {"name": "service_name", "type": "Nullable(String)"},
+                {"name": "product_family", "type": "Nullable(String)"},
+                {"name": "type", "type": "Nullable(String)"},
+            ],
+        },
+        {
+            "name": "organization.realtime_ingestion_mv",
+            "description": "Per-second adjusted MB/s throughput per organization for realtime ingestion (HFI and Kafka).",
+            "dateColumn": "second_slot",
+            "engine": {
+                "engine": "AggregatingMergeTree",
+                "sorting_key": "organization_id, workspace_id, datasource_id, type, second_slot",
+                "partition_key": "toYYYYMM(second_slot)",
+            },
+            "columns": [
+                {"name": "second_slot", "type": "DateTime"},
+                {"name": "type", "type": "LowCardinality(String)"},
+                {"name": "organization_id", "type": "String"},
+                {"name": "workspace_id", "type": "String"},
+                {"name": "datasource_id", "type": "String"},
+                {"name": "total_mbps", "type": "SimpleAggregateFunction(sum, Float64)"},
+                {"name": "mbs_ingestion_block", "type": "SimpleAggregateFunction(min, Float64)"},
+                {"name": "flush_time", "type": "SimpleAggregateFunction(min, Float64)"},
+            ],
+        },
     ]
 
 

{tinybird-3.5.3.dev0 → tinybird-3.5.4}/tinybird/sql_toolset.py

@@ -35,7 +35,7 @@ class InvalidFunction(ValueError):
         super().__init__(self.msg)
 
 
-class InvalidResource(ValueError):
+class MissingResource(ValueError):
     def __init__(self, database: str, table: str, default_database: str = ""):
         if default_database and database == default_database:
             database = ""
@@ -88,6 +88,36 @@ class ColumnInfo:
     is_primary_key: bool = False
 
 
+@dataclass
+class MaterializedViewTarget:
+    database: Optional[str]
+    table: str
+
+
+def parse_materialized_view_target(create_table_query: str) -> Optional[MaterializedViewTarget]:
+    """
+    Extract target table info from a CREATE MATERIALIZED VIEW statement.
+
+    Parses statements like:
+        CREATE MATERIALIZED VIEW db.view TO target_db.target_table AS SELECT...
+        CREATE MATERIALIZED VIEW db.view TO `target_db`.`target_table` AS SELECT...
+        CREATE MATERIALIZED VIEW db.view TO `target_db`.`target_table` (col1 Type1, col2 Nullable(DateTime64(3))) AS SELECT...
+
+    Returns:
+        MaterializedViewTarget with database (optional) and table,
+        or None if the query is not a CREATE MATERIALIZED VIEW statement.
+    """
+    try:
+        result = chquery.parse_create_materialized_view_target_table(create_table_query)
+    except ValueError:
+        return None
+
+    return MaterializedViewTarget(
+        database=result["database"],
+        table=result["table"],
+    )
+
+
 def get_columns_from_create_query(sql_schema: str) -> list[ColumnInfo]:
     columns = chquery.get_columns_from_create_query(sql_schema)
     return [ColumnInfo(**col) for col in columns]
@@ -605,7 +635,7 @@ def replace_tables(
                             "valid_tables": valid_tables,
                         },
                     )
-                    raise InvalidResource(database, table_name, default_database=default_database)
+                    raise MissingResource(database, table_name, default_database=default_database)
 
         if current_replacements:
             # We need to transform the dictionary into something cacheable, so a sorted tuple of tuples it is

{tinybird-3.5.3.dev0 → tinybird-3.5.4}/tinybird/syncasync.py

@@ -26,6 +26,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import asyncio
 import asyncio.coroutines
 import functools
 import inspect
@@ -680,3 +681,29 @@ def sync_to_async(
         thread_sensitive=thread_sensitive,
         executor=executor,
     )
+
+
+def run_function_in_event_loop(func_def: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
+    """Run an async function, handling the case where an event loop may already be running."""
+    try:
+        asyncio.get_running_loop()
+        # If we're already in an event loop, we need to run the coroutine in a thread
+        # to avoid blocking the current event loop
+        import concurrent.futures
+
+        def run_in_new_loop() -> Any:
+            # Create a new event loop for this thread
+            new_loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(new_loop)
+            try:
+                return new_loop.run_until_complete(func_def(*args, **kwargs))
+            finally:
+                new_loop.close()
+
+        # Run in a separate thread to avoid blocking the current event loop
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            future = executor.submit(run_in_new_loop)
+            return future.result()
+    except RuntimeError:
+        # No event loop running, use asyncio.run()
+        return asyncio.run(func_def(*args, **kwargs))

{tinybird-3.5.3.dev0 → tinybird-3.5.4}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '3.5.3.dev0'
-__revision__ = 'e85d34a'
+__version__ = '3.5.4'
+__revision__ = 'f879a5d'

{tinybird-3.5.3.dev0 → tinybird-3.5.4}/tinybird/tb/client.py

@@ -373,7 +373,10 @@ class TinyB:
 
     def datasource_file(self, datasource: str):
         try:
-            return self._req(f"/v0/datasources/{datasource}.datasource")
+            include_all_shared_with = "true" if "." in datasource else "false"
+            return self._req(
+                f"/v0/datasources/{datasource}.datasource?include_all_shared_with={include_all_shared_with}"
+            )
         except DoesNotExistException:
             raise Exception(f"Data Source {datasource} not found.")
 

{tinybird-3.5.3.dev0 → tinybird-3.5.4}/tinybird/tb/modules/connection_kafka.py

@@ -115,9 +115,9 @@ def connection_create_kafka(
         except Exception as e:
             raise CLIConnectionException(FeedbackManager.error(message=str(e)))
 
-    security_protocol_options = ["SASL_SSL", "PLAINTEXT"]
+    security_protocol_options = ["SASL_SSL", "SASL_PLAINTEXT", "PLAINTEXT"]
     security_protocol = security_protocol or click.prompt(
-        FeedbackManager.highlight(message="? Security Protocol (SASL_SSL, PLAINTEXT) [SASL_SSL]"),
+        FeedbackManager.highlight(message="? Security Protocol (SASL_SSL, SASL_PLAINTEXT, PLAINTEXT) [SASL_SSL]"),
         type=click.Choice(security_protocol_options),
         show_default=False,
         show_choices=False,

{tinybird-3.5.3.dev0 → tinybird-3.5.4}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 3.5.3.dev0
+Version: 3.5.4
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -57,6 +57,16 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+3.5.4
+*******
+
+- `Fixed` Pulling vendored data sources from a Classic workspace to migrate to Forward now includes all workspaces in SHARED_WITH.
+
+3.5.3
+*******
+
+- `Fixed` Support `SASL_PLAINTEXT` as a Kafka security protocol, fixing connections to brokers that use SASL authentication without TLS (e.g. WarpStream).
+
 3.5.2
 *******
 

{tinybird-3.5.3.dev0 → tinybird-3.5.4}/tinybird.egg-info/top_level.txt

@@ -1,5 +1,7 @@
+scripts
 tests
 tests_e2e
 tests_e2e_ingestion
 tests_tb
 tinybird
+tinybird_shared