tinybird 0.0.1.dev99__tar.gz → 0.0.1.dev101__tar.gz

{tinybird-0.0.1.dev99 → tinybird-0.0.1.dev101}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev99
+Version: 0.0.1.dev101
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev99 → tinybird-0.0.1.dev101}/tinybird/client.py

@@ -915,6 +915,16 @@ class TinyB:
         }
         return await self._req(f"/v1/infra?{urlencode(params)}", method="POST")
 
+    async def infra_update(self, infra_id: str, organization_id: str, name: str, host: str) -> Dict[str, Any]:
+        params = {
+            "organization_id": organization_id,
+        }
+        if name:
+            params["name"] = name
+        if host:
+            params["host"] = host
+        return await self._req(f"/v1/infra/{infra_id}?{urlencode(params)}", method="PUT")
+
     async def infra_list(self, organization_id: str) -> List[Dict[str, Any]]:
         data = await self._req(f"/v1/infra?organization_id={organization_id}")
         return data.get("infras", [])

{tinybird-0.0.1.dev99 → tinybird-0.0.1.dev101}/tinybird/sql.py

@@ -174,7 +174,7 @@ def try_to_fix_nullable_in_simple_aggregating_function(t: str) -> Optional[str]:
     return result
 
 
-def schema_to_sql_columns(schema: List[Dict[str, Any]]) -> List[str]:
+def schema_to_sql_columns(schema: List[Dict[str, Any]], skip_jsonpaths: bool = False) -> List[str]:
     """return an array with each column in SQL
     >>> schema_to_sql_columns([{'name': 'temperature', 'type': 'Float32', 'codec': None, 'default_value': None, 'nullable': False, 'normalized_name': 'temperature'}, {'name': 'temperature_delta', 'type': 'Float32', 'codec': 'CODEC(Delta(4), LZ4))', 'default_value': 'MATERIALIZED temperature', 'nullable': False, 'normalized_name': 'temperature_delta'}])
     ['`temperature` Float32', '`temperature_delta` Float32 MATERIALIZED temperature CODEC(Delta(4), LZ4))']
@@ -198,7 +198,7 @@ def schema_to_sql_columns(schema: List[Dict[str, Any]]) -> List[str]:
         else:
             _type = x["type"]
         parts = [col_name(name, backquotes=True), _type]
-        if x.get("jsonpath", None):
+        if x.get("jsonpath", None) and not skip_jsonpaths:
             parts.append(f"`json:{x['jsonpath']}`")
         if "default_value" in x and x["default_value"] not in ("", None):
             parts.append(x["default_value"])

{tinybird-0.0.1.dev99 → tinybird-0.0.1.dev101}/tinybird/sql_toolset.py

@@ -10,6 +10,9 @@ from toposort import toposort
 
 from tinybird.ch_utils.constants import COPY_ENABLED_TABLE_FUNCTIONS, ENABLED_TABLE_FUNCTIONS
 
+# VALID_REMOTE is used to explicitly vet queries sent to sql_toolset. In this module, when a table used in a query has
+# VALID_REMOTE in the database portion (as in (VALID_REMOTE, "select * from cluster(tinybird, public.t_blabla)"), the
+# query is not blocked, even if the rhs (the "select * ..." bit) is not found in the various collections of allowed tables.
 VALID_REMOTE = "VALID_REMOTE"
 
 
@@ -351,14 +354,24 @@ def is_invalid_resource(
     valid_tables: Optional[Set[Tuple[str, str]]] = None,
 ) -> bool:
     return is_invalid_resource_from_other_workspace(
-        r, database, default_database, _replaced_with
+        r, database, default_database, _replaced_with, valid_tables
     ) or is_invalid_resource_from_current_workspace(r, database, default_database, _replaced_with, valid_tables)
 
 
 def is_invalid_resource_from_other_workspace(
-    r: Tuple[str, str], database: str, default_database: str, _replaced_with: Set[Tuple[str, str]]
+    r: Tuple[str, str],
+    database: str,
+    default_database: str,
+    _replaced_with: Set[Tuple[str, str]],
+    valid_tables: Optional[Set[Tuple[str, str]]],
 ) -> bool:
-    return database not in [default_database, "tinybird", VALID_REMOTE] and r not in _replaced_with
+    # return database not in [default_database, "tinybird", VALID_REMOTE] and r not in _replaced_with
+    return bool(
+        database not in [default_database, "tinybird", VALID_REMOTE]
+        and valid_tables
+        and r not in valid_tables
+        and r not in _replaced_with
+    )
 
 
 def is_invalid_resource_from_current_workspace(

{tinybird-0.0.1.dev99 → tinybird-0.0.1.dev101}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev99'
-__revision__ = '1e30c94'
+__version__ = '0.0.1.dev101'
+__revision__ = '5dc0d4b'

{tinybird-0.0.1.dev99 → tinybird-0.0.1.dev101}/tinybird/tb/modules/datafile/common.py

@@ -183,6 +183,15 @@ class DatafileKind(Enum):
         return extension_map[extension]
 
 
+KAFKA_PARAMS = {
+    "kafka_connection_name",
+    "kafka_topic",
+    "kafka_group_id",
+}
+
+REQUIRED_KAFKA_PARAMS = KAFKA_PARAMS
+
+
 class Datafile:
     def __init__(self) -> None:
         self.maintainer: Optional[str] = None
@@ -248,9 +257,10 @@ class Datafile:
                         f"Invalid permission {token['permission']} for token {token['token_name']}. Only READ is allowed for pipes"
                     )
         elif self.kind == DatafileKind.datasource:
-            # TODO(eclbg):
             #  [x] Just one node
             #  [x] Engine is present
+            #  [x] Token permissions are valid
+            #  [x] If it's a kafka datasource, all required kafka params are present
             #  [ ] ...
             if len(self.nodes) > 1:
                 # Our users are not aware of data source data files being a single-node data file, hence this error
@@ -259,11 +269,20 @@ class Datafile:
             node = self.nodes[0]
             if "schema" not in node:
                 raise DatafileValidationError("SCHEMA is mandatory")
+            # Validate token permissions
             for token in self.tokens:
                 if token["permission"].upper() not in {"READ", "APPEND"}:
                     raise DatafileValidationError(
                         f"Invalid permission {token['permission']} for token {token['token_name']}. Only READ and APPEND are allowed for datasources"
                     )
+            # Validate Kafka params
+            if any(param in node for param in KAFKA_PARAMS) and not all(
+                param in node for param in REQUIRED_KAFKA_PARAMS
+            ):
+                missing = [param for param in REQUIRED_KAFKA_PARAMS if param not in node]
+                raise DatafileValidationError(
+                    f"Some Kafka params have been provided, but the following required ones are missing: {missing}"
+                )
         else:
             # We cannot validate a datafile whose kind is unknown
             pass
@@ -1427,6 +1446,7 @@ def parse(
         "include": include,
         "sql": sql("sql"),
         "version": version,
+        # TODO(eclbg): We should decide on a single place to define the kafka params. Definitely not here.
         "kafka_connection_name": assign_var("kafka_connection_name"),
         "kafka_topic": assign_var("kafka_topic"),
         "kafka_group_id": assign_var("kafka_group_id"),