tinybird 0.0.1.dev100__py3-none-any.whl → 0.0.1.dev101__py3-none-any.whl

tinybird/client.py

@@ -915,6 +915,16 @@ class TinyB:
         }
         return await self._req(f"/v1/infra?{urlencode(params)}", method="POST")
 
+    async def infra_update(self, infra_id: str, organization_id: str, name: str, host: str) -> Dict[str, Any]:
+        params = {
+            "organization_id": organization_id,
+        }
+        if name:
+            params["name"] = name
+        if host:
+            params["host"] = host
+        return await self._req(f"/v1/infra/{infra_id}?{urlencode(params)}", method="PUT")
+
     async def infra_list(self, organization_id: str) -> List[Dict[str, Any]]:
         data = await self._req(f"/v1/infra?organization_id={organization_id}")
         return data.get("infras", [])

tinybird/sql.py

@@ -174,7 +174,7 @@ def try_to_fix_nullable_in_simple_aggregating_function(t: str) -> Optional[str]:
     return result
 
 
-def schema_to_sql_columns(schema: List[Dict[str, Any]]) -> List[str]:
+def schema_to_sql_columns(schema: List[Dict[str, Any]], skip_jsonpaths: bool = False) -> List[str]:
     """return an array with each column in SQL
     >>> schema_to_sql_columns([{'name': 'temperature', 'type': 'Float32', 'codec': None, 'default_value': None, 'nullable': False, 'normalized_name': 'temperature'}, {'name': 'temperature_delta', 'type': 'Float32', 'codec': 'CODEC(Delta(4), LZ4))', 'default_value': 'MATERIALIZED temperature', 'nullable': False, 'normalized_name': 'temperature_delta'}])
     ['`temperature` Float32', '`temperature_delta` Float32 MATERIALIZED temperature CODEC(Delta(4), LZ4))']
@@ -198,7 +198,7 @@ def schema_to_sql_columns(schema: List[Dict[str, Any]]) -> List[str]:
         else:
             _type = x["type"]
         parts = [col_name(name, backquotes=True), _type]
-        if x.get("jsonpath", None):
+        if x.get("jsonpath", None) and not skip_jsonpaths:
             parts.append(f"`json:{x['jsonpath']}`")
         if "default_value" in x and x["default_value"] not in ("", None):
             parts.append(x["default_value"])

tinybird/sql_toolset.py

@@ -10,6 +10,9 @@ from toposort import toposort
 
 from tinybird.ch_utils.constants import COPY_ENABLED_TABLE_FUNCTIONS, ENABLED_TABLE_FUNCTIONS
 
+# VALID_REMOTE is used to explicitly vet queries sent to sql_toolset. In this module, when a table used in a query has
+# VALID_REMOTE in the database portion (as in (VALID_REMOTE, "select * from cluster(tinybird, public.t_blabla)"), the
+# query is not blocked, even if the rhs (the "select * ..." bit) is not found in the various collections of allowed tables.
 VALID_REMOTE = "VALID_REMOTE"
 
 
@@ -351,14 +354,24 @@ def is_invalid_resource(
     valid_tables: Optional[Set[Tuple[str, str]]] = None,
 ) -> bool:
     return is_invalid_resource_from_other_workspace(
-        r, database, default_database, _replaced_with
+        r, database, default_database, _replaced_with, valid_tables
     ) or is_invalid_resource_from_current_workspace(r, database, default_database, _replaced_with, valid_tables)
 
 
 def is_invalid_resource_from_other_workspace(
-    r: Tuple[str, str], database: str, default_database: str, _replaced_with: Set[Tuple[str, str]]
+    r: Tuple[str, str],
+    database: str,
+    default_database: str,
+    _replaced_with: Set[Tuple[str, str]],
+    valid_tables: Optional[Set[Tuple[str, str]]],
 ) -> bool:
-    return database not in [default_database, "tinybird", VALID_REMOTE] and r not in _replaced_with
+    # return database not in [default_database, "tinybird", VALID_REMOTE] and r not in _replaced_with
+    return bool(
+        database not in [default_database, "tinybird", VALID_REMOTE]
+        and valid_tables
+        and r not in valid_tables
+        and r not in _replaced_with
+    )
 
 
 def is_invalid_resource_from_current_workspace(

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev100'
-__revision__ = 'ccb1518'
+__version__ = '0.0.1.dev101'
+__revision__ = '5dc0d4b'

tinybird/tb/modules/datafile/common.py

@@ -183,6 +183,15 @@ class DatafileKind(Enum):
         return extension_map[extension]
 
 
+KAFKA_PARAMS = {
+    "kafka_connection_name",
+    "kafka_topic",
+    "kafka_group_id",
+}
+
+REQUIRED_KAFKA_PARAMS = KAFKA_PARAMS
+
+
 class Datafile:
     def __init__(self) -> None:
         self.maintainer: Optional[str] = None
@@ -248,9 +257,10 @@ class Datafile:
                         f"Invalid permission {token['permission']} for token {token['token_name']}. Only READ is allowed for pipes"
                     )
         elif self.kind == DatafileKind.datasource:
-            # TODO(eclbg):
             #  [x] Just one node
             #  [x] Engine is present
+            #  [x] Token permissions are valid
+            #  [x] If it's a kafka datasource, all required kafka params are present
             #  [ ] ...
             if len(self.nodes) > 1:
                 # Our users are not aware of data source data files being a single-node data file, hence this error
@@ -259,11 +269,20 @@ class Datafile:
             node = self.nodes[0]
             if "schema" not in node:
                 raise DatafileValidationError("SCHEMA is mandatory")
+            # Validate token permissions
             for token in self.tokens:
                 if token["permission"].upper() not in {"READ", "APPEND"}:
                     raise DatafileValidationError(
                         f"Invalid permission {token['permission']} for token {token['token_name']}. Only READ and APPEND are allowed for datasources"
                     )
+            # Validate Kafka params
+            if any(param in node for param in KAFKA_PARAMS) and not all(
+                param in node for param in REQUIRED_KAFKA_PARAMS
+            ):
+                missing = [param for param in REQUIRED_KAFKA_PARAMS if param not in node]
+                raise DatafileValidationError(
+                    f"Some Kafka params have been provided, but the following required ones are missing: {missing}"
+                )
         else:
             # We cannot validate a datafile whose kind is unknown
             pass
@@ -1427,6 +1446,7 @@ def parse(
         "include": include,
         "sql": sql("sql"),
         "version": version,
+        # TODO(eclbg): We should decide on a single place to define the kafka params. Definitely not here.
         "kafka_connection_name": assign_var("kafka_connection_name"),
         "kafka_topic": assign_var("kafka_topic"),
         "kafka_group_id": assign_var("kafka_group_id"),

tinybird/tb/modules/infra.py

@@ -6,6 +6,7 @@ from pathlib import Path
 from typing import Optional
 
 import click
+import pyperclip
 import requests
 from click import Context
 
@@ -217,11 +218,11 @@ def infra(ctx: Context) -> None:
 
 @infra.command(name="init")
 @click.option("--name", type=str, help="Name for identifying the self-managed infrastructure in Tinybird")
-@click.option("--provider", type=str, help="Infrastructure provider (aws, gcp, azure)")
-@click.option("--region", type=str, help="AWS region (for AWS provider)")
+@click.option("--provider", type=str, help="Infrastructure provider. Possible values are: aws, gcp, azure)")
+@click.option("--region", type=str, help="AWS region, when using aws as the provider")
 @click.option("--dns-zone-name", type=str, help="DNS zone name")
-@click.option("--namespace", type=str, help="Kubernetes namespace for deployment")
-@click.option("--dns-record", type=str, help="DNS record name to create (without domain, e.g. 'tinybird')")
+@click.option("--namespace", type=str, help="Kubernetes namespace for the deployment")
+@click.option("--dns-record", type=str, help="DNS record name to create, without domain. For example, 'tinybird')")
 @click.option("--storage-class", type=str, help="Storage class for the k8s StatefulSet")
 @click.option(
     "--auto-apply", is_flag=True, help="Automatically apply Terraform and kubectl configuration without prompting"
@@ -243,7 +244,7 @@ def infra_init(
     """Init infra"""
     # Check if provider is specified
     if not provider:
-        click.echo("Error: --provider option is required. Please specify a provider (aws, gcp, azure).")
+        click.echo("Error: --provider option is required. Specify a provider. Possible values are: aws, gcp, azure.")
         return
 
     # AWS-specific Terraform template creation
@@ -266,27 +267,27 @@ def infra_init(
                 config = json.load(f)
             click.echo("Loaded existing configuration from config.json")
         except json.JSONDecodeError:
-            click.echo("Warning: Could not parse existing config.json, will create a new one")
+            click.echo("Warning: Could not parse existing config.json. Creating a new file...")
 
     # Generate a random ID for default values
     random_id = str(uuid.uuid4())[:8]
 
     # Get or prompt for configuration values
-    name = name or click.prompt("Enter name", type=str)
-    region = region or config.get("region") or click.prompt("Enter aws region", default="us-east-1", type=str)
-    dns_zone_name = dns_zone_name or config.get("dns_zone_name") or click.prompt("Enter DNS zone name", type=str)
+    name = name or click.prompt("Enter the name for your self-managed region", type=str)
+    region = region or config.get("region") or click.prompt("Enter the AWS region", default="us-east-1", type=str)
+    dns_zone_name = dns_zone_name or config.get("dns_zone_name") or click.prompt("Enter the DNS zone name", type=str)
     namespace = (
         namespace
         or config.get("namespace")
-        or click.prompt("Enter namespace name", default=f"tinybird-{random_id}", type=str)
+        or click.prompt("Enter the Kubernetes namespace", default=f"tinybird-{random_id}", type=str)
     )
     dns_record = (
         dns_record
         or config.get("dns_record")
-        or click.prompt("Enter DNS record name (without domain)", default=f"tinybird-{random_id}", type=str)
+        or click.prompt("Enter the DNS record name, without domain", default=f"tinybird-{random_id}", type=str)
     )
     storage_class = config.get("storage_class") or click.prompt(
-        "Enter storage class", default="gp3-encrypted", type=str
+        "Enter the Kubernetes storage class", default="gp3-encrypted", type=str
     )
 
     # Save configuration
@@ -407,7 +408,7 @@ def infra_init(
             available_contexts = [context.strip() for context in contexts_result.stdout.splitlines() if context.strip()]
 
             if not available_contexts:
-                click.echo("No kubectl contexts found. Please configure kubectl first.")
+                click.echo("No kubectl contexts found. Configure kubectl first.")
                 return
 
             # Prompt user to select a context
@@ -562,3 +563,62 @@ async def infra_ls(ctx: click.Context):
     click.echo(FeedbackManager.info(message="\n** Infras:"))
     echo_safe_humanfriendly_tables_format_smart_table(table_human_readable, column_names=columns)
     click.echo("\n")
+
+
+@infra.command(name="create")
+@click.option("--name", type=str, help="Name for identifying the self-managed infrastructure in Tinybird")
+@click.option("--host", type=str, help="Host for the infrastructure")
+@click.pass_context
+@coro
+async def infra_create(ctx: click.Context, name: str, host: str):
+    """Create a self-managed infrastructure"""
+    try:
+        client: TinyB = ctx.ensure_object(dict)["client"]
+        user_workspaces = await client.user_workspaces_with_organization()
+        admin_org_id = user_workspaces.get("organization_id")
+        if not admin_org_id:
+            raise CLIException("No organization associated to this workspace")
+        name = name or click.prompt("Enter name", type=str)
+        host = host or click.prompt("Enter host", type=str)
+        click.echo(FeedbackManager.highlight(message=f"\n» Creating infrastructure '{name}' in Tinybird..."))
+        infra = await client.infra_create(organization_id=admin_org_id, name=name, host=host)
+        click.echo(FeedbackManager.success(message=f"\n✓ Infrastructure '{name}' created"))
+        pyperclip.copy(infra["token"])
+        click.echo(FeedbackManager.info(message="Access token has been copied to your clipboard."))
+        click.echo(
+            FeedbackManager.info(message="Pass it as an environment variable in your deployment as TB_INFRA_TOKEN")
+        )
+    except Exception as e:
+        click.echo(FeedbackManager.error(message=f"✗ Error: {str(e)}"))
+
+
+@infra.command(name="update")
+@click.argument("infra_name")
+@click.option("--name", type=str, help="Name for identifying the self-managed infrastructure in Tinybird")
+@click.option("--host", type=str, help="Host for the infrastructure")
+@click.pass_context
+@coro
+async def infra_update(ctx: click.Context, infra_name: str, name: str, host: str):
+    """Update a self-managed infrastructure"""
+    try:
+        client: TinyB = ctx.ensure_object(dict)["client"]
+        user_workspaces = await client.user_workspaces_with_organization()
+        admin_org_id = user_workspaces.get("organization_id")
+        if not admin_org_id:
+            raise CLIException("No organization associated to this workspace")
+
+        if not name and not host:
+            click.echo(
+                FeedbackManager.warning(message="No name or host provided. Please provide either a name or a host.")
+            )
+            return
+
+        if name or host:
+            infras = await client.infra_list(organization_id=admin_org_id)
+            infra_id = next((infra["id"] for infra in infras if infra["name"] == infra_name), None)
+            if not infra_id:
+                raise CLIException(f"Infrastructure '{infra_name}' not found")
+            click.echo(FeedbackManager.highlight(message=f"\n» Updating infrastructure '{infra_name}' in Tinybird..."))
+            await client.infra_update(infra_id=infra_id, organization_id=admin_org_id, name=name, host=host)
+    except Exception as e:
+        click.echo(FeedbackManager.error(message=f"✗ Error: {str(e)}"))

{tinybird-0.0.1.dev100.dist-info → tinybird-0.0.1.dev101.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev100
+Version: 0.0.1.dev101
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev100.dist-info → tinybird-0.0.1.dev101.dist-info}/RECORD

@@ -1,5 +1,5 @@
 tinybird/__cli__.py,sha256=esPl5QDTzuQgHe5FuxWLm-fURFigGGwjnYLh9GuWUw4,232
-tinybird/client.py,sha256=aDDifEWkVxJcm4WYaw6uUgXhav-e7JUW1N-mgtEjV4A,55087
+tinybird/client.py,sha256=yc2BP2HiQhaUjU-t7nBuZEiJsb2nQMH3vBfOhKCciyU,55468
 tinybird/config.py,sha256=5UP_UZ2Qtlm5aOH5W7SbtN8r7X-8u3-r853joKqU5zs,6072
 tinybird/connectors.py,sha256=7Gjms7b5MAaBFGi3xytsJurCylprONpFcYrzp4Fw2Rc,15241
 tinybird/context.py,sha256=FfqYfrGX_I7PKGTQo93utaKPDNVYWelg4Hsp3evX5wM,1291
@@ -7,15 +7,15 @@ tinybird/datatypes.py,sha256=XNypumfqNjsvLJ5iNXnbVHRvAJe0aQwI3lS6Cxox-e0,10979
 tinybird/feedback_manager.py,sha256=YSjtFDJvc8y66j2J0iIkb3SVzDdYAJbzFL-JPQ26pak,68761
 tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
 tinybird/prompts.py,sha256=syxUvbdkzxIrIK8pkS-Y7nzBx7DMS2Z9HaiXx7AjvnQ,33912
-tinybird/sql.py,sha256=LBi74GxhNAYTb6m2-KNGpAkguSKh7rcvBbERbE7nalA,46195
+tinybird/sql.py,sha256=J35bhdpuu84HW2tiLp-cs_nzkRwPhiy1yPcFhcWMCR4,46248
 tinybird/sql_template.py,sha256=VH8io4n5eP2z6TEw111d8hcA9FKQerFjprKKCW2MODw,99127
 tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
-tinybird/sql_toolset.py,sha256=32SNvxRFKQYWTvYPMJ_u3ukcd1hKZyEqx8T2cv2412w,14697
+tinybird/sql_toolset.py,sha256=KORVbNAUTfW1qo3U9oe7Z59xQ0QMsFhB0ji3HzY2JVo,15324
 tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
 tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
 tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
 tinybird/ch_utils/engine.py,sha256=BZuPM7MFS7vaEKK5tOMR2bwSAgJudPrJt27uVEwZmTY,40512
-tinybird/tb/__cli__.py,sha256=5rOoJNw0w56lLAZpNCTlWF_nWVfCJL-SxcTE0FJGiG4,252
+tinybird/tb/__cli__.py,sha256=9fN3V2hgGADGGvafwXcIrRciQG80C2cmQs_OlsP8u-A,252
 tinybird/tb/cli.py,sha256=H_HaZhkimKgkryYXpBjHfY9Qtg-ZORiONU3psDNpzDk,1135
 tinybird/tb/modules/auth.py,sha256=L1IatO2arRSzys3t8px8xVt8uPWUL5EVD0sFzAV_uVU,9022
 tinybird/tb/modules/build.py,sha256=h5drdmDFX8NHts9dA2Zepao7KSgMAl3DZGyFufVZP78,11085
@@ -32,7 +32,7 @@ tinybird/tb/modules/endpoint.py,sha256=EhVoGAXsFz-83Fiwj1gI-I73iRRvL49d0W81un7hv
 tinybird/tb/modules/exceptions.py,sha256=4A2sSjCEqKUMqpP3WI00zouCWW4uLaghXXLZBSw04mY,3363
 tinybird/tb/modules/feedback_manager.py,sha256=7nNiOx7OMebiheLED1r0d75SbuXCNxyBmF4e20rCBNc,69511
 tinybird/tb/modules/fmt.py,sha256=qpf9APqKTKL2uphNgdbj4OMVyLkAxZn6dn4eHF99L5g,3553
-tinybird/tb/modules/infra.py,sha256=l64FN62AB6N_ICisEonVsI_uQ_hzwZRBvupBjBJKac8,19707
+tinybird/tb/modules/infra.py,sha256=vb6WmVHFuU1ZhtQMh2wOzRd2x8g7CLt_2gR3EL_L2E4,22867
 tinybird/tb/modules/job.py,sha256=956Pj8BEEsiD2GZsV9RKKVM3I_CveOLgS82lykO5ukk,2963
 tinybird/tb/modules/llm.py,sha256=AC0VSphTOM2t-v1_3NLvNN_FIbgMo4dTyMqIv5nniPo,835
 tinybird/tb/modules/llm_utils.py,sha256=nS9r4FAElJw8yXtmdYrx-rtI2zXR8qXfi1QqUDCfxvg,3469
@@ -61,7 +61,7 @@ tinybird/tb/modules/datafile/build.py,sha256=jhfIJ2xt0N13XsLPe3iMQIyCPApHS13_Df2
 tinybird/tb/modules/datafile/build_common.py,sha256=rT7VJ5mnQ68R_8US91DAtkusfvjWuG_NObOzNgtN_ko,4562
 tinybird/tb/modules/datafile/build_datasource.py,sha256=CCU3eQ8Rax9RgHHfbAXDRL6rQ49N35h_GDQnGrUUUzA,17379
 tinybird/tb/modules/datafile/build_pipe.py,sha256=w-Wd08gZYAEcak9FdBijVfIU2_Wn_PPdgAZddPpoGTo,11382
-tinybird/tb/modules/datafile/common.py,sha256=i9Gvhz3JiR58MRBcYZDwqTqamQOj-46TnHU8Hm8bqmg,81399
+tinybird/tb/modules/datafile/common.py,sha256=ZRRx-9EXeHMOz6Lu9Y1TSWELF0ftfUHLLQAANjJZ334,82241
 tinybird/tb/modules/datafile/diff.py,sha256=-0J7PsBO64T7LOZSkZ4ZFHHCPvT7cKItnJkbz2PkndU,6754
 tinybird/tb/modules/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
 tinybird/tb/modules/datafile/fixture.py,sha256=si-9LB-LdKQSWDtVW82xDrHtFfko5bgBG1cvjqqrcPU,1064
@@ -81,8 +81,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev100.dist-info/METADATA,sha256=QaYp1xwcpo3MDoYY8ZGLSXI9ZHFKCV6I6bk2d9u6hRg,2586
-tinybird-0.0.1.dev100.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev100.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev100.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev100.dist-info/RECORD,,
+tinybird-0.0.1.dev101.dist-info/METADATA,sha256=73sj_cIfixFVZVrtmkn_MwEYvtkMLO4GYeJCjJQ1KA0,2586
+tinybird-0.0.1.dev101.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev101.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev101.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev101.dist-info/RECORD,,