tinybird 0.0.1.dev102__tar.gz → 0.0.1.dev104__tar.gz

{tinybird-0.0.1.dev102 → tinybird-0.0.1.dev104}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev102
+Version: 0.0.1.dev104
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -9,6 +9,7 @@ Requires-Python: >=3.9, <3.14
 Description-Content-Type: text/x-rst
 Requires-Dist: aiofiles==24.1.0
 Requires-Dist: anthropic==0.42.0
+Requires-Dist: boto3
 Requires-Dist: click<8.2,>=8.1.6
 Requires-Dist: clickhouse-toolset==0.34.dev0
 Requires-Dist: colorama==0.4.6
@@ -32,29 +33,11 @@ Requires-Dist: wheel
 Requires-Dist: packaging<24,>=23.1
 Requires-Dist: llm>=0.19
 Requires-Dist: thefuzz==0.22.1
-Provides-Extra: bigquery
-Requires-Dist: gsutil==4.58; extra == "bigquery"
-Requires-Dist: google-api-python-client==2.0.2; extra == "bigquery"
-Requires-Dist: google-auth==1.27.1; extra == "bigquery"
-Requires-Dist: google-auth-httplib2==0.1.0; extra == "bigquery"
-Requires-Dist: google-cloud-storage==2.4.0; extra == "bigquery"
-Requires-Dist: google-cloud-bigquery==2.11.0; extra == "bigquery"
-Provides-Extra: snowflake
-Requires-Dist: snowflake-connector-python~=3.12.3; extra == "snowflake"
-Requires-Dist: gsutil==4.58; extra == "snowflake"
-Requires-Dist: google-api-python-client==2.0.2; extra == "snowflake"
-Requires-Dist: google-auth==1.27.1; extra == "snowflake"
-Requires-Dist: google-auth-httplib2==0.1.0; extra == "snowflake"
-Requires-Dist: google-cloud-storage==2.4.0; extra == "snowflake"
-Requires-Dist: oauth2client==3.0.0; extra == "snowflake"
-Requires-Dist: chardet<4,>=3.0.2; extra == "snowflake"
-Requires-Dist: pyOpenSSL<20.0.0,>=16.2.0; extra == "snowflake"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
-Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary

{tinybird-0.0.1.dev102 → tinybird-0.0.1.dev104}/tinybird/feedback_manager.py

@@ -651,6 +651,12 @@ Ready? """
     warning_tag_remove = prompt_message(
         "Tag {tag_name} is used by {resources_len} resources. Do you want to remove it?"
     )
+    warning_bigquery_connector_deprecated = warning_message(
+        "🚨🚨🚨 [DEPRECATED] BigQuery connector is deprecated. Use ingestion from GCS instead. https://www.tinybird.co/docs/get-data-in/guides/ingest-from-bigquery-using-google-cloud-storage **"
+    )
+    warning_s3_access_key_secret_deprecated = warning_message(
+        "🚨🚨🚨 [DEPRECATED] S3 (Access Key + Secret) connector is deprecated. Use S3 IAM role instead. https://www.tinybird.co/docs/get-data-in/connectors/s3 **"
+    )
 
     info_fixtures_branch = info_message("** Data Fixtures are only pushed to Branches")
     info_materialize_push_datasource_exists = warning_message("** Data Source {name} already exists")

{tinybird-0.0.1.dev102 → tinybird-0.0.1.dev104}/tinybird/sql_template.py

@@ -2427,6 +2427,20 @@ def render_template_with_secrets(name: str, content: str, secrets: Optional[Dict
         ... )
         'KAFKA_BOOTSTRAP_SERVERS localhost:9092'
 
+        >>> render_template_with_secrets(
+        ...     "my_kafka_connection",
+        ...     "KAFKA_BOOTSTRAP_SERVERS {{ tb_secret('MISSING_SECRET', '') }}",
+        ...     secrets = {}
+        ... )
+        'KAFKA_BOOTSTRAP_SERVERS ""'
+
+        >>> render_template_with_secrets(
+        ...     "my_kafka_connection",
+        ...     "KAFKA_BOOTSTRAP_SERVERS {{ tb_secret('MISSING_SECRET', 0) }}",
+        ...     secrets = {}
+        ... )
+        'KAFKA_BOOTSTRAP_SERVERS 0'
+
         >>> render_template_with_secrets(
         ...     "my_kafka_connection",
         ...     "KAFKA_BOOTSTRAP_SERVERS {{ tb_secret('MISSING_SECRET') }}",
@@ -2453,8 +2467,13 @@ def render_template_with_secrets(name: str, content: str, secrets: Optional[Dict
             SQLTemplateException: If the secret is not found and no default is provided
         """
         if secret_name in secrets:
-            return secrets[secret_name]
+            value = secrets[secret_name]
+            if isinstance(value, str) and len(value) == 0:
+                return '""'
+            return value
         elif default is not None:
+            if isinstance(default, str) and len(default) == 0:
+                return '""'
             return default
         else:
             raise SQLTemplateException(
@@ -2462,7 +2481,7 @@ def render_template_with_secrets(name: str, content: str, secrets: Optional[Dict
             )
 
     # Create the template
-    t = Template(content, name=name)
+    t = Template(content, name=name, autoescape=None)
 
     try:
         # Create namespace with our tb_secret function

{tinybird-0.0.1.dev102 → tinybird-0.0.1.dev104}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev102'
-__revision__ = '627c8ef'
+__version__ = '0.0.1.dev104'
+__revision__ = 'f6c1941'

{tinybird-0.0.1.dev102 → tinybird-0.0.1.dev104}/tinybird/tb/modules/cicd.py

@@ -55,7 +55,7 @@ jobs:
       - name: Test project
         run: tb test run
       - name: Deployment check
-        run: tb --cloud --host ${{! TINYBIRD_HOST }} --token ${{! TINYBIRD_TOKEN }} deploy --check
+        run: tb --cloud --host ${{! env.TINYBIRD_HOST }} --token ${{! env.TINYBIRD_TOKEN }} deploy --check
 """
 
 

{tinybird-0.0.1.dev102 → tinybird-0.0.1.dev104}/tinybird/tb/modules/cli.py

@@ -158,47 +158,6 @@ async def pull(ctx: Context, force: bool, fmt: bool) -> None:
     return await folder_pull(client, project.path, force, fmt=fmt)
 
 
-@cli.command(hidden=True)
-@click.option("--no-deps", is_flag=True, default=False, help="Print only data sources with no pipes using them")
-@click.option("--match", default=None, help="Retrieve any resource matching the pattern")
-@click.option("--pipe", default=None, help="Retrieve any resource used by pipe")
-@click.option("--datasource", default=None, help="Retrieve resources depending on this data source")
-@click.option(
-    "--check-for-partial-replace",
-    is_flag=True,
-    default=False,
-    help="Retrieve dependant data sources that will have their data replaced if a partial replace is executed in the data source selected",
-)
-@click.option("--recursive", is_flag=True, default=False, help="Calculate recursive dependencies")
-@click.pass_context
-@coro
-async def dependencies(
-    ctx: Context,
-    no_deps: bool,
-    match: Optional[str],
-    pipe: Optional[str],
-    datasource: Optional[str],
-    check_for_partial_replace: bool,
-    recursive: bool,
-) -> None:
-    """Print all data sources dependencies."""
-
-    client = ctx.ensure_object(dict)["client"]
-
-    response = await client.datasource_dependencies(
-        no_deps, match, pipe, datasource, check_for_partial_replace, recursive
-    )
-    for ds in response["dependencies"]:
-        click.echo(FeedbackManager.info_dependency_list(dependency=ds))
-        for pipe in response["dependencies"][ds]:
-            click.echo(FeedbackManager.info_dependency_list_item(dependency=pipe))
-    if "incompatible_datasources" in response and len(response["incompatible_datasources"]):
-        click.echo(FeedbackManager.info_no_compatible_dependencies_found())
-        for ds in response["incompatible_datasources"]:
-            click.echo(FeedbackManager.info_dependency_list(dependency=ds))
-        raise CLIException(FeedbackManager.error_partial_replace_cant_be_executed(datasource=datasource))
-
-
 @cli.command(
     name="diff",
     short_help="Diff local datafiles to the corresponding remote files in the workspace. Only diffs VERSION and SCHEMA for .datasource files.",

{tinybird-0.0.1.dev102 → tinybird-0.0.1.dev104}/tinybird/tb/modules/common.py

@@ -81,8 +81,6 @@ SUPPORTED_FORMATS = ["csv", "ndjson", "json", "parquet"]
 OLDEST_ROLLBACK = "oldest_rollback"
 MAIN_BRANCH = "main"
 
-CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
-
 
 def obfuscate_token(value: Optional[str]) -> Optional[str]:
     if not value:
@@ -1984,12 +1982,12 @@ async def send_batch_events(
     click.echo(FeedbackManager.highlight(message=f"» {rows} rows x {cols} cols in {elapsed_time:.1f}s"))
 
 
-async def get_organizations_by_user(config: CLIConfig, user_token: str) -> List[Dict[str, Any]]:
+async def get_organizations_by_user(config: CLIConfig, user_token: Optional[str] = None) -> List[Dict[str, str]]:
     """Fetches all organizations by user using the provided user token"""
     organizations = []
 
     try:
-        user_client = config.get_client(token=user_token)
+        user_client = config.get_client(token=user_token) if user_token else config.get_user_client()
         user_workspaces = await user_client.user_workspaces_with_organization(version="v1")
         admin_org_id = user_workspaces.get("organization_id")
         seen_org_ids = set()
@@ -2089,7 +2087,7 @@ async def ask_for_organization_name(config: CLIConfig) -> str:
             user_email.split("@")[0] if user_email else None
         )  # Example: "jane.doe@gmail.com" -> "jane.doe"
     return click.prompt(
-        "\nYou need to create an organization to continue. We will include your existing workspaces in it.\nThis operation will only happen once and then your new workspaces will be automatically included in your organization.\n\nEnter organization name",
+        "\nYou need to create an organization to continue.\nEnter organization name",
         hide_input=False,
         show_default=True,
         default=default_organization_name,
@@ -2166,3 +2164,39 @@ def get_ui_url(api_host: str) -> str:
             return f"https://cloud.tinybird.co/{cloud}/{region}"
 
     return api_host
+
+
+async def ask_for_organization(
+    organizations: Optional[List[Dict[str, Any]]],
+    organization_id: Optional[str] = None,
+    user_token: Optional[str] = None,
+) -> Tuple[Optional[str], Optional[str]]:
+    config = CLIConfig.get_project_config()
+    user_email = config.get_user_email()
+
+    if organization_id:
+        if organizations and len(organizations) > 0:
+            organization = next((org for org in organizations if org.get("id") == organization_id), None)
+        if not organization:
+            raise CLIException(FeedbackManager.error_organization_not_found(organization_id=organization_id))
+        organization_name = organization.get("name")
+        return organization_id, organization_name
+
+    if organizations is None or len(organizations) == 0:
+        organization_name = await ask_for_organization_name(config)
+        user_token = await get_user_token(config, user_token)
+        organization = await create_organization_and_add_workspaces(config, organization_name, user_token)
+        organization_id = organization.get("id")
+    else:
+        if len(organizations) == 1:
+            organization_name = organizations[0]["name"]
+            organization_id = organizations[0]["id"]
+        else:
+            sorted_organizations = sort_organizations_by_user(organizations, user_email=user_email)
+            current_organization = await ask_for_organization_interactively(sorted_organizations)
+            if current_organization:
+                organization_id = current_organization.get("id")
+                organization_name = current_organization.get("name")
+            else:
+                return None, None
+    return organization_id, organization_name

{tinybird-0.0.1.dev102 → tinybird-0.0.1.dev104}/tinybird/tb/modules/create.py

@@ -299,9 +299,9 @@ def init_git(folder: str):
         if gitignore_file.exists():
             content = gitignore_file.read_text()
             if ".tinyb" not in content:
-                gitignore_file.write_text(content + "\n.tinyb\n")
+                gitignore_file.write_text(content + "\n.tinyb\n.terraform\n")
         else:
-            gitignore_file.write_text(".tinyb\n")
+            gitignore_file.write_text(".tinyb\n.terraform\n")
 
         click.echo(FeedbackManager.info_file_created(file=".gitignore"))
     except Exception as e:

{tinybird-0.0.1.dev102 → tinybird-0.0.1.dev104}/tinybird/tb/modules/datafile/common.py

@@ -1241,6 +1241,13 @@ def parse(
 
         return _f
 
+    def kafka_key_avro_deserialization(*args: str, **kwargs: Any):
+        raise DatafileSyntaxError(
+            f'{kwargs["cmd"].upper()} has been deprecated. Use "KAFKA_KEY_FORMAT avro" instead',
+            lineno=kwargs["lineno"],
+            pos=1,
+        )
+
     @deprecated
     def sources(x: str, **kwargs: Any) -> None:
         pass  # Deprecated
@@ -1459,11 +1466,13 @@ def parse(
         "kafka_store_raw_value": assign_var("kafka_store_raw_value"),
         "kafka_store_headers": assign_var("kafka_store_headers"),
         "kafka_store_binary_headers": assign_var("kafka_store_binary_headers"),
-        "kafka_key_avro_deserialization": assign_var("kafka_key_avro_deserialization"),
+        "kafka_key_avro_deserialization": kafka_key_avro_deserialization,
         "kafka_ssl_ca_pem": assign_var("kafka_ssl_ca_pem"),
         "kafka_security_protocol": assign_var("kafka_security_protocol"),
         "kafka_sasl_mechanism": assign_var("kafka_sasl_mechanism"),
         "import_service": assign_var("import_service"),
+        "s3_region": assign_var("s3_region"),
+        "s3_arn": assign_var("s3_arn"),
         "import_connection_name": assign_var("import_connection_name"),
         "import_schedule": assign_var("import_schedule"),
         "import_strategy": assign_var("import_strategy"),

{tinybird-0.0.1.dev102 → tinybird-0.0.1.dev104}/tinybird/tb/modules/deployment.py

@@ -19,6 +19,7 @@ from tinybird.tb.modules.project import Project
 def api_fetch(url: str, headers: dict) -> dict:
     r = requests.get(url, headers=headers)
     if r.status_code == 200:
+        logging.debug(json.dumps(r.json(), indent=2))
         return r.json()
     # Try to parse and print the error from the response
     try:
@@ -32,13 +33,33 @@ def api_fetch(url: str, headers: dict) -> dict:
         sys.exit(1)
 
 
+def api_post(
+    url: str,
+    headers: dict,
+    files: Optional[list] = None,
+    params: Optional[dict] = None,
+) -> dict:
+    r = requests.post(url, headers=headers, files=files, params=params)
+    if r.status_code == 200:
+        logging.debug(json.dumps(r.json(), indent=2))
+        return r.json()
+    # Try to parse and print the error from the response
+    try:
+        result = r.json()
+        logging.debug(json.dumps(result, indent=2))
+        error = result.get("error")
+        click.echo(FeedbackManager.error(message=f"Error: {error}"))
+        sys.exit(1)
+    except Exception:
+        click.echo(FeedbackManager.error(message="Error parsing response from API"))
+        sys.exit(1)
+
+
 # TODO(eclbg): This logic should be in the server, and there should be a dedicated endpoint for promoting a deployment
 # potato
 def promote_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
     TINYBIRD_API_URL = f"{host}/v1/deployments"
-    r = requests.get(TINYBIRD_API_URL, headers=headers)
-    result = r.json()
-    logging.debug(json.dumps(result, indent=2))
+    result = api_fetch(TINYBIRD_API_URL, headers)
 
     deployments = result.get("deployments")
     if not deployments:
@@ -64,12 +85,7 @@ def promote_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
         click.echo(FeedbackManager.success(message="Setting candidate deployment as live"))
 
         TINYBIRD_API_URL = f"{host}/v1/deployments/{candidate_deployment.get('id')}/set-live"
-        r = requests.post(TINYBIRD_API_URL, headers=headers)
-        result = r.json()
-        logging.debug(json.dumps(result, indent=2))
-        if result.get("error"):
-            click.echo(FeedbackManager.error(message=result.get("error")))
-            sys.exit(1)
+        result = api_post(TINYBIRD_API_URL, headers=headers)
 
     click.echo(FeedbackManager.success(message="Removing old deployment"))
 
@@ -86,9 +102,7 @@ def promote_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
     if wait:
         while True:
             TINYBIRD_API_URL = f"{host}/v1/deployments/{last_deployment.get('id')}"
-            r = requests.get(TINYBIRD_API_URL, headers=headers)
-            result = r.json()
-            logging.debug(json.dumps(result, indent=2))
+            result = api_fetch(TINYBIRD_API_URL, headers=headers)
 
             last_deployment = result.get("deployment")
             if last_deployment.get("status") == "deleted":
@@ -102,9 +116,7 @@ def promote_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
 # deployment
 def rollback_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
     TINYBIRD_API_URL = f"{host}/v1/deployments"
-    r = requests.get(TINYBIRD_API_URL, headers=headers)
-    result = r.json()
-    logging.debug(json.dumps(result, indent=2))
+    result = api_fetch(TINYBIRD_API_URL, headers=headers)
 
     deployments = result.get("deployments")
     if not deployments:
@@ -130,12 +142,7 @@ def rollback_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
         click.echo(FeedbackManager.success(message="Promoting previous deployment"))
 
         TINYBIRD_API_URL = f"{host}/v1/deployments/{previous_deployment.get('id')}/set-live"
-        r = requests.post(TINYBIRD_API_URL, headers=headers)
-        result = r.json()
-        logging.debug(json.dumps(result, indent=2))
-        if result.get("error"):
-            click.echo(FeedbackManager.error(message=result.get("error")))
-            sys.exit(1)
+        result = api_post(TINYBIRD_API_URL, headers=headers)
 
     click.echo(FeedbackManager.success(message="Removing current deployment"))
 
@@ -152,9 +159,7 @@ def rollback_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
     if wait:
         while True:
             TINYBIRD_API_URL = f"{host}/v1/deployments/{current_deployment.get('id')}"
-            r = requests.get(TINYBIRD_API_URL, headers=headers)
-            result = r.json()
-            logging.debug(json.dumps(result, indent=2))
+            result = api_fetch(TINYBIRD_API_URL, headers)
 
             current_deployment = result.get("deployment")
             if current_deployment.get("status") == "deleted":
@@ -219,7 +224,6 @@ def deployment_ls(ctx: click.Context) -> None:
     url = f"{client.host}/v1/deployments"
 
     result = api_fetch(url, HEADERS)
-    logging.debug(json.dumps(result, indent=2))
 
     status_map = {"data_ready": "Ready", "failed": "Failed"}
     columns = ["ID", "Status", "Created at", "Live"]
@@ -354,13 +358,13 @@ def create_deployment(
             params["check"] = "true"
         if allow_destructive_operations:
             params["allow_destructive_operations"] = "true"
-        r = requests.post(TINYBIRD_API_URL, files=files, headers=HEADERS, params=params)
-        result = r.json()
-        logging.debug(json.dumps(result, indent=2))
+
+        result = api_post(TINYBIRD_API_URL, headers=HEADERS, files=files, params=params)
 
         print_changes(result, project)
 
-        feedback = result.get("deployment", {}).get("feedback", [])
+        deployment = result.get("deployment", {})
+        feedback = deployment.get("feedback", [])
         for f in feedback:
             if f.get("level", "").upper() == "ERROR":
                 feedback_func = FeedbackManager.error
@@ -372,7 +376,7 @@ def create_deployment(
             resource_bit = f"{resource}: " if resource else ""
             click.echo(feedback_func(message=f"{feedback_icon}{f.get('level')}: {resource_bit}{f.get('message')}"))
 
-        deploy_errors = result.get("deployment", {}).get("errors")
+        deploy_errors = deployment.get("errors")
         for deploy_error in deploy_errors:
             if deploy_error.get("filename", None):
                 click.echo(
@@ -380,19 +384,21 @@ def create_deployment(
                 )
             else:
                 click.echo(FeedbackManager.error(message=f"{deploy_error.get('error')}"))
+        click.echo()  # For spacing
 
         status = result.get("result")
         if check:
             if status == "success":
                 click.echo(FeedbackManager.success(message="\n✓ Deployment is valid"))
                 sys.exit(0)
+            elif status == "no_changes":
+                sys.exit(0)
 
             click.echo(FeedbackManager.error(message="\n✗ Deployment is not valid"))
             sys.exit(1)
 
         status = result.get("result")
         if status == "success":
-            deployment = result.get("deployment", {})
             # TODO: This is a hack to show the url in the case of region is public. The URL should be returned by the API
             if client.host == "https://api.europe-west2.gcp.tinybird.co":
                 click.echo(
@@ -406,10 +412,14 @@ def create_deployment(
                 click.echo(FeedbackManager.info(message="\n✓ Deployment submitted successfully"))
             else:
                 click.echo(FeedbackManager.success(message="\n✓ Deployment submitted successfully"))
+        elif status == "no_changes":
+            click.echo(FeedbackManager.highlight(message="Not deploying. No changes."))
+            sys.exit(0)
         elif status == "failed":
             click.echo(FeedbackManager.error(message="Deployment failed"))
+            sys.exit(1)
         else:
-            click.echo(FeedbackManager.error(message=f"Unknown build result {status}"))
+            click.echo(FeedbackManager.error(message=f"Unknown deployment result {status}"))
     except Exception as e:
         click.echo(FeedbackManager.error_exception(error=e))
     finally:
@@ -463,7 +473,7 @@ def print_changes(result: dict, project: Project) -> None:
         resources.append(["new", p, project.get_resource_path(p, "pipe")])
 
     for dc in deployment.get("new_data_connector_names", []):
-        resources.append(["new", dc, project.get_resource_path(dc, "data_connector")])
+        resources.append(["new", dc, project.get_resource_path(dc, "connection")])
 
     for ds in deployment.get("changed_datasource_names", []):
         resources.append(["modified", ds, project.get_resource_path(ds, "datasource")])
@@ -472,7 +482,7 @@ def print_changes(result: dict, project: Project) -> None:
         resources.append(["modified", p, project.get_resource_path(p, "pipe")])
 
     for dc in deployment.get("changed_data_connector_names", []):
-        resources.append(["modified", dc, project.get_resource_path(dc, "data_connector")])
+        resources.append(["modified", dc, project.get_resource_path(dc, "connection")])
 
     for ds in deployment.get("deleted_datasource_names", []):
         resources.append(["deleted", ds, project.get_resource_path(ds, "datasource")])
@@ -481,7 +491,7 @@ def print_changes(result: dict, project: Project) -> None:
         resources.append(["deleted", p, project.get_resource_path(p, "pipe")])
 
     for dc in deployment.get("deleted_data_connector_names", []):
-        resources.append(["deleted", dc, project.get_resource_path(dc, "data_connector")])
+        resources.append(["deleted", dc, project.get_resource_path(dc, "connection")])
 
     for token_change in deployment.get("token_changes", []):
         token_name = token_change.get("token_name")