tinybird 0.0.1.dev212__py3-none-any.whl → 0.0.1.dev214__py3-none-any.whl

tinybird/tb/modules/datafile/playground.py

@@ -11,20 +11,7 @@ from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
 import click
 from toposort import toposort
 
-from tinybird.sql import parse_table_structure, schema_to_sql_columns
-from tinybird.sql_template import get_used_tables_in_template, render_sql_template
-from tinybird.tb.client import TinyB
-from tinybird.tb.modules.common import get_ca_pem_content
-from tinybird.tb.modules.config import CLIConfig
-from tinybird.tb.modules.datafile.build_datasource import is_datasource
-from tinybird.tb.modules.datafile.build_pipe import (
-    get_target_materialized_data_source_name,
-    is_endpoint,
-    is_endpoint_with_no_dependencies,
-    is_materialized,
-    new_pipe,
-)
-from tinybird.tb.modules.datafile.common import (
+from tinybird.datafile.common import (
     DEFAULT_CRON_PERIOD,
     INTERNAL_TABLES,
     ON_DEMAND,
@@ -40,9 +27,22 @@ from tinybird.tb.modules.datafile.common import (
     get_project_filenames,
     pp,
 )
-from tinybird.tb.modules.datafile.exceptions import AlreadyExistsException, IncludeFileNotFoundException
-from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
-from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
+from tinybird.datafile.exceptions import AlreadyExistsException, IncludeFileNotFoundException
+from tinybird.datafile.parse_datasource import parse_datasource
+from tinybird.datafile.parse_pipe import parse_pipe
+from tinybird.sql import parse_table_structure, schema_to_sql_columns
+from tinybird.sql_template import get_used_tables_in_template, render_sql_template
+from tinybird.tb.client import TinyB
+from tinybird.tb.modules.common import get_ca_pem_content
+from tinybird.tb.modules.config import CLIConfig
+from tinybird.tb.modules.datafile.build_datasource import is_datasource
+from tinybird.tb.modules.datafile.build_pipe import (
+    get_target_materialized_data_source_name,
+    is_endpoint,
+    is_endpoint_with_no_dependencies,
+    is_materialized,
+    new_pipe,
+)
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.project import Project
 

tinybird/tb/modules/datasource.py

@@ -18,6 +18,7 @@ import humanfriendly
 import requests
 from click import Context
 
+from tinybird.datafile.common import get_name_version
 from tinybird.prompts import quarantine_prompt
 from tinybird.syncasync import sync_to_async
 from tinybird.tb.client import AuthNoTokenException, DoesNotExistException, TinyB
@@ -40,7 +41,6 @@ from tinybird.tb.modules.create import (
     generate_aws_iamrole_connection_file_with_secret,
     generate_gcs_connection_file_with_secrets,
 )
-from tinybird.tb.modules.datafile.common import get_name_version
 from tinybird.tb.modules.datafile.fixture import persist_fixture
 from tinybird.tb.modules.exceptions import CLIDatasourceException
 from tinybird.tb.modules.feedback_manager import FeedbackManager

tinybird/tb/modules/endpoint.py

@@ -14,10 +14,10 @@ import pyperclip
 import requests
 from click import Context
 
+from tinybird.datafile.common import get_name_version
 from tinybird.tb.client import AuthNoTokenException, DoesNotExistException, TinyB
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import coro, echo_safe_humanfriendly_tables_format_smart_table
-from tinybird.tb.modules.datafile.common import get_name_version
 from tinybird.tb.modules.exceptions import CLIPipeException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 

tinybird/tb/modules/feedback_manager.py

@@ -587,7 +587,7 @@ STEP {step}: CREATE IAM ROLE FOR {environment} ENVIRONMENT
 Please enter the ARN of the role you just created"""
     )
 
-    prompt_s3_iamrole_success = success_message("""
+    prompt_s3_iamrole_success_read = success_message("""
 ✅ S3 CONNECTION CONFIGURED SUCCESSFULLY
 
   • File created at: {connection_path}
@@ -595,6 +595,14 @@ Please enter the ARN of the role you just created"""
   • Learn more about our S3 Connector: https://www.tinybird.co/docs/forward/get-data-in/connectors/s3
 """)
 
+    prompt_s3_iamrole_success_write = success_message("""
+✅ S3 CONNECTION CONFIGURED SUCCESSFULLY
+
+  • File created at: {connection_path}
+  • You can now use this connection in your Data Sources with: EXPORT_CONNECTION_NAME '{connection_name}'
+  • Learn more about our S3 Sinks: https://www.tinybird.co/docs/forward/work-with-data/publish-data/s3-sinks
+""")
+
     # S3 IAM Role Connection Messages
     prompt_gcs_connection_header = info_highlight_message("""
 ──────────────────────────────────────────────────────────────

tinybird/tb/modules/local.py

@@ -1,249 +1,20 @@
-import json
-import os
-import re
 import subprocess
-import time
-from typing import Optional
 
-import boto3
 import click
 import requests
 
-import docker
 from docker.client import DockerClient
-from docker.models.containers import Container
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import coro
-from tinybird.tb.modules.exceptions import CLIException, CLILocalException
+from tinybird.tb.modules.exceptions import CLIException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
-from tinybird.tb.modules.local_common import TB_CONTAINER_NAME, TB_IMAGE_NAME, TB_LOCAL_ADDRESS, TB_LOCAL_PORT
-from tinybird.tb.modules.telemetry import add_telemetry_event
-
-
-def start_tinybird_local(
-    docker_client: DockerClient,
-    use_aws_creds: bool,
-) -> None:
-    """Start the Tinybird container."""
-    pull_show_prompt = False
-    pull_required = False
-    try:
-        local_image = docker_client.images.get(TB_IMAGE_NAME)
-        local_image_id = local_image.attrs["RepoDigests"][0].split("@")[1]
-        remote_image = docker_client.images.get_registry_data(TB_IMAGE_NAME)
-        pull_show_prompt = local_image_id != remote_image.id
-    except Exception:
-        pull_show_prompt = False
-        pull_required = True
-
-    if pull_show_prompt and click.confirm(
-        FeedbackManager.warning(message="△ New version detected, download? [y/N]:"),
-        show_default=False,
-        prompt_suffix="",
-    ):
-        click.echo(FeedbackManager.info(message="* Downloading latest version of Tinybird Local..."))
-        pull_required = True
-
-    if pull_required:
-        docker_client.images.pull(TB_IMAGE_NAME, platform="linux/amd64")
-
-    environment = get_use_aws_creds() if use_aws_creds else {}
-
-    container = get_existing_container_with_matching_env(docker_client, TB_CONTAINER_NAME, environment)
-
-    if container and not pull_required:
-        # Container `start` is idempotent. It's safe to call it even if the container is already running.
-        container.start()
-    else:
-        if container:
-            container.remove(force=True)
-
-        container = docker_client.containers.run(
-            TB_IMAGE_NAME,
-            name=TB_CONTAINER_NAME,
-            detach=True,
-            ports={"7181/tcp": TB_LOCAL_PORT},
-            remove=False,
-            platform="linux/amd64",
-            environment=environment,
-        )
-
-    click.echo(FeedbackManager.info(message="* Waiting for Tinybird Local to be ready..."))
-    while True:
-        container.reload()  # Refresh container attributes
-        health = container.attrs.get("State", {}).get("Health", {}).get("Status")
-        if health == "healthy":
-            break
-        if health == "unhealthy":
-            raise CLILocalException(
-                FeedbackManager.error(
-                    message="Tinybird Local is unhealthy. Try running `tb local restart` in a few seconds."
-                )
-            )
-
-        time.sleep(5)
-
-    # Remove tinybird-local dangling images to avoid running out of disk space
-    images = docker_client.images.list(name=re.sub(r":.*$", "", TB_IMAGE_NAME), all=True, filters={"dangling": True})
-    for image in images:
-        image.remove(force=True)
-
-
-def get_existing_container_with_matching_env(
-    docker_client: DockerClient, container_name: str, required_env: dict[str, str]
-) -> Optional[Container]:
-    """
-    Checks if a container with the given name exists and has matching environment variables.
-    If it exists but environment doesn't match, it returns None.
-
-    Args:
-        docker_client: The Docker client instance
-        container_name: The name of the container to check
-        required_env: Dictionary of environment variables that must be present
-
-    Returns:
-        The container if it exists with matching environment, None otherwise
-    """
-    container = None
-    containers = docker_client.containers.list(all=True, filters={"name": container_name})
-    if containers:
-        container = containers[0]
-
-    if container and required_env:
-        container_info = container.attrs
-        container_env = container_info.get("Config", {}).get("Env", [])
-        env_missing = False
-        for key, value in required_env.items():
-            env_var = f"{key}={value}"
-            if env_var not in container_env:
-                env_missing = True
-                break
-
-        if env_missing:
-            container.remove(force=True)
-            container = None
-
-    return container
-
-
-def get_docker_client() -> DockerClient:
-    """Check if Docker is installed and running."""
-    try:
-        docker_host = os.getenv("DOCKER_HOST")
-        if not docker_host:
-            # Try to get docker host from docker context
-            try:
-                try:
-                    output = subprocess.check_output(["docker", "context", "inspect"], text=True)
-                except Exception as e:
-                    add_telemetry_event(
-                        "docker_error",
-                        error=f"docker_context_inspect_error: {str(e)}",
-                    )
-                    raise e
-                try:
-                    context = json.loads(output)
-                except Exception as e:
-                    add_telemetry_event(
-                        "docker_error",
-                        error=f"docker_context_inspect_parse_output_error: {str(e)}",
-                        data={
-                            "docker_context_inspect_output": output,
-                        },
-                    )
-                    raise e
-                if context and len(context) > 0:
-                    try:
-                        docker_host = context[0].get("Endpoints", {}).get("docker", {}).get("Host")
-                        if docker_host:
-                            os.environ["DOCKER_HOST"] = docker_host
-                    except Exception as e:
-                        add_telemetry_event(
-                            "docker_error",
-                            error=f"docker_context_parse_host_error: {str(e)}",
-                            data={
-                                "context": json.dumps(context),
-                            },
-                        )
-                        raise e
-            except Exception:
-                pass
-        try:
-            client = docker.from_env()  # type: ignore
-        except Exception as e:
-            add_telemetry_event(
-                "docker_error",
-                error=f"docker_get_client_from_env_error: {str(e)}",
-            )
-            raise e
-        try:
-            client.ping()
-        except Exception as e:
-            client_dict_non_sensitive = {k: v for k, v in client.api.__dict__.items() if "auth" not in k}
-            add_telemetry_event(
-                "docker_error",
-                error=f"docker_ping_error: {str(e)}",
-                data={
-                    "client": repr(client_dict_non_sensitive),
-                },
-            )
-            raise e
-        return client
-    except Exception:
-        docker_location_message = ""
-        if docker_host:
-            docker_location_message = f"Trying to connect to Docker-compatible runtime at {docker_host}"
-
-        raise CLILocalException(
-            FeedbackManager.error(
-                message=(
-                    f"No container runtime is running. Make sure a Docker-compatible runtime is installed and running. "
-                    f"{docker_location_message}\n\n"
-                    "If you're using a custom location, please provide it using the DOCKER_HOST environment variable."
-                )
-            )
-        )
-
-
-def get_use_aws_creds() -> dict[str, str]:
-    credentials: dict[str, str] = {}
-    try:
-        # Get the boto3 session and credentials
-        session = boto3.Session()
-        creds = session.get_credentials()
-
-        if creds:
-            # Create environment variables for the container based on boto credentials
-            credentials["AWS_ACCESS_KEY_ID"] = creds.access_key
-            credentials["AWS_SECRET_ACCESS_KEY"] = creds.secret_key
-
-            # Add session token if it exists (for temporary credentials)
-            if creds.token:
-                credentials["AWS_SESSION_TOKEN"] = creds.token
-
-            # Add region if available
-            if session.region_name:
-                credentials["AWS_DEFAULT_REGION"] = session.region_name
-
-            click.echo(
-                FeedbackManager.success(
-                    message=f"✓ AWS credentials found and will be passed to Tinybird Local (region: {session.region_name or 'not set'})"
-                )
-            )
-        else:
-            click.echo(
-                FeedbackManager.warning(
-                    message="△ No AWS credentials found. S3 operations will not work in Tinybird Local."
-                )
-            )
-    except Exception as e:
-        click.echo(
-            FeedbackManager.warning(
-                message=f"△ Error retrieving AWS credentials: {str(e)}. S3 operations will not work in Tinybird Local."
-            )
-        )
-
-    return credentials
+from tinybird.tb.modules.local_common import (
+    TB_CONTAINER_NAME,
+    TB_LOCAL_ADDRESS,
+    get_docker_client,
+    get_existing_container_with_matching_env,
+    start_tinybird_local,
+)
 
 
 def stop_tinybird_local(docker_client: DockerClient) -> None:

tinybird/tb/modules/local_common.py

@@ -384,11 +384,15 @@ def get_docker_client() -> DockerClient:
             raise e
         return client
     except Exception:
+        docker_location_message = ""
+        if docker_host:
+            docker_location_message = f"Trying to connect to Docker-compatible runtime at {docker_host}"
+
         raise CLILocalException(
             FeedbackManager.error(
                 message=(
                     f"No container runtime is running. Make sure a Docker-compatible runtime is installed and running. "
-                    f"Trying to connect to Docker-compatible runtime at {docker_host}\n\n"
+                    f"{docker_location_message}\n\n"
                     "If you're using a custom location, please provide it using the DOCKER_HOST environment variable."
                 )
             )

tinybird/tb/modules/materialization.py

@@ -3,6 +3,7 @@ import re
 
 import click
 
+from tinybird.datafile.common import PipeTypes, get_name_version
 from tinybird.tb.client import TinyB
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import (
@@ -10,7 +11,6 @@ from tinybird.tb.modules.common import (
     echo_safe_humanfriendly_tables_format_smart_table,
     wait_job,
 )
-from tinybird.tb.modules.datafile.common import PipeTypes, get_name_version
 from tinybird.tb.modules.exceptions import CLIPipeException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 

tinybird/tb/modules/pipe.py

@@ -9,13 +9,13 @@ import re
 import click
 from click import Context
 
+from tinybird.datafile.common import get_name_version
 from tinybird.tb.client import TinyB
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import (
     coro,
     echo_safe_humanfriendly_tables_format_smart_table,
 )
-from tinybird.tb.modules.datafile.common import get_name_version
 from tinybird.tb.modules.exceptions import CLIPipeException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 

tinybird/tb/modules/project.py

@@ -3,9 +3,9 @@ import re
 from pathlib import Path
 from typing import Dict, List, Optional
 
-from tinybird.tb.modules.datafile.common import Datafile
-from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
-from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
+from tinybird.datafile.common import Datafile
+from tinybird.datafile.parse_datasource import parse_datasource
+from tinybird.datafile.parse_pipe import parse_pipe
 
 
 class Project:
@@ -125,6 +125,8 @@ class Project:
                 return "materialization"
             elif re.search(r"TYPE copy", content, re.IGNORECASE):
                 return "copy"
+            elif re.search(r"TYPE sink", content, re.IGNORECASE):
+                return "sink"
             return "pipe"
         except Exception:
             return "pipe"

tinybird/tb/modules/sink.py

@@ -0,0 +1,105 @@
+import json
+import re
+from typing import Optional, Tuple
+
+import click
+from click import Context
+
+from tinybird.datafile.common import get_name_version
+from tinybird.tb.client import AuthNoTokenException, TinyB
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.common import coro, echo_safe_humanfriendly_tables_format_smart_table, wait_job
+from tinybird.tb.modules.exceptions import CLIPipeException
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+
+
+@cli.group()
+@click.pass_context
+def sink(ctx):
+    """Sink pipe commands."""
+
+
+@sink.command(name="ls")
+@click.option("--match", default=None, help="Retrieve any resourcing matching the pattern. eg --match _test")
+@click.option(
+    "--format",
+    "format_",
+    type=click.Choice(["json"], case_sensitive=False),
+    default=None,
+    help="Force a type of the output",
+)
+@click.pass_context
+@coro
+async def sink_ls(ctx: Context, match: str, format_: str):
+    """List sink pipes"""
+
+    client: TinyB = ctx.ensure_object(dict)["client"]
+    pipes = await client.pipes(dependencies=False, node_attrs="name", attrs="name,updated_at,type")
+    sinks = [p for p in pipes if p.get("type") == "sink"]
+    sinks = sorted(sinks, key=lambda p: p["updated_at"])
+    columns = ["name", "updated at", "nodes"]
+    table_human_readable = []
+    table_machine_readable = []
+    pattern = re.compile(match) if match else None
+    for t in sinks:
+        tk = get_name_version(t["name"])
+        if pattern and not pattern.search(tk["name"]):
+            continue
+        table_human_readable.append((tk["name"], t["updated_at"][:-7], len(t["nodes"])))
+        table_machine_readable.append(
+            {
+                "name": tk["name"],
+                "updated at": t["updated_at"][:-7],
+                "nodes": len(t["nodes"]),
+            }
+        )
+
+    if not format_:
+        click.echo(FeedbackManager.info_pipes())
+        echo_safe_humanfriendly_tables_format_smart_table(table_human_readable, column_names=columns)
+        click.echo("\n")
+    elif format_ == "json":
+        click.echo(json.dumps({"pipes": table_machine_readable}, indent=2))
+    else:
+        raise CLIPipeException(FeedbackManager.error_pipe_ls_type())
+
+
+@sink.command(name="run", short_help="Run an on-demand sink job")
+@click.argument("pipe_name_or_id")
+@click.option("--wait", is_flag=True, default=False, help="Wait for the sink job to finish")
+@click.option(
+    "--mode", type=click.Choice(["append", "replace"], case_sensitive=True), default=None, help="Sink strategy"
+)
+@click.option(
+    "--param",
+    nargs=1,
+    type=str,
+    multiple=True,
+    default=None,
+    help="Key and value of the params you want the Sink pipe to be called with. For example: tb pipe sink run <my_sink_pipe> --param foo=bar",
+)
+@click.pass_context
+@coro
+async def sink_run(ctx: click.Context, pipe_name_or_id: str, wait: bool, mode: str, param: Optional[Tuple[str]]):
+    """Run a sink pipe"""
+
+    params = dict(key_value.split("=") for key_value in param) if param else {}
+    click.echo(FeedbackManager.highlight(message=f"\n» Running sink '{pipe_name_or_id}'"))
+    client: TinyB = ctx.ensure_object(dict)["client"]
+
+    try:
+        response = await client.pipe_run(pipe_name_or_id, "sink", params, mode)
+        job_id = response["job"]["id"]
+        job_url = response["job"]["job_url"]
+        click.echo(FeedbackManager.gray(message="Job URL: ") + FeedbackManager.info(message=f"{job_url}"))
+        click.echo(FeedbackManager.success(message="✓ Sink job created"))
+
+        if wait:
+            click.echo("\n")
+            await wait_job(client, job_id, job_url, FeedbackManager.highlight(message="» Exporting data"))
+            click.echo(FeedbackManager.success(message="✓ Data exported"))
+
+    except AuthNoTokenException:
+        raise
+    except Exception as e:
+        raise CLIPipeException(FeedbackManager.error_creating_sink_job(error=e))

tinybird/tb/modules/watch.py

@@ -15,7 +15,7 @@ from watchdog.events import (
 )
 from watchdog.observers import Observer
 
-from tinybird.tb.modules.datafile.common import Datafile, DatafileKind
+from tinybird.datafile.common import Datafile, DatafileKind
 from tinybird.tb.modules.datafile.fixture import FixtureExtension
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.project import Project

{tinybird-0.0.1.dev212.dist-info → tinybird-0.0.1.dev214.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev212
+Version: 0.0.1.dev214
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird