tinybird 0.0.1.dev66__tar.gz → 0.0.1.dev68__tar.gz

tinybird-0.0.1.dev68/PKG-INFO

@@ -0,0 +1,64 @@
+Metadata-Version: 2.1
+Name: tinybird
+Version: 0.0.1.dev68
+Summary: Tinybird Command Line Tool
+Home-page: https://www.tinybird.co/docs/cli/introduction.html
+Author: Tinybird
+Author-email: support@tinybird.co
+Requires-Python: >=3.9, <3.13
+Description-Content-Type: text/x-rst
+Requires-Dist: aiofiles==24.1.0
+Requires-Dist: anthropic==0.42.0
+Requires-Dist: click<8.2,>=8.1.6
+Requires-Dist: clickhouse-toolset==0.33.dev0
+Requires-Dist: colorama==0.4.6
+Requires-Dist: cryptography~=41.0.0
+Requires-Dist: croniter==1.3.8
+Requires-Dist: docker==7.1.0
+Requires-Dist: GitPython~=3.1.32
+Requires-Dist: humanfriendly~=8.2
+Requires-Dist: prompt_toolkit==3.0.48
+Requires-Dist: pydantic~=2.8.0
+Requires-Dist: pyperclip==1.8.2
+Requires-Dist: pyyaml<6.1,>=6.0
+Requires-Dist: requests<3,>=2.28.1
+Requires-Dist: shandy-sqlfmt==0.11.1
+Requires-Dist: shandy-sqlfmt[jinjafmt]==0.11.1
+Requires-Dist: toposort==1.10
+Requires-Dist: tornado~=6.0.0
+Requires-Dist: urllib3<2,>=1.26.14
+Requires-Dist: watchdog==6.0.0
+Requires-Dist: wheel
+Requires-Dist: packaging<24,>=23.1
+Requires-Dist: llm>=0.19
+Requires-Dist: thefuzz==0.22.1
+Provides-Extra: bigquery
+Requires-Dist: gsutil==4.58; extra == "bigquery"
+Requires-Dist: google-api-python-client==2.0.2; extra == "bigquery"
+Requires-Dist: google-auth==1.27.1; extra == "bigquery"
+Requires-Dist: google-auth-httplib2==0.1.0; extra == "bigquery"
+Requires-Dist: google-cloud-storage==2.4.0; extra == "bigquery"
+Requires-Dist: google-cloud-bigquery==2.11.0; extra == "bigquery"
+Provides-Extra: snowflake
+Requires-Dist: snowflake-connector-python~=3.12.3; extra == "snowflake"
+Requires-Dist: gsutil==4.58; extra == "snowflake"
+Requires-Dist: google-api-python-client==2.0.2; extra == "snowflake"
+Requires-Dist: google-auth==1.27.1; extra == "snowflake"
+Requires-Dist: google-auth-httplib2==0.1.0; extra == "snowflake"
+Requires-Dist: google-cloud-storage==2.4.0; extra == "snowflake"
+Requires-Dist: oauth2client==3.0.0; extra == "snowflake"
+Requires-Dist: chardet<4,>=3.0.2; extra == "snowflake"
+Requires-Dist: pyOpenSSL<20.0.0,>=16.2.0; extra == "snowflake"
+
+Tinybird CLI
+=============
+
+The Tinybird command-line tool allows you to use all the Tinybird functionality directly from the command line. Additionally, it includes several functions to create and manage data projects easily.
+
+Changelog
+----------
+
+0.0.1dev1
+***********
+
+* Initial release of the Tinybird CLI

{tinybird-0.0.1.dev66 → tinybird-0.0.1.dev68}/tinybird/prompts.py

@@ -699,8 +699,8 @@ pipe_instructions = """
     - Nodes can't have the same exact name as the Pipe they belong to.
     - Avoid more than one node per pipe unless it is really necessary or requested by the user.
     - No indentation is allowed for property names: DESCRIPTION, NODE, SQL, TYPE, etc.
-    - Allowed TYPE values are: endpoint, copy, materialized, sink
-    - Add always the output node in the TYPE section.
+    - Allowed TYPE values are: endpoint, copy, materialized.
+    - Add always the output node in the TYPE section or in the last node of the pipe.
 </pipe_file_instructions>
 """
 
@@ -788,7 +788,7 @@ When you need to work with resources or data in cloud, add always the --cloud fl
 ├── endpoints
 ├── fixtures
 ├── materializations
-├── sinks
+├── pipes
 └── tests
 - The local development server will be available at http://localhost:80. Even if some response uses another base url, use always http://localhost:80.
 - After every change in your .datasource, .pipe or .ndjson files, run `{base_command} build` to build the project locally.

{tinybird-0.0.1.dev66 → tinybird-0.0.1.dev68}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev66'
-__revision__ = '449a9aa'
+__version__ = '0.0.1.dev68'
+__revision__ = 'e28d545'

{tinybird-0.0.1.dev66 → tinybird-0.0.1.dev68}/tinybird/tb/modules/cli.py

@@ -392,7 +392,7 @@ async def create_ctx_client(ctx: Context, config: Dict[str, Any], cloud: bool, b
         return None
 
     commands_always_cloud = ["pull"]
-    commands_always_build = ["build", "test"]
+    commands_always_build = ["build", "test", "dev"]
     commands_always_local = ["create", "mock"]
     if (
         (cloud or command in commands_always_cloud)

{tinybird-0.0.1.dev66 → tinybird-0.0.1.dev68}/tinybird/tb/modules/common.py

@@ -33,6 +33,7 @@ from click import Context
 from click._termui_impl import ProgressBar
 from humanfriendly.tables import format_pretty_table
 from packaging.version import Version
+from thefuzz import process
 
 from tinybird.client import (
     AuthException,
@@ -277,6 +278,77 @@ variable to '1' or 'true'."""
 
         sys.exit(exit_code)
 
+    def get_command(self, ctx: click.Context, cmd_name: str) -> Optional[click.Command]:
+        """
+        Override get_command to add suggestion functionality.
+
+        Args:
+            ctx: Click context
+            cmd_name: Command name entered by user
+
+        Returns:
+            Click command if found, None otherwise
+        """
+        # First, try to get the command normally
+        rv = click.Group.get_command(self, ctx, cmd_name)
+        if rv is not None:
+            return rv
+
+        # Get all available commands
+        commands: List[str] = self.list_commands(ctx)
+
+        # Find closest matching command using thefuzz
+        matches = process.extract(cmd_name, commands, limit=1)
+        if not matches:
+            return None
+
+        suggestion, score = matches[0]
+
+        # Only suggest if the similarity score is high enough (adjust threshold as needed)
+        if score >= 65:
+            # Initialize lists for arguments before and after command
+            args_before_cmd: List[str] = []
+            args_after_cmd: List[str] = []
+
+            # Reconstruct the full command with all arguments
+            full_command = ctx.command_path  # Gets the base command (e.g., 'tb')
+
+            # Add any options/arguments that came before the command
+            if ctx.args:
+                found_cmd = False
+                for arg in ctx.args:
+                    if arg == cmd_name:
+                        found_cmd = True
+                    elif found_cmd:
+                        args_after_cmd.append(arg)
+                    else:
+                        args_before_cmd.append(arg)
+
+                if args_before_cmd:
+                    full_command += " " + " ".join(args_before_cmd)
+
+            # Add any options from params
+            if ctx.params:
+                for param, value in ctx.params.items():
+                    if value is True:
+                        full_command += f" --{param}"
+                    elif value is not False and value is not None:
+                        full_command += f" --{param} {value}"
+
+            # Add the suggested command
+            full_command += f" {suggestion}"
+
+            # Add any arguments that came after the command
+            if args_after_cmd:
+                full_command += " " + " ".join(args_after_cmd)
+
+            click.echo(f"\nWARNING: '{cmd_name}' is not a valid command. ", nl=False)
+            # Ask for confirmation
+            if click.confirm(f"Execute '{full_command}' instead?"):
+                return click.Group.get_command(self, ctx, suggestion)
+
+        return None
+
 
 def load_connector_config(ctx: Context, connector_name: str, debug: bool, check_uninstalled: bool = False):
     config_file = Path(getcwd()) / f".tinyb_{connector_name}"

{tinybird-0.0.1.dev66 → tinybird-0.0.1.dev68}/tinybird/tb/modules/create.py

@@ -35,17 +35,9 @@ from tinybird.tb.modules.project import Project
 )
 @click.option("--rows", type=int, default=10, help="Number of events to send")
 @click.option("--source", type=str, default="tb", help="Source of the command")
-@click.option("--skip", is_flag=True, default=False, help="Skip following up on the generated resources")
 @click.pass_context
 @coro
-async def create(
-    ctx: click.Context,
-    data: Optional[str],
-    prompt: Optional[str],
-    rows: int,
-    source: str,
-    skip: bool,
-) -> None:
+async def create(ctx: click.Context, data: Optional[str], prompt: Optional[str], rows: int, source: str) -> None:
     """Initialize a new project."""
     project: Project = ctx.ensure_object(dict)["project"]
     local_client: TinyB = ctx.ensure_object(dict)["client"]
@@ -80,7 +72,7 @@ async def create(
         result = ""
         if data or prompt:
             click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
-            result = await create_resources(local_client, tb_client, user_token, data, prompt, folder, skip)
+            result = await create_resources(local_client, tb_client, user_token, data, prompt, folder)
             click.echo(FeedbackManager.success(message="✓ Done!\n"))
 
         if not already_has_cicd(folder):
@@ -125,7 +117,7 @@ async def create(
         click.echo(FeedbackManager.error(message=f"Error: {str(e)}"))
 
 
-PROJECT_PATHS = ("datasources", "endpoints", "materializations", "copies", "sinks", "fixtures", "tests")
+PROJECT_PATHS = ("datasources", "endpoints", "materializations", "copies", "pipes", "fixtures", "tests")
 
 
 def validate_project_structure(folder: str) -> bool:
@@ -166,7 +158,6 @@ async def create_resources(
     data: Optional[str],
     prompt: Optional[str],
     folder: str,
-    skip: bool,
 ):
     result = ""
     folder_path = Path(folder)
@@ -215,73 +206,39 @@ TYPE ENDPOINT
             ]
         )
         llm = LLM(user_token=user_token, host=tb_client.host)
-        result = ""
-        iterations = 0
-        history = ""
-        generated_paths: list[Path] = []
-
-        while iterations < 10:
-            feedback = ""
-            if iterations > 0:
-                feedback = click.prompt("\nFollow-up instructions or continue", default="continue")
-            if iterations > 0 and (not feedback or feedback in ("continue", "ok", "exit", "quit", "q")):
-                break
-            else:
-                if iterations > 0:
-                    click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
-                    for path in generated_paths:
-                        path.unlink()
-                    generated_paths = []
-
-                save_context(prompt, feedback)
-                result = llm.ask(system_prompt=create_prompt(resources_xml, feedback, history), prompt=prompt)
-                result = extract_xml(result, "response")
-                history = (
-                    history
-                    + f"""
-                <result_iteration_{iterations}>
-                    {result}
-                </result_iteration_{iterations}>
-                """
-                )
-                resources = parse_xml(result, "resource")
-                datasources = []
-                pipes = []
-                for resource_xml in resources:
-                    resource_type = extract_xml(resource_xml, "type")
-                    name = extract_xml(resource_xml, "name")
-                    content = extract_xml(resource_xml, "content")
-                    resource = {
-                        "name": name,
-                        "content": content,
-                    }
-                    if resource_type.lower() == "datasource":
-                        datasources.append(resource)
-                    elif resource_type.lower() == "pipe":
-                        pipes.append(resource)
-
-                for ds in datasources:
-                    content = ds["content"].replace("```", "")
-                    filename = f"{ds['name']}.datasource"
-                    datasource_path = generate_datafile(
-                        content,
-                        filename=filename,
-                        data=None,
-                        _format="ndjson",
-                        force=True,
-                        folder=folder,
-                    )
-                    generated_paths.append(datasource_path)
-                for pipe in pipes:
-                    content = pipe["content"].replace("```", "")
-                    pipe_path = generate_pipe_file(pipe["name"], content, folder)
-                    generated_paths.append(pipe_path)
-                if skip:
-                    break
-                iterations += 1
-
-        if iterations == 10:
-            click.echo(FeedbackManager.info(message="Too many iterations. Change the prompt and try again."))
+        result = llm.ask(system_prompt=create_prompt(resources_xml), prompt=prompt)
+        result = extract_xml(result, "response")
+        resources = parse_xml(result, "resource")
+        datasources = []
+        pipes = []
+        for resource_xml in resources:
+            resource_type = extract_xml(resource_xml, "type")
+            name = extract_xml(resource_xml, "name")
+            content = extract_xml(resource_xml, "content")
+            resource = {
+                "name": name,
+                "content": content,
+            }
+            if resource_type.lower() == "datasource":
+                datasources.append(resource)
+            elif resource_type.lower() == "pipe":
+                pipes.append(resource)
+
+        for ds in datasources:
+            content = ds["content"].replace("```", "")
+            filename = f"{ds['name']}.datasource"
+            generate_datafile(
+                content,
+                filename=filename,
+                data=None,
+                _format="ndjson",
+                force=True,
+                folder=folder,
+            )
+
+        for pipe in pipes:
+            content = pipe["content"].replace("```", "")
+            generate_pipe_file(pipe["name"], content, folder)
 
         return result
 
@@ -313,14 +270,19 @@ def generate_pipe_file(name: str, content: str, folder: str) -> Path:
     def is_sink(content: str) -> bool:
         return re.search(r"TYPE sink", content, re.IGNORECASE) is not None
 
+    def is_endpoint(content: str) -> bool:
+        return re.search(r"TYPE endpoint", content, re.IGNORECASE) is not None
+
     if is_copy(content):
         pathname = "copies"
     elif is_materialization(content):
         pathname = "materializations"
     elif is_sink(content):
         pathname = "sinks"
-    else:
+    elif is_endpoint(content):
         pathname = "endpoints"
+    else:
+        pathname = "pipes"
 
     base = Path(folder) / pathname
     if not base.exists():

{tinybird-0.0.1.dev66 → tinybird-0.0.1.dev68}/tinybird/tb/modules/datafile/common.py

@@ -232,8 +232,6 @@ class Datafile:
             node = self.nodes[0]
             if "schema" not in node:
                 raise DatafileValidationError("SCHEMA is mandatory")
-            if "engine" not in node:
-                raise DatafileValidationError("ENGINE is mandatory")
         else:
             # We cannot validate a datafile whose kind is unknown
             pass

{tinybird-0.0.1.dev66 → tinybird-0.0.1.dev68}/tinybird/tb/modules/deployment.py

@@ -58,6 +58,8 @@ def promote_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
     result = r.json()
     logging.debug(json.dumps(result, indent=2))
 
+    click.echo(FeedbackManager.success(message="Deployment promotion successfully started"))
+
     if wait:
         while True:
             TINYBIRD_API_URL = f"{host}/v1/deployments/{last_deployment.get('id')}"
@@ -67,12 +69,11 @@ def promote_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
 
             last_deployment = result.get("deployment")
             if last_deployment.get("status") == "deleted":
+                click.echo(FeedbackManager.success(message="Deployment promoted successfully"))
                 break
 
             time.sleep(5)
 
-    click.echo(FeedbackManager.success(message="Deployment promoted successfully"))
-
 
 # TODO(eclbg): This logic should be in the server, and there should be a dedicated endpoint for rolling back a
 # deployment
@@ -117,6 +118,8 @@ def rollback_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
     result = r.json()
     logging.debug(json.dumps(result, indent=2))
 
+    click.echo(FeedbackManager.success(message="Deployment rollback successfully started"))
+
     if wait:
         while True:
             TINYBIRD_API_URL = f"{host}/v1/deployments/{current_deployment.get('id')}"
@@ -126,11 +129,10 @@ def rollback_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
 
             current_deployment = result.get("deployment")
             if current_deployment.get("status") == "deleted":
+                click.echo(FeedbackManager.success(message="Deployment rolled back successfully"))
                 break
             time.sleep(5)
 
-    click.echo(FeedbackManager.success(message="Deployment rolled back successfully"))
-
 
 @cli.group(name="deployment")
 def deployment_group() -> None:
@@ -159,12 +161,18 @@ def deployment_group() -> None:
     default=False,
     help="Validate the deployment before creating it. Disabled by default.",
 )
+@click.option(
+    "--allow-remove-datasources/--no-allow-remove-datasources",
+    is_flag=True,
+    default=False,
+    help="Allow removing datasources. Disabled by default.",
+)
 @click.pass_context
-def deployment_create(ctx: click.Context, wait: bool, auto: bool, check: bool) -> None:
+def deployment_create(ctx: click.Context, wait: bool, auto: bool, check: bool, allow_remove_datasources: bool) -> None:
     """
     Validate and deploy the project server side.
     """
-    create_deployment(ctx, wait, auto, check)
+    create_deployment(ctx, wait, auto, check, allow_remove_datasources)
 
 
 @deployment_group.command(name="ls")
@@ -187,6 +195,9 @@ def deployment_ls(ctx: click.Context) -> None:
     columns = ["ID", "Status", "Created at", "Live"]
     table = []
     for deployment in result.get("deployments"):
+        if deployment.get("id") == "0":
+            continue
+
         table.append(
             [
                 deployment.get("id"),
@@ -258,15 +269,27 @@ def deployment_rollback(ctx: click.Context, wait: bool) -> None:
     default=False,
     help="Validate the deployment before creating it. Disabled by default.",
 )
+@click.option(
+    "--allow-remove-datasources/--no-allow-remove-datasources",
+    is_flag=True,
+    default=False,
+    help="Allow removing datasources. Disabled by default.",
+)
 @click.pass_context
-def deploy(ctx: click.Context, wait: bool, auto: bool, check: bool) -> None:
+def deploy(ctx: click.Context, wait: bool, auto: bool, check: bool, allow_remove_datasources: bool) -> None:
     """
     Deploy the project.
     """
-    create_deployment(ctx, wait, auto, check)
+    create_deployment(ctx, wait, auto, check, allow_remove_datasources)
 
 
-def create_deployment(ctx: click.Context, wait: bool, auto: bool, check: Optional[bool] = None) -> None:
+def create_deployment(
+    ctx: click.Context,
+    wait: bool,
+    auto: bool,
+    check: Optional[bool] = None,
+    allow_remove_datasources: Optional[bool] = None,
+) -> None:
     # TODO: This code is duplicated in build_server.py
     # Should be refactored to be shared
     MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
@@ -297,21 +320,31 @@ def create_deployment(ctx: click.Context, wait: bool, auto: bool, check: Optiona
         if check:
             click.echo(FeedbackManager.highlight(message="\n» Validating deployment...\n"))
             params["check"] = "true"
+        if allow_remove_datasources:
+            params["allow_remove_datasources"] = "true"
         r = requests.post(TINYBIRD_API_URL, files=files, headers=HEADERS, params=params)
         result = r.json()
         logging.debug(json.dumps(result, indent=2))
 
         if check:
             print_changes(result, project)
+            feedback = result.get("deployment", {}).get("feedback", [])
+            for f in feedback:
+                click.echo(
+                    FeedbackManager.warning(message=f"△ {f.get('level')}: {f.get('resource')}: {f.get('message')}")
+                )
+
             status = result.get("result")
             if status == "success":
                 click.echo(FeedbackManager.success(message="\n✓ Deployment is valid"))
             else:
                 click.echo(FeedbackManager.error(message="\n✗ Deployment is not valid"))
+
             return
 
         deploy_result = result.get("result")
         if deploy_result == "success":
+            print_changes(result, project)
             click.echo(FeedbackManager.success(message="Deployment submitted successfully"))
             deployment = result.get("deployment")
             feedback = deployment.get("feedback", [])
@@ -394,4 +427,8 @@ def print_changes(result: dict, project: Project) -> None:
     for p in deployment.get("deleted_pipe_names", []):
         resources.append(["deleted", p, project.get_resource_path(p, "pipe")])
 
-    echo_safe_humanfriendly_tables_format_smart_table(resources, column_names=columns)
+    if resources:
+        click.echo(FeedbackManager.highlight(message="\n» Changes to be deployed...\n"))
+        echo_safe_humanfriendly_tables_format_smart_table(resources, column_names=columns)
+    else:
+        click.echo(FeedbackManager.highlight(message="\n» No changes to be deployed\n"))

{tinybird-0.0.1.dev66 → tinybird-0.0.1.dev68}/tinybird/tb/modules/mock.py

@@ -2,7 +2,6 @@ import glob
 import logging
 import os
 from pathlib import Path
-from typing import Optional
 
 import click
 
@@ -27,10 +26,9 @@ from tinybird.tb.modules.project import Project
     default="",
     help="Extra context to use for data generation",
 )
-@click.option("--skip", is_flag=True, default=False, help="Skip following up on the generated data")
 @click.pass_context
 @coro
-async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, skip: bool) -> None:
+async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str) -> None:
     """Generate sample data for a datasource.
 
     Args:
@@ -80,44 +78,18 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, skip
             return
         llm = LLM(user_token=user_token, host=user_client.host)
         prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
-        iterations = 0
-        history = ""
-        fixture_path: Optional[Path] = None
         sql = ""
-        while iterations < 10:
-            feedback = ""
-            if iterations > 0:
-                feedback = click.prompt("\nFollow-up instructions or continue", default="continue")
-            if iterations > 0 and (not feedback or feedback in ("continue", "ok", "exit", "quit", "q")):
-                break
-            else:
-                if iterations > 0:
-                    if fixture_path:
-                        fixture_path.unlink()
-                    fixture_path = None
-                    click.echo(FeedbackManager.highlight(message=f"\n» Creating fixture for {datasource_name}..."))
 
-                response = llm.ask(system_prompt=mock_prompt(rows, feedback, history), prompt=prompt)
-                sql = extract_xml(response, "sql")
-                result = await tb_client.query(f"{sql} FORMAT JSON")
-                data = result.get("data", [])[:rows]
-                fixture_path = persist_fixture(datasource_name, data, folder)
-                click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}.ndjson created"))
-
-                if os.environ.get("TB_DEBUG", "") != "":
-                    logging.debug(sql)
-
-                history = (
-                    history
-                    + f"""
-                <result_iteration_{iterations}>
-                    {response}
-                </result_iteration_{iterations}>
-                """
-                )
-                if skip:
-                    break
-                iterations += 1
+        click.echo(FeedbackManager.highlight(message=f"\n» Creating fixture for {datasource_name}..."))
+        response = llm.ask(system_prompt=mock_prompt(rows), prompt=prompt)
+        sql = extract_xml(response, "sql")
+        result = await tb_client.query(f"{sql} FORMAT JSON")
+        data = result.get("data", [])[:rows]
+        persist_fixture(datasource_name, data, folder)
+        click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}.ndjson created"))
+
+        if os.environ.get("TB_DEBUG", "") != "":
+            logging.debug(sql)
 
         click.echo(FeedbackManager.success(message=f"✓ Sample data for {datasource_name} created with {rows} rows"))