tinybird 0.0.1.dev47__tar.gz → 0.0.1.dev49__tar.gz

{tinybird-0.0.1.dev47 → tinybird-0.0.1.dev49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird
-Version: 0.0.1.dev47
+Version: 0.0.1.dev49
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev47 → tinybird-0.0.1.dev49}/tinybird/client.py

@@ -697,12 +697,7 @@ class TinyB:
         if local_host != self.host:
             return data
 
-        local_workspaces = [
-            x
-            for x in data["workspaces"]
-            if x["name"].startswith("Tinybird_Local_") and not x["name"].startswith("Tinybird_Local_Build_")
-        ]
-
+        local_workspaces = [x for x in data["workspaces"] if not x["name"].startswith("Tinybird_Local_")]
         return {**data, "workspaces": local_workspaces}
 
     async def user_workspaces_and_branches(self):

{tinybird-0.0.1.dev47 → tinybird-0.0.1.dev49}/tinybird/prompts.py

@@ -651,32 +651,6 @@ DATASOURCE sales_by_hour
 """
 
 
-def ask_prompt(existing_resources: str) -> str:
-    return """
-You are a Tinybird expert. You will be given a prompt to ask questions about Tinybird resources.
-<existing_resources>{existing_resources}</existing_resources>
-{datasource_instructions}
-{pipe_instructions}
-{sql_instructions}
-{datasource_example}
-{pipe_example}
-{copy_pipe_instructions}
-{materialized_pipe_instructions}
-
-The previous instructions are explanations of how things work in Tinybird. Answer in natural language.
-
-""".format(
-        existing_resources=existing_resources,
-        datasource_instructions=datasource_instructions,
-        datasource_example=datasource_example,
-        pipe_instructions=pipe_instructions,
-        pipe_example=pipe_example,
-        sql_instructions=sql_instructions,
-        copy_pipe_instructions=copy_pipe_instructions,
-        materialized_pipe_instructions=materialized_pipe_instructions,
-    )
-
-
 datasource_instructions = """
 <datasource_file_instructions>
     - Content cannot be empty.
@@ -790,39 +764,6 @@ sql_instructions = """
 )
 
 
-def update_prompt(existing_resources: str) -> str:
-    return """
-You are a Tinybird expert. You will be given a prompt to update the existing Tinybird resources: datasources and/or pipes. 
-You will return the resources that need to be updated.
-<existing_resources>{existing_resources}</existing_resources>
-{datasource_instructions}
-{pipe_instructions}
-{sql_instructions}
-{datasource_example}
-{pipe_example}
-{copy_pipe_instructions}
-{materialized_pipe_instructions}
-Use the following format to generate the response and do not wrap it in any other text, including the <response> tag.
-<response>
-    <resource>
-        <type>[datasource or pipe]</type>
-        <name>[resource name here]</name>
-        <content>[resource content here]</content>
-    </resource>
-</response>
-
-""".format(
-        existing_resources=existing_resources,
-        datasource_instructions=datasource_instructions,
-        pipe_instructions=pipe_instructions,
-        sql_instructions=sql_instructions,
-        datasource_example=datasource_example,
-        pipe_example=pipe_example,
-        copy_pipe_instructions=copy_pipe_instructions,
-        materialized_pipe_instructions=materialized_pipe_instructions,
-    )
-
-
 def rules_prompt(source: Optional[str] = None) -> str:
     base_command = source or "tb"
     return """
@@ -837,7 +778,8 @@ You have commands at your disposal to develop a tinybird project:
 - {base_command} --build endpoint data <pipe_name>: to get the data of an endpoint. You can pass parameters to the endpoint like this: {base_command} --build endpoint data <pipe_name> --param1 value1 --param2 value2
 - {base_command} --build token ls: to list all the tokens
 There are other commands that you can use, but these are the most common ones. Run `{base_command} -h` to see all the commands if needed.
-When you need to check resources or data in the Tinybird environment that you updated with the build command, add always the --build flag before the command. Example: {base_command} --build datasource ls
+When you need to work with resources or data in the Tinybird environment that you updated with the build command, add always the --build flag before the command. Example: {base_command} --build datasource ls
+When you need to work with resources or data in production, add always the --prod flag before the command. Example: {base_command} --prod datasource ls
 </command_calling>
 <development_instructions>
 - When asking to create a tinybird data project, if the needed folders are not already created, use the following structure:
@@ -854,7 +796,12 @@ When you need to check resources or data in the Tinybird environment that you up
 - The format of the generated api endpoint urls is: http://localhost:80/v0/pipe/<pipe_name>.json?token=<token>
 - Before running the tests, remember to have the project built with `{base_command} build` with the latest changes.
 </development_instructions>
-
+When asking for ingesting data, adding data or appending data do the following depending on the environment you want to work with:
+<ingest_data_instructions>
+- When building locally, create a .ndjson file with the data you want to ingest and do `{base_command} build` to ingest the data in the build env.
+- When appending data in production, use `{base_command} --prod datasource append <datasource_name> <file_name>`
+- When you have a response that says “there are rows in quarantine”, do `{base_command} --build|--prod datasource data <datasource_name>_quarantine` to understand what is the problem.
+</ingest_data_instructions>
 <datasource_file_instructions>
 Follow these instructions when creating or updating .datasource files:
 {datasource_instructions}

{tinybird-0.0.1.dev47 → tinybird-0.0.1.dev49}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev47'
-__revision__ = '05e3b46'
+__version__ = '0.0.1.dev49'
+__revision__ = 'a0a2378'

{tinybird-0.0.1.dev47 → tinybird-0.0.1.dev49}/tinybird/tb/cli.py

@@ -23,7 +23,6 @@ import tinybird.tb.modules.pipe
 import tinybird.tb.modules.tag
 import tinybird.tb.modules.test
 import tinybird.tb.modules.token
-import tinybird.tb.modules.update
 import tinybird.tb.modules.workspace
 import tinybird.tb.modules.workspace_members
 

{tinybird-0.0.1.dev47 → tinybird-0.0.1.dev49}/tinybird/tb/modules/build.py

@@ -1,6 +1,7 @@
 import asyncio
 import json
 import logging
+import sys
 import threading
 import time
 from pathlib import Path
@@ -15,7 +16,6 @@ from tinybird.tb.modules.common import push_data
 from tinybird.tb.modules.datafile.build import folder_build
 from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
 from tinybird.tb.modules.feedback_manager import FeedbackManager
-from tinybird.tb.modules.local_common import get_tinybird_local_client
 from tinybird.tb.modules.project import Project
 from tinybird.tb.modules.shell import Shell, print_table_formatted
 from tinybird.tb.modules.watch import watch_project
@@ -29,7 +29,7 @@ def build(ctx: click.Context, watch: bool) -> None:
     Validate and build the project server side.
     """
     project: Project = ctx.ensure_object(dict)["project"]
-    tb_client = asyncio.run(get_tinybird_local_client(str(project.path), build=True))
+    tb_client: TinyB = ctx.ensure_object(dict)["client"]
     click.echo(FeedbackManager.highlight_building_project())
     time_start = time.time()
 
@@ -40,15 +40,21 @@ def build(ctx: click.Context, watch: bool) -> None:
             build_project(project, tb_client, file_changed)
             try:
                 if file_changed:
-                    asyncio.run(folder_build(project, filenames=[file_changed]))
-                    build_and_print_resource(tb_client, file_changed, diff)
+                    asyncio.run(folder_build(project, tb_client, filenames=[file_changed]))
+                    show_data(tb_client, file_changed, diff)
             except Exception:
                 pass
 
-    process()
-    time_end = time.time()
-    elapsed_time = time_end - time_start
-    click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s"))
+    try:
+        process()
+    except click.ClickException as e:
+        click.echo(e)
+        if not watch:
+            sys.exit(1)
+    else:
+        time_end = time.time()
+        elapsed_time = time_end - time_start
+        click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s"))
 
     if watch:
         shell = Shell(project=project, tb_client=tb_client)
@@ -137,9 +143,16 @@ def build_project(project: Project, tb_client: TinyB, file_changed: Optional[str
 
                         if fixture_path.exists():
                             append_fixture(tb_client, ds_name, str(fixture_path))
+
             except Exception:
                 pass
 
+            feedback = result.get("feedback", [])
+            for f in feedback:
+                click.echo(
+                    FeedbackManager.warning(message=f"△ {f.get('level')}: {f.get('resource')}: {f.get('message')}")
+                )
+
         elif build_result == "failed":
             build_errors = result.get("errors")
             error = True
@@ -198,7 +211,7 @@ def rebuild_fixture(project: Project, tb_client: TinyB, fixture: str) -> None:
         click.echo(FeedbackManager.error_exception(error=e))
 
 
-def build_and_print_resource(tb_client: TinyB, filename: str, diff: Optional[str] = None):
+def show_data(tb_client: TinyB, filename: str, diff: Optional[str] = None):
     table_name = diff
     resource_path = Path(filename)
     resource_name = resource_path.stem

{tinybird-0.0.1.dev47 → tinybird-0.0.1.dev49}/tinybird/tb/modules/cli.py

@@ -7,7 +7,6 @@ import json
 import logging
 import os
 from os import getcwd
-from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import click
@@ -20,7 +19,6 @@ from tinybird.client import (
     TinyB,
 )
 from tinybird.config import get_config
-from tinybird.prompts import ask_prompt
 from tinybird.tb import __cli__
 from tinybird.tb.modules.common import (
     CatchAuthExceptions,
@@ -37,7 +35,6 @@ from tinybird.tb.modules.datafile.build import build_graph
 from tinybird.tb.modules.datafile.diff import diff_command
 from tinybird.tb.modules.datafile.pull import folder_pull
 from tinybird.tb.modules.feedback_manager import FeedbackManager
-from tinybird.tb.modules.llm import LLM
 from tinybird.tb.modules.local_common import get_tinybird_local_client
 from tinybird.tb.modules.project import Project
 
@@ -124,13 +121,9 @@ async def cli(
 
     logging.debug("debug enabled")
 
-    skip_client = ctx.invoked_subcommand in ["auth", "check", "login", "local", "build"]
-    client = await create_ctx_client(config, prod, build, skip_client, project)
+    client = await create_ctx_client(ctx, config, prod, build, project)
 
     if client:
-        if not build:
-            target = config.get("name", "production") if prod else "Tinybird local"
-            click.echo(FeedbackManager.gray(message=f"Running against {target}\n"))
         ctx.ensure_object(dict)["client"] = client
 
     ctx.ensure_object(dict)["project"] = project
@@ -358,56 +351,6 @@ async def sql(
         click.echo(FeedbackManager.info_no_rows())
 
 
-@cli.command(hidden=True)
-@click.argument("prompt")
-@click.option("--folder", default=os.getcwd(), help="The folder to use for the project")
-@coro
-async def ask(prompt: str, folder: str) -> None:
-    """Ask things about your data project."""
-    try:
-        config = CLIConfig.get_project_config(folder)
-        user_token = config.get_user_token()
-
-        if not user_token:
-            raise CLIException("This action requires authentication. Run 'tb login' first.")
-
-        datasource_paths = [
-            Path(folder) / "datasources" / f
-            for f in os.listdir(Path(folder) / "datasources")
-            if f.endswith(".datasource")
-        ]
-        pipe_folders = ["endpoints", "pipes", "copies", "materializations", "sinks"]
-        pipe_paths = [
-            Path(folder) / pipe_folder / f
-            for pipe_folder in pipe_folders
-            if (Path(folder) / pipe_folder).exists()
-            for f in os.listdir(Path(folder) / pipe_folder)
-            if f.endswith(".pipe")
-        ]
-        resources_xml = "\n".join(
-            [
-                f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
-                for resource_type, resource_name, resource_content in [
-                    ("datasource", ds.stem, ds.read_text()) for ds in datasource_paths
-                ]
-                + [
-                    (
-                        "pipe",
-                        pipe.stem,
-                        pipe.read_text(),
-                    )
-                    for pipe in pipe_paths
-                ]
-            ]
-        )
-
-        client = config.get_client()
-        llm = LLM(user_token=user_token, host=client.host)
-        click.echo(llm.ask(system_prompt=ask_prompt(resources_xml), prompt=prompt))
-    except Exception as e:
-        raise CLIException(FeedbackManager.error_exception(error=e))
-
-
 def __patch_click_output():
     import re
 
@@ -425,7 +368,7 @@ def __patch_click_output():
             if isinstance(substitution, str):
                 msg = re.sub(pattern, substitution, str(msg))
             else:
-                msg = re.sub(pattern, lambda m: substitution(m.group(0)), str(msg))  # noqa
+                msg = re.sub(pattern, lambda m: substitution(m.group(0)), str(msg))  # noqa: B023
         return msg
 
     def _obfuscate_echo(msg: Any, *args: Any, **kwargs: Any) -> None:
@@ -445,11 +388,23 @@ def __unpatch_click_output():
     click.secho = __old_click_secho
 
 
-async def create_ctx_client(config: Dict[str, Any], prod: bool, build: bool, skip_client: bool, project: Project):
-    if skip_client:
+async def create_ctx_client(ctx: Context, config: Dict[str, Any], prod: bool, build: bool, project: Project):
+    commands_without_ctx_client = ["auth", "check", "login", "local"]
+    command = ctx.invoked_subcommand
+    if command in commands_without_ctx_client:
         return None
 
-    if prod:
+    commands_always_prod = ["pull"]
+    commands_always_build = ["build", "test"]
+    commands_always_local = ["create", "mock"]
+    if (
+        (prod or command in commands_always_prod)
+        and command not in commands_always_build
+        and command not in commands_always_local
+    ):
+        click.echo(FeedbackManager.gray(message=f"Running against {config.get('name') or 'production'}"))
         return _get_tb_client(config.get("token", None), config["host"])
-
+    build = command in commands_always_build or build
+    if not build and command not in commands_always_local and command not in commands_always_build:
+        click.echo(FeedbackManager.gray(message="Running against Tinybird local\n"))
     return await get_tinybird_local_client(str(project.path), build=build)

{tinybird-0.0.1.dev47 → tinybird-0.0.1.dev49}/tinybird/tb/modules/create.py

@@ -17,7 +17,6 @@ from tinybird.tb.modules.exceptions import CLIException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.llm import LLM
 from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
-from tinybird.tb.modules.local_common import get_tinybird_local_client
 from tinybird.tb.modules.project import Project
 
 
@@ -41,8 +40,8 @@ from tinybird.tb.modules.project import Project
     help="Folder where datafiles will be placed",
 )
 @click.option("--rows", type=int, default=10, help="Number of events to send")
-@click.option("--cursor", default=False, is_flag=True, help="Create .cursorrules file with Tinybird rules")
 @click.option("--source", type=str, default="tb", help="Source of the command")
+@click.option("--skip", is_flag=True, default=False, help="Skip following up on the generated data")
 @click.pass_context
 @coro
 async def create(
@@ -51,11 +50,12 @@ async def create(
     prompt: Optional[str],
     folder: Optional[str],
     rows: int,
-    cursor: bool,
     source: str,
+    skip: bool,
 ) -> None:
     """Initialize a new project."""
     project: Project = ctx.ensure_object(dict)["project"]
+    local_client: TinyB = ctx.ensure_object(dict)["client"]
     folder = folder or getcwd()
     folder_path = Path(folder)
     if not folder_path.exists():
@@ -78,7 +78,6 @@ async def create(
                     )
                 )
                 return
-        local_client = await get_tinybird_local_client(folder)
 
         if not validate_project_structure(folder):
             click.echo(FeedbackManager.highlight(message="\n» Creating new project structure..."))
@@ -88,7 +87,7 @@ async def create(
         result = ""
         if data or prompt:
             click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
-            result = await create_resources(local_client, tb_client, user_token, data, prompt, folder)
+            result = await create_resources(local_client, tb_client, user_token, data, prompt, folder, skip)
             click.echo(FeedbackManager.success(message="✓ Done!\n"))
 
         if not already_has_cicd(folder):
@@ -97,6 +96,11 @@ async def create(
             await init_cicd(data_project_dir=os.path.relpath(folder))
             click.echo(FeedbackManager.success(message="✓ Done!\n"))
 
+        if not already_has_cursor_rules(folder):
+            click.echo(FeedbackManager.highlight(message="\n» Creating .cursorrules..."))
+            create_rules(folder, source, "cursor")
+            click.echo(FeedbackManager.success(message="✓ Done!\n"))
+
         if should_generate_fixtures(result):
             click.echo(FeedbackManager.highlight(message="\n» Generating fixtures..."))
 
@@ -130,12 +134,6 @@ async def create(
                         if data:
                             persist_fixture(fixture_name, data, folder)
                             click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}"))
-
-        if cursor:
-            click.echo(FeedbackManager.highlight(message="\n» Creating .cursorrules..."))
-            create_rules(folder, source, "cursor")
-            click.echo(FeedbackManager.success(message="✓ Done!\n"))
-
     except Exception as e:
         click.echo(FeedbackManager.error(message=f"Error: {str(e)}"))
 
@@ -158,6 +156,11 @@ def already_has_cicd(folder: str) -> bool:
     return any((Path(folder) / path).exists() for path in ci_cd_paths)
 
 
+def already_has_cursor_rules(folder: str) -> bool:
+    cursor_rules_paths = (".cursorrules", ".windsurfrules")
+    return any((Path(folder) / path).exists() for path in cursor_rules_paths)
+
+
 def create_project_structure(folder: str):
     folder_path = Path(folder)
     for x in PROJECT_PATHS:
@@ -176,6 +179,7 @@ async def create_resources(
     data: Optional[str],
     prompt: Optional[str],
     folder: str,
+    skip: bool,
 ):
     result = ""
     folder_path = Path(folder)
@@ -285,7 +289,8 @@ TYPE ENDPOINT
                     content = pipe["content"].replace("```", "")
                     pipe_path = generate_pipe_file(pipe["name"], content, folder)
                     generated_paths.append(pipe_path)
-
+                if skip:
+                    break
                 iterations += 1
 
         if iterations == 10:

{tinybird-0.0.1.dev47 → tinybird-0.0.1.dev49}/tinybird/tb/modules/datafile/build.py

@@ -42,12 +42,12 @@ from tinybird.tb.modules.datafile.exceptions import AlreadyExistsException, Incl
 from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
 from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 from tinybird.tb.modules.feedback_manager import FeedbackManager
-from tinybird.tb.modules.local_common import get_tinybird_local_client
 from tinybird.tb.modules.project import Project
 
 
 async def folder_build(
     project: Project,
+    tb_client: TinyB,
     filenames: Optional[List[str]] = None,
     is_internal: bool = False,
     is_vendor: bool = False,
@@ -67,7 +67,6 @@ async def folder_build(
     fork = False
     release_created = False
     folder = str(project.path)
-    tb_client = await get_tinybird_local_client(folder)
     datasources: List[Dict[str, Any]] = await tb_client.datasources()
     pipes: List[Dict[str, Any]] = await tb_client.pipes(dependencies=True)
 

{tinybird-0.0.1.dev47 → tinybird-0.0.1.dev49}/tinybird/tb/modules/datafile/common.py

@@ -207,7 +207,7 @@ class Datafile:
             #  [x] SQL in all nodes
             #  [x] Materialized nodes have target datasource
             #  [x] Only one materialized node
-            #  [ ] Only one node of any specific type
+            #  [x] Only one node of any specific type
             #  [ ] ...
             repeated_node_names = [
                 name for name, count in filter(lambda x: x[1] > 1, Counter(n["name"] for n in self.nodes).items())
@@ -219,16 +219,14 @@ class Datafile:
             for node in self.nodes:
                 if "sql" not in node:
                     raise DatafileValidationError(f"SQL missing for node {repr(node['name'])}")
-            materialized_nodes_count = 0
+            non_standard_nodes_count = 0
             for node in self.nodes:
-                if node.get("type", "").lower() == "materialized":
-                    materialized_nodes_count += 1
-                    if materialized_nodes_count > 1:
-                        raise DatafileValidationError("Multiple materialized nodes in pipe. There can only be one")
-                    if "datasource" not in node:
-                        raise DatafileValidationError(
-                            f"Materialized node {repr(node['name'])} missing target datasource"
-                        )
+                if node.get("type", "").lower() not in {PipeNodeTypes.STANDARD, ""}:
+                    non_standard_nodes_count += 1
+                    if non_standard_nodes_count > 1:
+                        raise DatafileValidationError("Multiple non-standard nodes in pipe. There can only be one")
+                if node.get("type", "").lower() == PipeNodeTypes.MATERIALIZED and "datasource" not in node:
+                    raise DatafileValidationError(f"Materialized node {repr(node['name'])} missing target datasource")
         elif self.kind == DatafileKind.datasource:
             # TODO(eclbg):
             #  [x] Just one node

{tinybird-0.0.1.dev47 → tinybird-0.0.1.dev49}/tinybird/tb/modules/datasource.py

@@ -358,3 +358,36 @@ async def datasource_delete_rows(ctx, datasource_name, sql_condition, yes, wait,
             raise CLIDatasourceException(FeedbackManager.error_datasource_does_not_exist(datasource=datasource_name))
         except Exception as e:
             raise CLIDatasourceException(FeedbackManager.error_exception(error=e))
+
+
+@datasource.command(
+    name="data",
+    context_settings=dict(
+        allow_extra_args=True,
+        ignore_unknown_options=True,
+    ),
+)
+@click.argument("datasource")
+@click.option("--limit", type=int, default=5, help="Limit the number of rows to return")
+@click.pass_context
+@coro
+async def datasource_data(ctx: Context, datasource: str, limit: int):
+    """Print data returned by an endpoint
+
+    Syntax: tb datasource data <datasource_name>
+    """
+
+    client: TinyB = ctx.ensure_object(dict)["client"]
+    try:
+        res = await client.query(f"SELECT * FROM {datasource} LIMIT {limit} FORMAT JSON")
+    except AuthNoTokenException:
+        raise
+    except Exception as e:
+        raise CLIDatasourceException(FeedbackManager.error_exception(error=str(e)))
+
+    if not res["data"]:
+        click.echo(FeedbackManager.info_no_rows())
+    else:
+        echo_safe_humanfriendly_tables_format_smart_table(
+            data=[d.values() for d in res["data"]], column_names=res["data"][0].keys()
+        )