tinybird 0.0.1.dev32__tar.gz → 0.0.1.dev34__tar.gz

{tinybird-0.0.1.dev32 → tinybird-0.0.1.dev34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird
-Version: 0.0.1.dev32
+Version: 0.0.1.dev34
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev32 → tinybird-0.0.1.dev34}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev32'
-__revision__ = 'fc755d1'
+__version__ = '0.0.1.dev34'
+__revision__ = '08098c9'

{tinybird-0.0.1.dev32 → tinybird-0.0.1.dev34}/tinybird/tb/modules/build.py

@@ -11,6 +11,7 @@ import requests
 
 from tinybird.client import TinyB
 from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.common import push_data
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.local_common import get_tinybird_local_client
 from tinybird.tb.modules.shell import Shell
@@ -61,6 +62,7 @@ def build_project(folder: str, tb_client: TinyB) -> None:
         ".pipe": "text/plain",
     }
     TINYBIRD_API_URL = tb_client.host + "/v1/build"
+    logging.debug(TINYBIRD_API_URL)
     TINYBIRD_API_KEY = tb_client.token
     try:
         files = [
@@ -78,7 +80,12 @@ def build_project(folder: str, tb_client: TinyB) -> None:
         HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
 
         r = requests.post(TINYBIRD_API_URL, files=files, headers=HEADERS)
-        result = r.json()
+        try:
+            result = r.json()
+        except Exception as e:
+            logging.debug(e, exc_info=True)
+            click.echo(FeedbackManager.error(message="Couldn't parse response from server"))
+            return
 
         logging.debug(json.dumps(result, indent=2))
 
@@ -90,7 +97,7 @@ def build_project(folder: str, tb_client: TinyB) -> None:
             build_errors = result.get("errors")
             for build_error in build_errors:
                 filename_bit = f"{build_error.get('filename', '')}"
-                error_msg = (filename_bit + "\n\n") if filename_bit else "" + build_error.get("error")
+                error_msg = ((filename_bit + "\n\n") if filename_bit else "") + build_error.get("error")
                 click.echo(FeedbackManager.error(message=error_msg))
         else:
             click.echo(FeedbackManager.error(message=f"Unknown build result. Error: {result.get('error')}"))
@@ -99,3 +106,19 @@ def build_project(folder: str, tb_client: TinyB) -> None:
     finally:
         for fd in fds:
             fd.close()
+
+
+async def append_fixture(
+    tb_client: TinyB,
+    datasource_name: str,
+    url: str,
+):
+    await tb_client.datasource_truncate(datasource_name)
+    await push_data(
+        tb_client,
+        datasource_name,
+        url,
+        mode="append",
+        concurrency=1,
+        silent=True,
+    )

{tinybird-0.0.1.dev32 → tinybird-0.0.1.dev34}/tinybird/tb/modules/build_client.py

@@ -172,21 +172,7 @@ def build_client(
     build_ok = asyncio.run(build_once(filenames))
 
     if watch:
-        paths = [Path(f) for f in get_project_filenames(folder, with_vendor=True)]
-
-        datasource_paths = [f for f in paths if f.suffix == ".datasource"]
-        datasources = [f.stem for f in datasource_paths if not is_vendor(f)]
-        shared_datasources = [f"{get_vendor_workspace(f)}.{f.stem}" for f in datasource_paths if is_vendor(f)]
-        pipes = [f.stem for f in paths if is_pipe(f) and not is_endpoint(f)]
-        endpoints = [f.stem for f in paths if is_endpoint(f)]
-        shell = Shell(
-            folder=folder,
-            client=tb_client,
-            datasources=datasources,
-            shared_datasources=shared_datasources,
-            pipes=pipes,
-            endpoints=endpoints,
-        )
+        shell = Shell(folder=folder, client=tb_client)
         click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
         watcher_thread = threading.Thread(
             target=watch_files, args=(filenames, process, shell, folder, build_ok), daemon=True

{tinybird-0.0.1.dev32 → tinybird-0.0.1.dev34}/tinybird/tb/modules/cli.py

@@ -94,7 +94,7 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
     "--with-headers", help="Flag to enable connector to export with headers", is_flag=True, default=False, hidden=True
 )
 @click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
-@click.option("--prod", is_flag=True, default=False, help="Run against production")
+@click.option("--prod/--local", is_flag=True, default=False, help="Run against production or local")
 @click.version_option(version=VERSION)
 @click.pass_context
 @coro

{tinybird-0.0.1.dev32 → tinybird-0.0.1.dev34}/tinybird/tb/modules/datafile/build.py

@@ -44,7 +44,7 @@ from tinybird.tb.modules.datafile.exceptions import AlreadyExistsException, Incl
 from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
 from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 from tinybird.tb.modules.feedback_manager import FeedbackManager
-from tinybird.tb.modules.local_common import get_tinybird_local_config
+from tinybird.tb.modules.local_common import get_client_config_for_build
 
 
 async def folder_build(
@@ -58,7 +58,7 @@ async def folder_build(
     local_ws: Optional[Dict[str, Any]] = None,
     watch: bool = False,
 ):
-    config = await get_tinybird_local_config(folder)
+    config = await get_client_config_for_build(folder)
     build = True
     dry_run = False
     force = True

{tinybird-0.0.1.dev32 → tinybird-0.0.1.dev34}/tinybird/tb/modules/datafile/common.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import functools
 import glob
 import itertools
@@ -9,8 +11,9 @@ import shlex
 import string
 import textwrap
 import traceback
-from collections import namedtuple
+from collections import Counter, namedtuple
 from dataclasses import dataclass
+from enum import Enum
 from io import StringIO
 from pathlib import Path
 from string import Template
@@ -29,6 +32,9 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
 # from tinybird.sql import parse_indexes_structure, parse_table_structure, schema_to_sql_columns
 
 
+class DatafileValidationError(Exception): ...
+
+
 class DatafileSyntaxError(Exception):
     def __init__(self, message: str, lineno: int, pos: int, hint: Optional[str] = None):
         super().__init__(message)
@@ -155,6 +161,22 @@ valid_chars_name: str = string.ascii_letters + string.digits + "._`*<>+-'"
 valid_chars_fn: str = valid_chars_name + "[](),=!?:/ \n\t\r"
 
 
+class UnkownExtensionerror(Exception): ...
+
+
+class DatafileKind(Enum):
+    pipe = "pipe"
+    datasource = "datasource"
+
+    @classmethod
+    def from_extension(cls, extension: str) -> DatafileKind:
+        if extension == ".pipe":
+            return cls.pipe
+        elif extension == ".datasource":
+            return cls.datasource
+        raise UnkownExtensionerror(f"Unknown extension {extension} for data file")
+
+
 class Datafile:
     def __init__(self) -> None:
         self.maintainer: Optional[str] = None
@@ -167,6 +189,7 @@ class Datafile:
         self.shared_with: List[str] = []
         self.warnings: List[str] = []
         self.filtering_tags: Optional[List[str]] = None
+        self.kind: Optional[DatafileKind] = None
 
     def is_equal(self, other):
         if len(self.nodes) != len(other.nodes):
@@ -174,6 +197,40 @@ class Datafile:
 
         return all(self.nodes[i] == other.nodes[i] for i, _ in enumerate(self.nodes))
 
+    def set_kind(self, kind: DatafileKind):
+        self.kind = kind
+
+    def validate(self):
+        if self.kind == DatafileKind.pipe:
+            # TODO(eclbg):
+            #  [x] node names are unique
+            #  [ ] ...
+            repeated_node_names = [
+                name for name, count in filter(lambda x: x[1] > 1, Counter(n["name"] for n in self.nodes).items())
+            ]
+            if repeated_node_names:
+                raise DatafileValidationError(
+                    f"Pipe node names must be unique. These names are repeated: {repeated_node_names}"
+                )
+            pass
+        elif self.kind == DatafileKind.datasource:
+            # TODO(eclbg):
+            #  [x] Just one node
+            #  [x] Engine is present
+            #  [ ] ...
+            if len(self.nodes) > 1:
+                # Our users are not aware of data source data files being a single-node data file, hence this error
+                # message which might be confusing for us devs
+                raise DatafileValidationError("Datasource files cannot have nodes defined")
+            node = self.nodes[0]
+            if "schema" not in node:
+                raise DatafileValidationError("SCHEMA is mandatory")
+            if "engine" not in node:
+                raise DatafileValidationError("ENGINE is mandatory")
+        else:
+            # We cannot validate a datafile whose kind is unknown
+            pass
+
 
 def format_filename(filename: str, hide_folders: bool = False):
     return os.path.basename(filename) if hide_folders else filename
@@ -1016,6 +1073,7 @@ def parse(
     basepath: str = ".",
     replace_includes: bool = True,
     skip_eval: bool = False,
+    kind: Optional[DatafileKind] = None,
 ) -> Datafile:
     """
     Parses `s` string into a document
@@ -1032,6 +1090,8 @@ def parse(
     lines = list(StringIO(s, newline=None))
 
     doc = Datafile()
+    if kind is not None:
+        doc.set_kind(kind)
     doc.raw = list(StringIO(s, newline=None))
 
     parser_state = namedtuple(
@@ -1098,7 +1158,8 @@ def parse(
         except IndexesSyntaxError as e:
             raise e
         except Exception as e:
-            # TODO(eclbg): We get here when an unidentified error happens but we still report a parsing error. We could rething this.
+            # TODO(eclbg): We get here when an unidentified error happens but we still report a parsing error. We should
+            # rethink this.
             raise ParseException(FeedbackManager.error_parsing_indices(line=kwargs["lineno"], error=e))
 
         parser_state.current_node["indexes"] = indexes

{tinybird-0.0.1.dev32 → tinybird-0.0.1.dev34}/tinybird/tb/modules/datafile/parse_datasource.py

@@ -5,6 +5,7 @@ import click
 
 from tinybird.tb.modules.datafile.common import (
     Datafile,
+    DatafileKind,
     DatafileSyntaxError,
     format_filename,
     parse,
@@ -31,20 +32,21 @@ def parse_datasource(
 
     filename = format_filename(filename, hide_folders)
     try:
-        doc = parse(s, "default", basepath, replace_includes=replace_includes, skip_eval=skip_eval)
+        doc = parse(
+            s, "default", basepath, replace_includes=replace_includes, skip_eval=skip_eval, kind=DatafileKind.datasource
+        )
+        doc.validate()
     except DatafileSyntaxError as e:
         try:
             if add_context_to_datafile_syntax_errors:
                 e.get_context_from_file_contents(s)
         finally:
             raise e
+    # TODO(eclbg): all these exceptions that trigger a ClickException shouldn't be here, as this code will only run in
+    # the server soon
     except ParseException as e:
         raise click.ClickException(
             FeedbackManager.error_parsing_file(filename=filename, lineno=e.lineno, error=e)
         ) from None
 
-    if len(doc.nodes) > 1:
-        # TODO(eclbg): Turn this into a custom exception with a better message
-        raise ValueError(f"{filename}: datasources can't have more than one node")
-
     return doc

{tinybird-0.0.1.dev32 → tinybird-0.0.1.dev34}/tinybird/tb/modules/datafile/parse_pipe.py

@@ -6,6 +6,7 @@ import click
 from tinybird.sql_template import get_template_and_variables, render_sql_template
 from tinybird.tb.modules.datafile.common import (
     Datafile,
+    DatafileKind,
     DatafileSyntaxError,
     format_filename,
     parse,
@@ -35,7 +36,10 @@ def parse_pipe(
     try:
         sql = ""
         try:
-            doc = parse(s, basepath=basepath, replace_includes=replace_includes, skip_eval=skip_eval)
+            doc = parse(
+                s, basepath=basepath, replace_includes=replace_includes, skip_eval=skip_eval, kind=DatafileKind.pipe
+            )
+            doc.validate()
         except DatafileSyntaxError as e:
             try:
                 if add_context_to_datafile_syntax_errors:
@@ -53,6 +57,8 @@ def parse_pipe(
             from tinybird.sql_toolset import format_sql as toolset_format_sql
 
             toolset_format_sql(sql)
+    # TODO(eclbg): all these exceptions that trigger a ClickException shouldn't be here, as this code will only run in
+    # the server soon
     except ParseException as e:
         raise click.ClickException(
             FeedbackManager.error_parsing_file(

{tinybird-0.0.1.dev32 → tinybird-0.0.1.dev34}/tinybird/tb/modules/deploy.py

@@ -10,7 +10,6 @@ import requests
 
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import echo_safe_humanfriendly_tables_format_smart_table
-from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 
@@ -128,7 +127,8 @@ def deploy_group() -> None:
     default=False,
     help="Auto-promote the deployment. Only works if --wait is enabled. Disabled by default.",
 )
-def create(project_path: Path, wait: bool, auto: bool) -> None:
+@click.pass_context
+def create(ctx: click.Context, project_path: Path, wait: bool, auto: bool) -> None:
     """
     Validate and deploy the project server side.
     """
@@ -140,9 +140,9 @@ def create(project_path: Path, wait: bool, auto: bool) -> None:
         ".pipe": "text/plain",
     }
 
-    config = CLIConfig.get_project_config(str(project_path))
-    TINYBIRD_API_URL = f"{config.get_host()}/v1/deploy"
-    TINYBIRD_API_KEY = config.get_token()
+    client = ctx.ensure_object(dict)["client"]
+    TINYBIRD_API_URL = f"{client.host}/v1/deploy"
+    TINYBIRD_API_KEY = client.token
 
     files = [
         ("context://", ("cli-version", "1.0.0", "text/plain")),
@@ -186,7 +186,7 @@ def create(project_path: Path, wait: bool, auto: bool) -> None:
     if deployment and wait:
         while deployment.get("status") != "data_ready":
             time.sleep(5)
-            TINYBIRD_API_URL = f"{config.get_host()}/v1/deployments/{deployment.get('id')}"
+            TINYBIRD_API_URL = f"{client.host}/v1/deployments/{deployment.get('id')}"
             r = requests.get(TINYBIRD_API_URL, headers=HEADERS)
             result = r.json()
             deployment = result.get("deployment")
@@ -197,20 +197,21 @@ def create(project_path: Path, wait: bool, auto: bool) -> None:
         click.echo(FeedbackManager.success(message="Deployment is ready"))
 
         if auto:
-            promote_deployment(config.get_host(), HEADERS)
+            promote_deployment(client.host, HEADERS)
 
 
 @deploy_group.command(name="list")
 @click.argument("project_path", type=click.Path(exists=True), default=Path.cwd())
-def deploy_list(project_path: Path) -> None:
+@click.pass_context
+def deploy_list(ctx: click.Context, project_path: Path) -> None:
     """
     List all the deployments you have in the project.
     """
-    config = CLIConfig.get_project_config(str(project_path))
+    client = ctx.ensure_object(dict)["client"]
 
-    TINYBIRD_API_KEY = config.get_token()
+    TINYBIRD_API_KEY = client.token
     HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
-    TINYBIRD_API_URL = f"{config.get_host()}/v1/deployments"
+    TINYBIRD_API_URL = f"{client.host}/v1/deployments"
 
     r = requests.get(TINYBIRD_API_URL, headers=HEADERS)
     result = r.json()
@@ -228,27 +229,29 @@ def deploy_list(project_path: Path) -> None:
 
 @deploy_group.command(name="promote")
 @click.argument("project_path", type=click.Path(exists=True), default=Path.cwd())
-def deploy_promote(project_path: Path) -> None:
+@click.pass_context
+def deploy_promote(ctx: click.Context, project_path: Path) -> None:
     """
     Promote last deploy to ready and remove old one.
     """
-    config = CLIConfig.get_project_config(str(project_path))
+    client = ctx.ensure_object(dict)["client"]
 
-    TINYBIRD_API_KEY = config.get_token()
+    TINYBIRD_API_KEY = client.token
     HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
 
-    promote_deployment(config.get_host(), HEADERS)
+    promote_deployment(client.host, HEADERS)
 
 
 @deploy_group.command(name="rollback")
 @click.argument("project_path", type=click.Path(exists=True), default=Path.cwd())
-def deploy_rollback(project_path: Path) -> None:
+@click.pass_context
+def deploy_rollback(ctx: click.Context, project_path: Path) -> None:
     """
     Rollback to the previous deployment.
     """
-    config = CLIConfig.get_project_config(str(project_path))
+    client = ctx.ensure_object(dict)["client"]
 
-    TINYBIRD_API_KEY = config.get_token()
+    TINYBIRD_API_KEY = client.token
     HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
 
-    rollback_deployment(config.get_host(), HEADERS)
+    rollback_deployment(client.host, HEADERS)

{tinybird-0.0.1.dev32 → tinybird-0.0.1.dev34}/tinybird/tb/modules/local.py

@@ -1,5 +1,6 @@
 import os
 import re
+import subprocess
 import time
 
 import click
@@ -82,7 +83,9 @@ def get_docker_client():
         client.ping()
         return client
     except Exception:
-        raise CLIException("No container runtime is running. Make sure a Docker-compatible runtime is installed and running.")
+        raise CLIException(
+            "No container runtime is running. Make sure a Docker-compatible runtime is installed and running."
+        )
 
 
 def stop_tinybird_local(docker_client):
@@ -107,8 +110,17 @@ def remove_tinybird_local(docker_client):
 def upgrade():
     """Upgrade Tinybird CLI to the latest version"""
     click.echo(FeedbackManager.highlight(message="» Upgrading Tinybird CLI..."))
-    os.system(f"{os.getenv('HOME')}/.local/bin/uv tool upgrade tinybird")
-    click.echo(FeedbackManager.success(message="✓ Tinybird CLI upgraded"))
+    process = subprocess.Popen(
+        [f"{os.getenv('HOME')}/.local/bin/uv", "tool", "upgrade", "tinybird"],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    stdout, stderr = process.communicate()
+    if "Nothing to upgrade" not in stdout + stderr:
+        click.echo(FeedbackManager.success(message="✓ Tinybird CLI upgraded"))
+    else:
+        click.echo(FeedbackManager.info(message="✓ Tinybird CLI is already up to date"))
 
 
 @cli.group()

{tinybird-0.0.1.dev32 → tinybird-0.0.1.dev34}/tinybird/tb/modules/local_common.py

@@ -1,4 +1,5 @@
 import hashlib
+import logging
 import os
 from typing import Optional
 
@@ -16,11 +17,15 @@ TB_LOCAL_HOST = f"http://localhost:{TB_LOCAL_PORT}"
 
 async def get_tinybird_local_client(path: Optional[str] = None) -> TinyB:
     """Get a Tinybird client connected to the local environment."""
-    config = await get_tinybird_local_config(path)
+    config = await get_client_config_for_build(path)
     return config.get_client(host=TB_LOCAL_HOST)
 
 
-async def get_tinybird_local_config(path: Optional[str] = None) -> CLIConfig:
+async def get_client_config_for_build(path: Optional[str] = None) -> CLIConfig:
+    """Craft a client config with a workspace name based on the path of the project files
+
+    It uses the tokens from tinybird local
+    """
     config = CLIConfig.get_project_config(path)
     try:
         # ruff: noqa: ASYNC210
@@ -37,6 +42,7 @@ async def get_tinybird_local_config(path: Optional[str] = None) -> CLIConfig:
         user_client = config.get_client(host=TB_LOCAL_HOST, token=user_token)
 
         ws_name = f"Tinybird_Local_Build_{folder_hash}"
+        logging.debug(f"Workspace used for build: {ws_name}")
 
         user_workspaces = await user_client.user_workspaces()
         ws = next((ws for ws in user_workspaces["workspaces"] if ws["name"] == ws_name), None)

{tinybird-0.0.1.dev32 → tinybird-0.0.1.dev34}/tinybird/tb/modules/mock.py

@@ -18,7 +18,12 @@ from tinybird.tb.modules.local_common import get_tinybird_local_client
 @cli.command()
 @click.argument("datasource", type=str)
 @click.option("--rows", type=int, default=10, help="Number of events to send")
-@click.option("--prompt", type=str, default="", help="Extra context to use for data generation")
+@click.option(
+    "--prompt",
+    type=str,
+    default="Use the datasource schema to generate sample data",
+    help="Extra context to use for data generation",
+)
 @click.option("--folder", type=str, default=".", help="Folder where datafiles will be placed")
 @coro
 async def mock(datasource: str, rows: int, prompt: str, folder: str) -> None:

{tinybird-0.0.1.dev32 → tinybird-0.0.1.dev34}/tinybird/tb/modules/test.py

@@ -68,7 +68,9 @@ def test(ctx: click.Context) -> None:
     type=click.Path(exists=True, file_okay=False),
     help="Folder where datafiles will be placed",
 )
-@click.option("--prompt", type=str, default="", help="Prompt to be used to create the test")
+@click.option(
+    "--prompt", type=str, default="Create a test for the selected pipe", help="Prompt to be used to create the test"
+)
 @coro
 async def test_create(name_or_filename: str, prompt: str, folder: str) -> None:
     """

{tinybird-0.0.1.dev32 → tinybird-0.0.1.dev34}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird
-Version: 0.0.1.dev32
+Version: 0.0.1.dev34
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird