tinybird 0.0.1.dev17__py3-none-any.whl → 0.0.1.dev18__py3-none-any.whl

tinybird/tb/modules/build.py

@@ -6,7 +6,6 @@ from pathlib import Path
 from typing import Any, Awaitable, Callable, List, Union
 
 import click
-from click import Context
 from watchdog.events import FileSystemEventHandler
 from watchdog.observers import Observer
 
@@ -16,7 +15,7 @@ from tinybird.config import FeatureFlags
 from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb.modules.build_shell import BuildShell, print_table_formatted
 from tinybird.tb.modules.cli import cli
-from tinybird.tb.modules.common import coro, push_data
+from tinybird.tb.modules.common import push_data
 from tinybird.tb.modules.datafile.build import folder_build
 from tinybird.tb.modules.datafile.common import get_project_filenames, get_project_fixtures, has_internal_datafiles
 from tinybird.tb.modules.datafile.exceptions import ParseException
@@ -101,10 +100,7 @@ def watch_files(
     is_flag=True,
     help="Watch for changes in the files and re-check them.",
 )
-@click.pass_context
-@coro
-async def build(
-    ctx: Context,
+def build(
     folder: str,
     watch: bool,
 ) -> None:
@@ -115,7 +111,7 @@ async def build(
     context.disable_template_security_validation.set(True)
     is_internal = has_internal_datafiles(folder)
     folder_path = os.path.abspath(folder)
-    tb_client = await get_tinybird_local_client(folder_path)
+    tb_client = asyncio.run(get_tinybird_local_client(folder_path))
 
     def check_filenames(filenames: List[str]):
         parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
@@ -191,16 +187,28 @@ async def build(
             ok = False
         return ok
 
-    build_ok = await build_once(filenames)
+    build_ok = asyncio.run(build_once(filenames))
 
     if watch:
-        shell = BuildShell(folder=folder, client=tb_client)
+        paths = [Path(f) for f in get_project_filenames(folder, with_vendor=True)]
+
+        def is_vendor(f: Path) -> bool:
+            return "vendor/" in f.parts
+
+        def get_vendor_workspace(f: Path) -> str:
+            return f.parts[1]
+
+        datasource_paths = [f for f in paths if f.suffix == ".datasource"]
+        datasources = [f.stem for f in datasource_paths if not is_vendor(f)]
+        shared_datasources = [f"{get_vendor_workspace(f)}.{f.stem}" for f in datasource_paths if is_vendor(f)]
+        pipes = [f.stem for f in paths if f.suffix == ".pipe" and not is_vendor(f)]
+        shell = BuildShell(folder=folder, client=tb_client, datasources=datasources + shared_datasources, pipes=pipes)
         click.echo(FeedbackManager.highlight(message="◎ Watching for changes..."))
         watcher_thread = threading.Thread(
             target=watch_files, args=(filenames, process, shell, folder, build_ok), daemon=True
         )
         watcher_thread.start()
-        shell.cmdloop()
+        shell.run_shell()
 
 
 async def build_and_print_resource(tb_client: TinyB, filename: str):

tinybird/tb/modules/build_shell.py

@@ -1,11 +1,19 @@
 import asyncio
-import cmd
+import concurrent.futures
+import os
 import random
 import subprocess
 import sys
+from typing import List
 
 import click
 import humanfriendly
+from prompt_toolkit import PromptSession
+from prompt_toolkit.completion import Completer, Completion
+from prompt_toolkit.history import FileHistory
+from prompt_toolkit.key_binding import KeyBindings
+from prompt_toolkit.shortcuts import CompleteStyle
+from prompt_toolkit.styles import Style
 
 from tinybird.client import TinyB
 from tinybird.feedback_manager import FeedbackManager, bcolors
@@ -13,43 +21,217 @@ from tinybird.tb.modules.exceptions import CLIException
 from tinybird.tb.modules.table import format_table
 
 
-class BuildShell(cmd.Cmd):
-    prompt = "\n\001\033[1;32m\002tb > \001\033[0m\002"
+class DynamicCompleter(Completer):
+    def __init__(self, datasources: List[str], pipes: List[str]):
+        self.datasources = datasources
+        self.pipes = pipes
+        self.static_commands = ["create", "mock", "test", "select"]
+        self.mock_flags = ["--prompt", "--rows"]
+        self.common_rows = ["10", "50", "100", "500", "1000"]
+        self.sql_keywords = ["select", "from", "where", "group by", "order by", "limit"]
 
-    def __init__(self, folder: str, client: TinyB):
-        super().__init__()
+    def get_completions(self, document, complete_event):
+        text = document.text_before_cursor.strip()
+        words = text.split()
+
+        # Normalize command by removing 'tb' prefix if present
+        if words and words[0] == "tb":
+            words = words[1:]
+
+        if not words:
+            # Show all available commands when no input
+            yield from self._yield_static_commands("")
+            return
+
+        command = words[0].lower()
+
+        if command == "mock":
+            yield from self._handle_mock_completions(words)
+        elif command == "select" or self._is_sql_query(text.lower()):
+            yield from self._handle_sql_completions(text)
+        else:
+            # Handle general command completions
+            yield from self._yield_static_commands(words[-1])
+
+    def _is_sql_query(self, text: str) -> bool:
+        """Check if the input looks like a SQL query."""
+        sql_starters = ["select", "with"]
+        return any(text.startswith(starter) for starter in sql_starters)
+
+    def _handle_sql_completions(self, text: str):
+        """Handle completions for SQL queries."""
+        text_lower = text.lower()
+
+        # Find the last complete word
+        words = text_lower.split()
+        if not words:
+            return
+
+        # If we just typed 'from' or there's a space after 'from', suggest datasources
+        if words[-1] == "from" or (
+            "from" in words and len(words) > words.index("from") + 1 and text_lower.endswith(" ")
+        ):
+            for x in self.datasources:
+                yield Completion(x, start_position=0, display=x, style="class:completion.datasource")
+            for x in self.pipes:
+                yield Completion(x, start_position=0, display=x, style="class:completion.pipe")
+            return
+
+        # If we're starting a query, suggest SQL keywords
+        if len(words) <= 2:
+            for keyword in self.sql_keywords:
+                if keyword.lower().startswith(words[-1]):
+                    yield Completion(
+                        keyword, start_position=-len(words[-1]), display=keyword, style="class:completion.keyword"
+                    )
+
+    def _handle_mock_completions(self, words: List[str]):
+        if len(words) == 1:
+            # After 'mock', show datasources
+            for ds in self.datasources:
+                yield Completion(ds, start_position=0, display=ds, style="class:completion.cmd")
+            return
+
+        if len(words) == 2 or len(words) == 4:
+            # After datasource or after a flag value, show available flags
+            available_flags = [f for f in self.mock_flags if f not in words]
+            for flag in available_flags:
+                yield Completion(flag, start_position=0, display=flag)
+            return
+
+        last_word = words[-1]
+        if last_word == "--prompt":
+            yield Completion('""', start_position=0, display='"Enter your prompt..."')
+        elif last_word == "--rows":
+            for rows in self.common_rows:
+                yield Completion(rows, start_position=0, display=rows)
+
+    def _yield_static_commands(self, current_word: str):
+        for cmd in self.static_commands:
+            if cmd.startswith(current_word):
+                yield Completion(
+                    cmd,
+                    start_position=-len(current_word) if current_word else 0,
+                    display=cmd,
+                    style="class:completion.cmd",
+                )
+
+
+style = Style.from_dict(
+    {
+        "prompt": "fg:#34D399 bold",
+        "completion.cmd": "fg:#34D399 bg:#111111 bold",
+        "completion.datasource": "fg:#AB49D0 bg:#111111",
+        "completion.pipe": "fg:#FEA827 bg:#111111",
+        "completion.keyword": "fg:#34D399 bg:#111111",
+    }
+)
+
+key_bindings = KeyBindings()
+
+
+@key_bindings.add("c-d")
+def _(event):
+    """
+    Start auto completion. If the menu is showing already, select the next
+    completion.
+    """
+    b = event.app.current_buffer
+    if b.complete_state:
+        b.complete_next()
+    else:
+        b.start_completion(select_first=False)
+
+
+class BuildShell:
+    def __init__(self, folder: str, client: TinyB, datasources: List[str], pipes: List[str]):
+        self.history = self.get_history()
         self.folder = folder
         self.client = client
+        self.datasources = datasources
+        self.pipes = pipes
+        self.prompt_message = "\ntb > "
+        self.commands = ["create", "mock", "test", "tb", "select"]
+
+        self.session = PromptSession(
+            completer=DynamicCompleter(self.datasources, self.pipes),
+            complete_style=CompleteStyle.COLUMN,
+            complete_while_typing=True,
+            history=self.history,
+        )
+
+    def get_history(self):
+        try:
+            history_file = os.path.expanduser("~/.tb_history")
+            return FileHistory(history_file)
+        except Exception:
+            return None
+
+    def run_shell(self):
+        while True:
+            try:
+                user_input = self.session.prompt(
+                    [("class:prompt", self.prompt_message)], style=style, key_bindings=key_bindings
+                )
+                self.handle_input(user_input)
+            except (EOFError, KeyboardInterrupt):
+                sys.exit(0)
+            except CLIException as e:
+                click.echo(str(e))
+            except Exception as e:
+                # Catch-all for unexpected exceptions
+                click.echo(FeedbackManager.error_exception(error=str(e)))
 
-    def do_exit(self, arg):
-        sys.exit(0)
+    def handle_input(self, argline):
+        line = argline.strip()
+        if not line:
+            return
 
-    def do_quit(self, arg):
-        sys.exit(0)
+        # Implement the command logic here
+        # Replace do_* methods with equivalent logic:
+        command_parts = line.split(maxsplit=1)
+        cmd = command_parts[0].lower()
+        arg = command_parts[1] if len(command_parts) > 1 else ""
+
+        if cmd in ["exit", "quit"]:
+            sys.exit(0)
+        elif cmd == "build":
+            self.handle_build(arg)
+        elif cmd == "auth":
+            self.handle_auth(arg)
+        elif cmd == "workspace":
+            self.handle_workspace(arg)
+        elif cmd == "mock":
+            self.handle_mock(arg)
+        elif cmd == "tb":
+            self.handle_tb(arg)
+        else:
+            # Check if it looks like a SQL query or run as a tb command
+            self.default(line)
 
-    def do_build(self, arg):
+    def handle_build(self, arg):
         click.echo(FeedbackManager.error(message=f"'tb {arg}' command is not available in watch mode"))
 
-    def do_auth(self, arg):
+    def handle_auth(self, arg):
         click.echo(FeedbackManager.error(message=f"'tb {arg}' command is not available in watch mode"))
 
-    def do_workspace(self, arg):
+    def handle_workspace(self, arg):
         click.echo(FeedbackManager.error(message=f"'tb {arg}' command is not available in watch mode"))
 
-    def do_mock(self, arg):
+    def handle_mock(self, arg):
         subprocess.run(f"tb mock {arg} --folder {self.folder}", shell=True, text=True)
 
-    def do_tb(self, arg):
+    def handle_tb(self, arg):
         click.echo("")
         arg = arg.strip().lower()
         if arg.startswith("build"):
-            self.do_build(arg)
+            self.handle_build(arg)
         elif arg.startswith("auth"):
-            self.do_auth(arg)
+            self.handle_auth(arg)
         elif arg.startswith("workspace"):
-            self.do_workspace(arg)
+            self.handle_workspace(arg)
         elif arg.startswith("mock"):
-            self.do_mock(arg)
+            self.handle_mock(arg)
         else:
             subprocess.run(f"tb --local {arg}", shell=True, text=True)
 
@@ -66,20 +248,15 @@ class BuildShell(cmd.Cmd):
         else:
             subprocess.run(f"tb --local {arg}", shell=True, text=True)
 
-    def reprint_prompt(self):
-        self.stdout.write(self.prompt)
-        self.stdout.flush()
-
     def run_sql(self, query, rows_limit=20):
         try:
             q = query.strip()
-            if q.startswith("insert"):
+            if q.lower().startswith("insert"):
                 click.echo(FeedbackManager.info_append_data())
                 raise CLIException(FeedbackManager.error_invalid_query())
-            if q.startswith("delete"):
+            if q.lower().startswith("delete"):
                 raise CLIException(FeedbackManager.error_invalid_query())
 
-            # fuck my life
             def run_query_in_thread():
                 loop = asyncio.new_event_loop()
                 asyncio.set_event_loop(loop)
@@ -90,9 +267,6 @@ class BuildShell(cmd.Cmd):
                 finally:
                     loop.close()
 
-            # Run the query in a separate thread
-            import concurrent.futures
-
             with concurrent.futures.ThreadPoolExecutor() as executor:
                 res = executor.submit(run_query_in_thread).result()
 
@@ -107,6 +281,9 @@ class BuildShell(cmd.Cmd):
         else:
             click.echo(FeedbackManager.info_no_rows())
 
+    def reprint_prompt(self):
+        click.echo(f"{bcolors.OKGREEN}{self.prompt_message}{bcolors.ENDC}", nl=False)
+
 
 def print_table_formatted(res: dict, name: str):
     rebuild_colors = [bcolors.FAIL, bcolors.OKBLUE, bcolors.WARNING, bcolors.OKGREEN, bcolors.HEADER]

tinybird/tb/modules/cicd.py

@@ -14,9 +14,7 @@ class Provider(Enum):
     GitLab = 1
 
 
-WORKFLOW_VERSION = "v3.1.0"
-
-DEFAULT_REQUIREMENTS_FILE = "tinybird-cli>=5,<6"
+WORKFLOW_VERSION = "v0.0.1"
 
 GITHUB_CI_YML = """
 name: Tinybird - CI Workflow
@@ -50,6 +48,8 @@ jobs:
         run: curl -LsSf https://api.tinybird.co/static/install.sh | sh
       - name: Build project
         run: tb build
+      - name: Test project
+        run: tb test run
 """
 
 
@@ -64,110 +64,30 @@ stages:
 
 GITLAB_CI_YML = """
 tinybird_ci_workflow:
+  image: ubuntu:latest
   stage: tests
   interruptible: true
   needs: []
   rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"{% if data_project_dir != '.' %}
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
-        - .gitlab/tinybird/*
+        - .gitlab/tinybird/*{% if data_project_dir != '.' %}
         - {{ data_project_dir }}/*
         - {{ data_project_dir }}/**/*{% end %}
   before_script:
+    - apt update && apt install -y curl
     - curl -LsSf https://api.tinybird.co/static/install.sh | sh
   script:
+    - export PATH="$HOME/.local/bin:$PATH"
     - cd $CI_PROJECT_DIR/{{ data_project_dir }}
     - tb build
+    - tb test run
   services:
     - name: tinybirdco/tinybird-local:latest
       alias: tinybird-local
 """
 
 
-EXEC_TEST_SH = """
-#!/usr/bin/env bash
-set -euxo pipefail
-
-export TB_VERSION_WARNING=0
-
-run_test() {
-    t=$1
-    echo "** Running $t **"
-    echo "** $(cat $t)"
-    tmpfile=$(mktemp)
-    retries=0
-    TOTAL_RETRIES=3
-
-    # When appending fixtures, we need to retry in case of the data is not replicated in time
-    while [ $retries -lt $TOTAL_RETRIES ]; do
-        # Run the test and store the output in a temporary file
-        bash $t $2 >$tmpfile
-        exit_code=$?
-        if [ "$exit_code" -eq 0 ]; then
-            # If the test passed, break the loop
-            if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
-                break
-            # If the test failed, increment the retries counter and try again
-            else
-                retries=$((retries+1))
-            fi
-        # If the bash command failed, print an error message and break the loop
-        else
-            break
-        fi
-    done
-
-    if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
-        echo "✅ Test $t passed"
-        rm $tmpfile
-        return 0
-    elif [ $retries -eq $TOTAL_RETRIES ]; then
-        echo "🚨 ERROR: Test $t failed, diff:";
-        diff -B ${t}.result $tmpfile
-        rm $tmpfile
-        return 1
-    else
-        echo "🚨 ERROR: Test $t failed with bash command exit code $?"
-        cat $tmpfile
-        rm $tmpfile
-        return 1
-    fi
-    echo ""
-}
-export -f run_test
-
-fail=0
-find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} || fail=1
-
-if [ $fail == 1 ]; then
-  exit -1;
-fi
-"""
-
-APPEND_FIXTURES_SH = """
-#!/usr/bin/env bash
-set -euxo pipefail
-
-directory="datasources/fixtures"
-extensions=("csv" "ndjson")
-
-absolute_directory=$(realpath "$directory")
-
-for extension in "${extensions[@]}"; do
-  file_list=$(find "$absolute_directory" -type f -name "*.$extension")
-
-  for file_path in $file_list; do
-    file_name=$(basename "$file_path")
-    file_name_without_extension="${file_name%.*}"
-
-    command="tb datasource append $file_name_without_extension datasources/fixtures/$file_name"
-    echo $command
-    $command
-  done
-done
-"""
-
-
 class CICDFile:
     def __init__(
         self,

tinybird/tb/modules/common.py

@@ -17,7 +17,7 @@ from contextlib import closing
 from copy import deepcopy
 from enum import Enum
 from functools import wraps
-from os import chmod, environ, getcwd, getenv
+from os import environ, getcwd, getenv
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Union
 from urllib.parse import urlparse
@@ -61,9 +61,7 @@ if TYPE_CHECKING:
     from tinybird.connectors import Connector
 
 from tinybird.feedback_manager import FeedbackManager, warning_message
-from tinybird.git_settings import DEFAULT_TINYENV_FILE
 from tinybird.syncasync import async_to_sync, sync_to_async
-from tinybird.tb.modules.cicd import APPEND_FIXTURES_SH, DEFAULT_REQUIREMENTS_FILE, EXEC_TEST_SH
 from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.exceptions import (
     CLIAuthException,
@@ -371,111 +369,6 @@ async def folder_init(
             for path in Path(folder).glob(f"*.{format}"):
                 await _generate_datafile(str(path), client, format=format, force=force)
 
-    if generate_releases:
-        base = Path(".")
-        f = base / (".tinyenv")
-        if not f.exists() or force:
-            async with aiofiles.open(".tinyenv", "w") as file:
-                await file.write(DEFAULT_TINYENV_FILE)
-                click.echo(FeedbackManager.info_file_created(file=".tinyenv"))
-        else:
-            click.echo(FeedbackManager.info_dottinyenv_already_exists())
-
-        base = Path(".")
-        f = base / ("requirements.txt")
-        if not f.exists() or force:
-            async with aiofiles.open("requirements.txt", "w") as file:
-                await file.write(DEFAULT_REQUIREMENTS_FILE)
-                click.echo(FeedbackManager.info_file_created(file="requirements.txt"))
-
-        base = Path("scripts")
-        if not base.exists():
-            base = Path()
-        f = base / ("exec_test.sh")
-        if not f.exists() or force:
-            async with aiofiles.open(f"{f}", "w") as t_file:
-                await t_file.write(EXEC_TEST_SH)
-            click.echo(FeedbackManager.info_file_created(file="scripts/exec_test.sh"))
-        chmod(f, 0o755)
-
-        f = base / ("append_fixtures.sh")
-        if not f.exists() or force:
-            async with aiofiles.open(f"{f}", "w") as t_file:
-                await t_file.write(APPEND_FIXTURES_SH)
-            click.echo(FeedbackManager.info_file_created(file="scripts/append_fixtures.sh"))
-        chmod(f, 0o755)
-
-        base = Path("tests")
-        if not base.exists():
-            base = Path()
-        f = base / ("example.yml")
-        if not base.exists() or force:
-            async with aiofiles.open(f"{f}", "w") as t_file:
-                await t_file.write(
-                    """
-##############################################################################################################################
-###   Visit https://www.tinybird.co/docs/production/implementing-test-strategies.html#data-quality-tests              ###
-###   for more details on Data Quality tests                                                                               ###
-##############################################################################################################################
-
-- example_no_negative_numbers:
-    max_bytes_read: null
-    max_time: null
-    sql: |
-        SELECT
-            number
-        FROM numbers(10)
-        WHERE
-            number < 0
-
-#    - example_top_products_params_no_empty_top_10_on_2023:
-#       max_bytes_read: null
-#       max_time: null
-#       sql: |
-#          SELECT *
-#          FROM top_products_params
-#          WHERE empty(top_10)
-#       pipe:
-#          name: top_products_params
-#          params:
-#             start: '2023-01-01'
-#             end: '2023-12-31'
-
-"""
-                )
-
-        f = base / ("regression.yaml")
-        if not base.exists() or force:
-            async with aiofiles.open(f"{f}", "w") as t_file:
-                await t_file.write(
-                    """
-############################################################################################################################
-###   Visit https://www.tinybird.co/docs/production/implementing-test-strategies.html#regression-tests              ###
-###   for more details on Regression tests                                                                               ###
-############################################################################################################################
-
-###
-### New pipes are covered by this rule, rules below this one supersede this setting
-###
-- pipe: '.*'
-  tests:
-    - coverage:
-
-
-
-###
-### These are rules to customize regression testing by pipe using regular expressions
-### For instance skip regression tests for the pipes matching `endpoint_name.*`
-###
-- pipe: 'endpoint_name.*'
-  tests:
-    - coverage:
-        config:
-          skip: True
-
-"""
-                )
-
 
 async def configure_connector(connector):
     if connector not in SUPPORTED_CONNECTORS:

tinybird/tb/modules/create.py

@@ -1,5 +1,4 @@
 import os
-import subprocess
 from os import getcwd
 from pathlib import Path
 from typing import Optional
@@ -126,7 +125,7 @@ async def create(
             test_name = "api_token_usage"
             test_path = Path(folder) / "tests" / f"{test_name}.yaml"
             test_content = fetch_gist_content(
-                "https://gist.githubusercontent.com/gnzjgo/e58620bbb977d6f42f1d0c2a7b46ac8f/raw/a5f61d5019111f937484f941111829dfce69f648/api_token_usage.yaml"
+                "https://gist.githubusercontent.com/gnzjgo/e58620bbb977d6f42f1d0c2a7b46ac8f/raw/a3a1cd0ce3a90bcd2f6dfce00da51e6051443612/api_token_usage.yaml"
             )
             test_path.write_text(test_content)
             click.echo(FeedbackManager.info(message=f"✓ /tests/{test_name}.yaml"))
@@ -147,7 +146,7 @@ async def create(
                 datasource_content = datasource_path.read_text()
                 has_json_path = "`json:" in datasource_content
                 if has_json_path:
-                    sql = await llm.generate_sql_sample_data(schema=datasource_content, rows=rows, context=prompt)
+                    sql = await llm.generate_sql_sample_data(schema=datasource_content, rows=rows, prompt=prompt)
                     result = await tb_client.query(f"{sql} FORMAT JSON")
                     data = result.get("data", [])
                     fixture_name = build_fixture_name(datasource_path.absolute(), datasource_name, datasource_content)
@@ -214,9 +213,6 @@ def init_git(folder: str):
     try:
         path = Path(folder)
         gitignore_file = path / ".gitignore"
-        git_folder = path / ".git"
-        if not git_folder.exists():
-            subprocess.run(["git", "init"], cwd=path, check=True, capture_output=True)
 
         if gitignore_file.exists():
             content = gitignore_file.read_text()