tinybird 0.0.1.dev16__py3-none-any.whl → 0.0.1.dev18__py3-none-any.whl

tinybird/tb/cli.py

@@ -5,7 +5,6 @@ if sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 
 import tinybird.tb.modules.auth
-import tinybird.tb.modules.branch
 import tinybird.tb.modules.build
 import tinybird.tb.modules.cli
 import tinybird.tb.modules.common

tinybird/tb/modules/build.py

@@ -6,7 +6,6 @@ from pathlib import Path
 from typing import Any, Awaitable, Callable, List, Union
 
 import click
-from click import Context
 from watchdog.events import FileSystemEventHandler
 from watchdog.observers import Observer
 
@@ -16,7 +15,7 @@ from tinybird.config import FeatureFlags
 from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb.modules.build_shell import BuildShell, print_table_formatted
 from tinybird.tb.modules.cli import cli
-from tinybird.tb.modules.common import coro, push_data
+from tinybird.tb.modules.common import push_data
 from tinybird.tb.modules.datafile.build import folder_build
 from tinybird.tb.modules.datafile.common import get_project_filenames, get_project_fixtures, has_internal_datafiles
 from tinybird.tb.modules.datafile.exceptions import ParseException
@@ -27,9 +26,10 @@ from tinybird.tb.modules.local_common import get_tinybird_local_client
 
 
 class FileChangeHandler(FileSystemEventHandler):
-    def __init__(self, filenames: List[str], process: Callable[[List[str]], None]):
+    def __init__(self, filenames: List[str], process: Callable[[List[str]], None], build_ok: bool):
         self.filenames = filenames
         self.process = process
+        self.build_ok = build_ok
 
     def on_modified(self, event: Any) -> None:
         is_not_vendor = "vendor/" not in event.src_path
@@ -41,7 +41,9 @@ class FileChangeHandler(FileSystemEventHandler):
             filename = event.src_path.split("/")[-1]
             click.echo(FeedbackManager.highlight(message=f"\n\n⟲ Changes detected in {filename}\n"))
             try:
-                self.process([event.src_path])
+                to_process = [event.src_path] if self.build_ok else self.filenames
+                self.process(to_process)
+                self.build_ok = True
             except Exception as e:
                 click.echo(FeedbackManager.error_exception(error=e))
 
@@ -51,6 +53,7 @@ def watch_files(
     process: Union[Callable[[List[str]], None], Callable[[List[str]], Awaitable[None]]],
     shell: BuildShell,
     folder: str,
+    build_ok: bool,
 ) -> None:
     # Handle both sync and async process functions
     async def process_wrapper(files: List[str]) -> None:
@@ -68,7 +71,7 @@ def watch_files(
         )
         shell.reprint_prompt()
 
-    event_handler = FileChangeHandler(filenames, lambda f: asyncio.run(process_wrapper(f)))
+    event_handler = FileChangeHandler(filenames, lambda f: asyncio.run(process_wrapper(f)), build_ok)
     observer = Observer()
 
     observer.schedule(event_handler, path=folder, recursive=True)
@@ -97,10 +100,7 @@ def watch_files(
     is_flag=True,
     help="Watch for changes in the files and re-check them.",
 )
-@click.pass_context
-@coro
-async def build(
-    ctx: Context,
+def build(
     folder: str,
     watch: bool,
 ) -> None:
@@ -110,9 +110,8 @@ async def build(
     ignore_sql_errors = FeatureFlags.ignore_sql_errors()
     context.disable_template_security_validation.set(True)
     is_internal = has_internal_datafiles(folder)
-
     folder_path = os.path.abspath(folder)
-    tb_client = await get_tinybird_local_client(folder_path)
+    tb_client = asyncio.run(get_tinybird_local_client(folder_path))
 
     def check_filenames(filenames: List[str]):
         parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
@@ -143,30 +142,31 @@ async def build(
                 is_internal=is_internal,
                 watch=watch,
             )
-
-        filename = filenames[0]
-        if filename.endswith(".ndjson"):
-            fixture_path = Path(filename)
-            name = "_".join(fixture_path.stem.split("_")[:-1])
-            ds_path = Path(folder) / "datasources" / f"{name}.datasource"
-            if ds_path.exists():
-                await append_datasource({}, tb_client, name, str(fixture_path), silent=True)
-
-        if watch:
-            if filename.endswith(".datasource"):
-                ds_path = Path(filename)
-                name = build_fixture_name(filename, ds_path.stem, ds_path.read_text())
-                fixture_path = get_fixture_dir() / f"{name}.ndjson"
-                if fixture_path.exists():
-                    await append_datasource({}, tb_client, ds_path.stem, str(fixture_path), silent=True)
-            if not filename.endswith(".ndjson"):
-                await build_and_print_resource(tb_client, filename)
+        if len(filenames) > 0:
+            filename = filenames[0]
+            if filename.endswith(".ndjson"):
+                fixture_path = Path(filename)
+                name = "_".join(fixture_path.stem.split("_")[:-1])
+                ds_path = Path(folder) / "datasources" / f"{name}.datasource"
+                if ds_path.exists():
+                    await append_datasource({}, tb_client, name, str(fixture_path), silent=True)
+
+            if watch:
+                if filename.endswith(".datasource"):
+                    ds_path = Path(filename)
+                    name = build_fixture_name(filename, ds_path.stem, ds_path.read_text())
+                    fixture_path = get_fixture_dir() / f"{name}.ndjson"
+                    if fixture_path.exists():
+                        await append_datasource({}, tb_client, ds_path.stem, str(fixture_path), silent=True)
+                if not filename.endswith(".ndjson"):
+                    await build_and_print_resource(tb_client, filename)
 
     datafiles = get_project_filenames(folder)
     fixtures = get_project_fixtures(folder)
     filenames = datafiles + fixtures
 
     async def build_once(filenames: List[str]):
+        ok = False
         try:
             click.echo("⚡ Building project...\n")
             time_start = time.time()
@@ -181,17 +181,34 @@ async def build(
                     if fixture_path.exists():
                         await append_datasource({}, tb_client, ds_path.stem, str(fixture_path), silent=True)
             click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s\n"))
+            ok = True
         except Exception as e:
             click.echo(FeedbackManager.error(message=str(e)))
+            ok = False
+        return ok
 
-    await build_once(filenames)
+    build_ok = asyncio.run(build_once(filenames))
 
     if watch:
-        shell = BuildShell(folder=folder, client=tb_client)
+        paths = [Path(f) for f in get_project_filenames(folder, with_vendor=True)]
+
+        def is_vendor(f: Path) -> bool:
+            return "vendor/" in f.parts
+
+        def get_vendor_workspace(f: Path) -> str:
+            return f.parts[1]
+
+        datasource_paths = [f for f in paths if f.suffix == ".datasource"]
+        datasources = [f.stem for f in datasource_paths if not is_vendor(f)]
+        shared_datasources = [f"{get_vendor_workspace(f)}.{f.stem}" for f in datasource_paths if is_vendor(f)]
+        pipes = [f.stem for f in paths if f.suffix == ".pipe" and not is_vendor(f)]
+        shell = BuildShell(folder=folder, client=tb_client, datasources=datasources + shared_datasources, pipes=pipes)
         click.echo(FeedbackManager.highlight(message="◎ Watching for changes..."))
-        watcher_thread = threading.Thread(target=watch_files, args=(filenames, process, shell, folder), daemon=True)
+        watcher_thread = threading.Thread(
+            target=watch_files, args=(filenames, process, shell, folder, build_ok), daemon=True
+        )
         watcher_thread.start()
-        shell.cmdloop()
+        shell.run_shell()
 
 
 async def build_and_print_resource(tb_client: TinyB, filename: str):

tinybird/tb/modules/build_shell.py

@@ -1,11 +1,19 @@
 import asyncio
-import cmd
+import concurrent.futures
+import os
 import random
 import subprocess
 import sys
+from typing import List
 
 import click
 import humanfriendly
+from prompt_toolkit import PromptSession
+from prompt_toolkit.completion import Completer, Completion
+from prompt_toolkit.history import FileHistory
+from prompt_toolkit.key_binding import KeyBindings
+from prompt_toolkit.shortcuts import CompleteStyle
+from prompt_toolkit.styles import Style
 
 from tinybird.client import TinyB
 from tinybird.feedback_manager import FeedbackManager, bcolors
@@ -13,57 +21,242 @@ from tinybird.tb.modules.exceptions import CLIException
 from tinybird.tb.modules.table import format_table
 
 
-class BuildShell(cmd.Cmd):
-    prompt = "\n\001\033[1;32m\002TB > \001\033[0m\002"
+class DynamicCompleter(Completer):
+    def __init__(self, datasources: List[str], pipes: List[str]):
+        self.datasources = datasources
+        self.pipes = pipes
+        self.static_commands = ["create", "mock", "test", "select"]
+        self.mock_flags = ["--prompt", "--rows"]
+        self.common_rows = ["10", "50", "100", "500", "1000"]
+        self.sql_keywords = ["select", "from", "where", "group by", "order by", "limit"]
 
-    def __init__(self, folder: str, client: TinyB):
-        super().__init__()
+    def get_completions(self, document, complete_event):
+        text = document.text_before_cursor.strip()
+        words = text.split()
+
+        # Normalize command by removing 'tb' prefix if present
+        if words and words[0] == "tb":
+            words = words[1:]
+
+        if not words:
+            # Show all available commands when no input
+            yield from self._yield_static_commands("")
+            return
+
+        command = words[0].lower()
+
+        if command == "mock":
+            yield from self._handle_mock_completions(words)
+        elif command == "select" or self._is_sql_query(text.lower()):
+            yield from self._handle_sql_completions(text)
+        else:
+            # Handle general command completions
+            yield from self._yield_static_commands(words[-1])
+
+    def _is_sql_query(self, text: str) -> bool:
+        """Check if the input looks like a SQL query."""
+        sql_starters = ["select", "with"]
+        return any(text.startswith(starter) for starter in sql_starters)
+
+    def _handle_sql_completions(self, text: str):
+        """Handle completions for SQL queries."""
+        text_lower = text.lower()
+
+        # Find the last complete word
+        words = text_lower.split()
+        if not words:
+            return
+
+        # If we just typed 'from' or there's a space after 'from', suggest datasources
+        if words[-1] == "from" or (
+            "from" in words and len(words) > words.index("from") + 1 and text_lower.endswith(" ")
+        ):
+            for x in self.datasources:
+                yield Completion(x, start_position=0, display=x, style="class:completion.datasource")
+            for x in self.pipes:
+                yield Completion(x, start_position=0, display=x, style="class:completion.pipe")
+            return
+
+        # If we're starting a query, suggest SQL keywords
+        if len(words) <= 2:
+            for keyword in self.sql_keywords:
+                if keyword.lower().startswith(words[-1]):
+                    yield Completion(
+                        keyword, start_position=-len(words[-1]), display=keyword, style="class:completion.keyword"
+                    )
+
+    def _handle_mock_completions(self, words: List[str]):
+        if len(words) == 1:
+            # After 'mock', show datasources
+            for ds in self.datasources:
+                yield Completion(ds, start_position=0, display=ds, style="class:completion.cmd")
+            return
+
+        if len(words) == 2 or len(words) == 4:
+            # After datasource or after a flag value, show available flags
+            available_flags = [f for f in self.mock_flags if f not in words]
+            for flag in available_flags:
+                yield Completion(flag, start_position=0, display=flag)
+            return
+
+        last_word = words[-1]
+        if last_word == "--prompt":
+            yield Completion('""', start_position=0, display='"Enter your prompt..."')
+        elif last_word == "--rows":
+            for rows in self.common_rows:
+                yield Completion(rows, start_position=0, display=rows)
+
+    def _yield_static_commands(self, current_word: str):
+        for cmd in self.static_commands:
+            if cmd.startswith(current_word):
+                yield Completion(
+                    cmd,
+                    start_position=-len(current_word) if current_word else 0,
+                    display=cmd,
+                    style="class:completion.cmd",
+                )
+
+
+style = Style.from_dict(
+    {
+        "prompt": "fg:#34D399 bold",
+        "completion.cmd": "fg:#34D399 bg:#111111 bold",
+        "completion.datasource": "fg:#AB49D0 bg:#111111",
+        "completion.pipe": "fg:#FEA827 bg:#111111",
+        "completion.keyword": "fg:#34D399 bg:#111111",
+    }
+)
+
+key_bindings = KeyBindings()
+
+
+@key_bindings.add("c-d")
+def _(event):
+    """
+    Start auto completion. If the menu is showing already, select the next
+    completion.
+    """
+    b = event.app.current_buffer
+    if b.complete_state:
+        b.complete_next()
+    else:
+        b.start_completion(select_first=False)
+
+
+class BuildShell:
+    def __init__(self, folder: str, client: TinyB, datasources: List[str], pipes: List[str]):
+        self.history = self.get_history()
         self.folder = folder
         self.client = client
+        self.datasources = datasources
+        self.pipes = pipes
+        self.prompt_message = "\ntb > "
+        self.commands = ["create", "mock", "test", "tb", "select"]
 
-    def do_exit(self, arg):
-        sys.exit(0)
+        self.session = PromptSession(
+            completer=DynamicCompleter(self.datasources, self.pipes),
+            complete_style=CompleteStyle.COLUMN,
+            complete_while_typing=True,
+            history=self.history,
+        )
 
-    def do_quit(self, arg):
-        sys.exit(0)
+    def get_history(self):
+        try:
+            history_file = os.path.expanduser("~/.tb_history")
+            return FileHistory(history_file)
+        except Exception:
+            return None
 
-    def default(self, argline):
-        click.echo("")
-        if argline.startswith("tb build"):
-            click.echo(FeedbackManager.error(message="Build command is already running"))
+    def run_shell(self):
+        while True:
+            try:
+                user_input = self.session.prompt(
+                    [("class:prompt", self.prompt_message)], style=style, key_bindings=key_bindings
+                )
+                self.handle_input(user_input)
+            except (EOFError, KeyboardInterrupt):
+                sys.exit(0)
+            except CLIException as e:
+                click.echo(str(e))
+            except Exception as e:
+                # Catch-all for unexpected exceptions
+                click.echo(FeedbackManager.error_exception(error=str(e)))
+
+    def handle_input(self, argline):
+        line = argline.strip()
+        if not line:
+            return
+
+        # Implement the command logic here
+        # Replace do_* methods with equivalent logic:
+        command_parts = line.split(maxsplit=1)
+        cmd = command_parts[0].lower()
+        arg = command_parts[1] if len(command_parts) > 1 else ""
+
+        if cmd in ["exit", "quit"]:
+            sys.exit(0)
+        elif cmd == "build":
+            self.handle_build(arg)
+        elif cmd == "auth":
+            self.handle_auth(arg)
+        elif cmd == "workspace":
+            self.handle_workspace(arg)
+        elif cmd == "mock":
+            self.handle_mock(arg)
+        elif cmd == "tb":
+            self.handle_tb(arg)
         else:
-            arg_stripped = argline.strip().lower()
-            if not arg_stripped:
-                return
-            if arg_stripped.startswith("tb"):
-                arg_stripped = arg_stripped.replace("tb", "tb --local")
-                extra_args = f" --folder {self.folder}" if arg_stripped.startswith("tb mock") else ""
-                subprocess.run(arg_stripped + extra_args, shell=True, text=True)
-            elif arg_stripped.startswith("with") or arg_stripped.startswith("select"):
-                try:
-                    self.run_sql(self.client, argline)
-                except Exception as e:
-                    click.echo(FeedbackManager.error(message=str(e)))
+            # Check if it looks like a SQL query or run as a tb command
+            self.default(line)
 
-            elif arg_stripped.startswith("mock "):
-                subprocess.run(f"tb {arg_stripped} --folder {self.folder}", shell=True, text=True)
-            else:
-                click.echo(FeedbackManager.error(message="Invalid command"))
+    def handle_build(self, arg):
+        click.echo(FeedbackManager.error(message=f"'tb {arg}' command is not available in watch mode"))
 
-    def reprint_prompt(self):
-        self.stdout.write(self.prompt)
-        self.stdout.flush()
+    def handle_auth(self, arg):
+        click.echo(FeedbackManager.error(message=f"'tb {arg}' command is not available in watch mode"))
+
+    def handle_workspace(self, arg):
+        click.echo(FeedbackManager.error(message=f"'tb {arg}' command is not available in watch mode"))
+
+    def handle_mock(self, arg):
+        subprocess.run(f"tb mock {arg} --folder {self.folder}", shell=True, text=True)
+
+    def handle_tb(self, arg):
+        click.echo("")
+        arg = arg.strip().lower()
+        if arg.startswith("build"):
+            self.handle_build(arg)
+        elif arg.startswith("auth"):
+            self.handle_auth(arg)
+        elif arg.startswith("workspace"):
+            self.handle_workspace(arg)
+        elif arg.startswith("mock"):
+            self.handle_mock(arg)
+        else:
+            subprocess.run(f"tb --local {arg}", shell=True, text=True)
+
+    def default(self, argline):
+        click.echo("")
+        arg = argline.strip().lower()
+        if not arg:
+            return
+        if arg.startswith("with") or arg.startswith("select"):
+            try:
+                self.run_sql(argline)
+            except Exception as e:
+                click.echo(FeedbackManager.error(message=str(e)))
+        else:
+            subprocess.run(f"tb --local {arg}", shell=True, text=True)
 
     def run_sql(self, query, rows_limit=20):
         try:
             q = query.strip()
-            if q.startswith("insert"):
+            if q.lower().startswith("insert"):
                 click.echo(FeedbackManager.info_append_data())
                 raise CLIException(FeedbackManager.error_invalid_query())
-            if q.startswith("delete"):
+            if q.lower().startswith("delete"):
                 raise CLIException(FeedbackManager.error_invalid_query())
 
-            # fuck my life
             def run_query_in_thread():
                 loop = asyncio.new_event_loop()
                 asyncio.set_event_loop(loop)
@@ -74,9 +267,6 @@ class BuildShell(cmd.Cmd):
                 finally:
                     loop.close()
 
-            # Run the query in a separate thread
-            import concurrent.futures
-
             with concurrent.futures.ThreadPoolExecutor() as executor:
                 res = executor.submit(run_query_in_thread).result()
 
@@ -91,6 +281,9 @@ class BuildShell(cmd.Cmd):
         else:
             click.echo(FeedbackManager.info_no_rows())
 
+    def reprint_prompt(self):
+        click.echo(f"{bcolors.OKGREEN}{self.prompt_message}{bcolors.ENDC}", nl=False)
+
 
 def print_table_formatted(res: dict, name: str):
     rebuild_colors = [bcolors.FAIL, bcolors.OKBLUE, bcolors.WARNING, bcolors.OKGREEN, bcolors.HEADER]

tinybird/tb/modules/cicd.py

@@ -14,9 +14,7 @@ class Provider(Enum):
     GitLab = 1
 
 
-WORKFLOW_VERSION = "v3.1.0"
-
-DEFAULT_REQUIREMENTS_FILE = "tinybird-cli>=5,<6"
+WORKFLOW_VERSION = "v0.0.1"
 
 GITHUB_CI_YML = """
 name: Tinybird - CI Workflow
@@ -50,6 +48,8 @@ jobs:
         run: curl -LsSf https://api.tinybird.co/static/install.sh | sh
       - name: Build project
         run: tb build
+      - name: Test project
+        run: tb test run
 """
 
 
@@ -64,110 +64,30 @@ stages:
 
 GITLAB_CI_YML = """
 tinybird_ci_workflow:
+  image: ubuntu:latest
   stage: tests
   interruptible: true
   needs: []
   rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"{% if data_project_dir != '.' %}
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
-        - .gitlab/tinybird/*
+        - .gitlab/tinybird/*{% if data_project_dir != '.' %}
         - {{ data_project_dir }}/*
         - {{ data_project_dir }}/**/*{% end %}
   before_script:
+    - apt update && apt install -y curl
     - curl -LsSf https://api.tinybird.co/static/install.sh | sh
   script:
+    - export PATH="$HOME/.local/bin:$PATH"
     - cd $CI_PROJECT_DIR/{{ data_project_dir }}
     - tb build
+    - tb test run
   services:
     - name: tinybirdco/tinybird-local:latest
       alias: tinybird-local
 """
 
 
-EXEC_TEST_SH = """
-#!/usr/bin/env bash
-set -euxo pipefail
-
-export TB_VERSION_WARNING=0
-
-run_test() {
-    t=$1
-    echo "** Running $t **"
-    echo "** $(cat $t)"
-    tmpfile=$(mktemp)
-    retries=0
-    TOTAL_RETRIES=3
-
-    # When appending fixtures, we need to retry in case of the data is not replicated in time
-    while [ $retries -lt $TOTAL_RETRIES ]; do
-        # Run the test and store the output in a temporary file
-        bash $t $2 >$tmpfile
-        exit_code=$?
-        if [ "$exit_code" -eq 0 ]; then
-            # If the test passed, break the loop
-            if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
-                break
-            # If the test failed, increment the retries counter and try again
-            else
-                retries=$((retries+1))
-            fi
-        # If the bash command failed, print an error message and break the loop
-        else
-            break
-        fi
-    done
-
-    if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
-        echo "✅ Test $t passed"
-        rm $tmpfile
-        return 0
-    elif [ $retries -eq $TOTAL_RETRIES ]; then
-        echo "🚨 ERROR: Test $t failed, diff:";
-        diff -B ${t}.result $tmpfile
-        rm $tmpfile
-        return 1
-    else
-        echo "🚨 ERROR: Test $t failed with bash command exit code $?"
-        cat $tmpfile
-        rm $tmpfile
-        return 1
-    fi
-    echo ""
-}
-export -f run_test
-
-fail=0
-find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} || fail=1
-
-if [ $fail == 1 ]; then
-  exit -1;
-fi
-"""
-
-APPEND_FIXTURES_SH = """
-#!/usr/bin/env bash
-set -euxo pipefail
-
-directory="datasources/fixtures"
-extensions=("csv" "ndjson")
-
-absolute_directory=$(realpath "$directory")
-
-for extension in "${extensions[@]}"; do
-  file_list=$(find "$absolute_directory" -type f -name "*.$extension")
-
-  for file_path in $file_list; do
-    file_name=$(basename "$file_path")
-    file_name_without_extension="${file_name%.*}"
-
-    command="tb datasource append $file_name_without_extension datasources/fixtures/$file_name"
-    echo $command
-    $command
-  done
-done
-"""
-
-
 class CICDFile:
     def __init__(
         self,