tinybird 0.0.1.dev17__py3-none-any.whl → 0.0.1.dev19__py3-none-any.whl

tinybird/client.py

@@ -103,7 +103,7 @@ class TinyB:
         self.send_telemetry = send_telemetry
         self.semver = semver
 
-    async def _req(
+    async def _req_raw(
         self,
         endpoint: str,
         data=None,
@@ -155,10 +155,15 @@ class TinyB:
                     response = await sync_to_async(session.get, thread_sensitive=False)(
                         url, verify=verify_ssl, **kwargs
                     )
-
         except Exception as e:
             raise e
 
+        if self.send_telemetry:
+            try:
+                add_telemetry_event("api_request", endpoint=url, token=self.token, status_code=response.status_code)
+            except Exception as ex:
+                logging.exception(f"Can't send telemetry: {ex}")
+
         logging.debug("== server response ==")
         logging.debug(response.content)
         logging.debug("==      end        ==")
@@ -169,6 +174,21 @@ class TinyB:
             except Exception as ex:
                 logging.exception(f"Can't send telemetry: {ex}")
 
+        return response
+
+    async def _req(
+        self,
+        endpoint: str,
+        data=None,
+        files=None,
+        method: str = "GET",
+        retries: int = LIMIT_RETRIES,
+        use_token: Optional[str] = None,
+        **kwargs,
+    ):
+        token_to_use = use_token if use_token else self.token
+        response = await self._req_raw(endpoint, data, files, method, retries, use_token, **kwargs)
+
         if response.status_code == 403:
             error = parse_error_response(response)
             if not token_to_use:

tinybird/tb/__cli__.py

@@ -0,0 +1,8 @@
+
+__name__ = 'tinybird'
+__description__ = 'Tinybird Command Line Tool'
+__url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
+__author__ = 'Tinybird'
+__author_email__ = 'support@tinybird.co'
+__version__ = '0.0.1.dev19'
+__revision__ = '5ef65e7'

tinybird/tb/modules/build.py

@@ -6,7 +6,6 @@ from pathlib import Path
 from typing import Any, Awaitable, Callable, List, Union
 
 import click
-from click import Context
 from watchdog.events import FileSystemEventHandler
 from watchdog.observers import Observer
 
@@ -16,7 +15,7 @@ from tinybird.config import FeatureFlags
 from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb.modules.build_shell import BuildShell, print_table_formatted
 from tinybird.tb.modules.cli import cli
-from tinybird.tb.modules.common import coro, push_data
+from tinybird.tb.modules.common import push_data
 from tinybird.tb.modules.datafile.build import folder_build
 from tinybird.tb.modules.datafile.common import get_project_filenames, get_project_fixtures, has_internal_datafiles
 from tinybird.tb.modules.datafile.exceptions import ParseException
@@ -101,10 +100,7 @@ def watch_files(
     is_flag=True,
     help="Watch for changes in the files and re-check them.",
 )
-@click.pass_context
-@coro
-async def build(
-    ctx: Context,
+def build(
     folder: str,
     watch: bool,
 ) -> None:
@@ -115,7 +111,7 @@ async def build(
     context.disable_template_security_validation.set(True)
     is_internal = has_internal_datafiles(folder)
     folder_path = os.path.abspath(folder)
-    tb_client = await get_tinybird_local_client(folder_path)
+    tb_client = asyncio.run(get_tinybird_local_client(folder_path))
 
     def check_filenames(filenames: List[str]):
         parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
@@ -191,16 +187,42 @@ async def build(
             ok = False
         return ok
 
-    build_ok = await build_once(filenames)
+    build_ok = asyncio.run(build_once(filenames))
 
     if watch:
-        shell = BuildShell(folder=folder, client=tb_client)
+        paths = [Path(f) for f in get_project_filenames(folder, with_vendor=True)]
+
+        def is_vendor(f: Path) -> bool:
+            return f.parts[0] == "vendor"
+
+        def get_vendor_workspace(f: Path) -> str:
+            return f.parts[1]
+
+        def is_endpoint(f: Path) -> bool:
+            return f.suffix == ".pipe" and not is_vendor(f) and f.parts[0] == "endpoints"
+
+        def is_pipe(f: Path) -> bool:
+            return f.suffix == ".pipe" and not is_vendor(f)
+
+        datasource_paths = [f for f in paths if f.suffix == ".datasource"]
+        datasources = [f.stem for f in datasource_paths if not is_vendor(f)]
+        shared_datasources = [f"{get_vendor_workspace(f)}.{f.stem}" for f in datasource_paths if is_vendor(f)]
+        pipes = [f.stem for f in paths if is_pipe(f) and not is_endpoint(f)]
+        endpoints = [f.stem for f in paths if is_endpoint(f)]
+        shell = BuildShell(
+            folder=folder,
+            client=tb_client,
+            datasources=datasources,
+            shared_datasources=shared_datasources,
+            pipes=pipes,
+            endpoints=endpoints,
+        )
         click.echo(FeedbackManager.highlight(message="◎ Watching for changes..."))
         watcher_thread = threading.Thread(
             target=watch_files, args=(filenames, process, shell, folder, build_ok), daemon=True
         )
         watcher_thread.start()
-        shell.cmdloop()
+        shell.run_shell()
 
 
 async def build_and_print_resource(tb_client: TinyB, filename: str):

tinybird/tb/modules/build_shell.py

@@ -1,11 +1,19 @@
 import asyncio
-import cmd
+import concurrent.futures
+import os
 import random
 import subprocess
 import sys
+from typing import List
 
 import click
 import humanfriendly
+from prompt_toolkit import PromptSession
+from prompt_toolkit.completion import Completer, Completion
+from prompt_toolkit.history import FileHistory
+from prompt_toolkit.key_binding import KeyBindings
+from prompt_toolkit.shortcuts import CompleteStyle
+from prompt_toolkit.styles import Style
 
 from tinybird.client import TinyB
 from tinybird.feedback_manager import FeedbackManager, bcolors
@@ -13,43 +21,260 @@ from tinybird.tb.modules.exceptions import CLIException
 from tinybird.tb.modules.table import format_table
 
 
-class BuildShell(cmd.Cmd):
-    prompt = "\n\001\033[1;32m\002tb > \001\033[0m\002"
+class DynamicCompleter(Completer):
+    def __init__(self, datasources: List[str], shared_datasources: List[str], endpoints: List[str], pipes: List[str]):
+        self.datasources = datasources
+        self.shared_datasources = shared_datasources
+        self.endpoints = endpoints
+        self.pipes = pipes
+        self.static_commands = ["create", "mock", "test", "select"]
+        self.test_commands = ["create", "run", "update"]
+        self.mock_flags = ["--prompt", "--rows"]
+        self.common_rows = ["10", "50", "100", "500", "1000"]
+        self.sql_keywords = ["select", "from", "where", "group by", "order by", "limit"]
 
-    def __init__(self, folder: str, client: TinyB):
-        super().__init__()
+    def get_completions(self, document, complete_event):
+        text = document.text_before_cursor.strip()
+        words = text.split()
+
+        # Normalize command by removing 'tb' prefix if present
+        if words and words[0] == "tb":
+            words = words[1:]
+
+        if not words:
+            # Show all available commands when no input
+            yield from self._yield_static_commands("")
+            return
+
+        command = words[0].lower()
+
+        if command == "mock":
+            yield from self._handle_mock_completions(words)
+        elif command == "test":
+            yield from self._handle_test_completions(words)
+        elif command == "select" or self._is_sql_query(text.lower()):
+            yield from self._handle_sql_completions(text)
+        else:
+            # Handle general command completions
+            yield from self._yield_static_commands(words[-1])
+
+    def _is_sql_query(self, text: str) -> bool:
+        """Check if the input looks like a SQL query."""
+        sql_starters = ["select", "with"]
+        return any(text.startswith(starter) for starter in sql_starters)
+
+    def _handle_sql_completions(self, text: str):
+        """Handle completions for SQL queries."""
+        text_lower = text.lower()
+
+        # Find the last complete word
+        words = text_lower.split()
+        if not words:
+            return
+
+        # If we just typed 'from' or there's a space after 'from', suggest datasources
+        if words[-1] == "from" or (
+            "from" in words and len(words) > words.index("from") + 1 and text_lower.endswith(" ")
+        ):
+            for x in self.datasources + self.shared_datasources:
+                yield Completion(x, start_position=0, display=x, style="class:completion.datasource")
+            for x in self.endpoints + self.pipes:
+                yield Completion(x, start_position=0, display=x, style="class:completion.pipe")
+            return
+
+        # If we're starting a query, suggest SQL keywords
+        if len(words) <= 2:
+            for keyword in self.sql_keywords:
+                if keyword.lower().startswith(words[-1]):
+                    yield Completion(
+                        keyword, start_position=-len(words[-1]), display=keyword, style="class:completion.keyword"
+                    )
+
+    def _handle_mock_completions(self, words: List[str]):
+        if len(words) == 1:
+            # After 'mock', show datasources
+            for ds in self.datasources:
+                yield Completion(ds, start_position=0, display=ds, style="class:completion.datasource")
+            return
+
+        if len(words) == 2 or len(words) == 4:
+            # After datasource or after a flag value, show available flags
+            available_flags = [f for f in self.mock_flags if f not in words]
+            for flag in available_flags:
+                yield Completion(flag, start_position=0, display=flag, style="class:completion.cmd")
+            return
+
+        last_word = words[-1]
+        if last_word == "--prompt":
+            yield Completion('""', start_position=0, display='"Enter your prompt..."', style="class:completion.cmd")
+        elif last_word == "--rows":
+            for rows in self.common_rows:
+                yield Completion(rows, start_position=0, display=rows, style="class:completion.cmd")
+
+    def _handle_test_completions(self, words: List[str]):
+        if len(words) == 1:
+            for cmd in self.test_commands:
+                yield Completion(cmd, start_position=0, display=cmd, style="class:completion.cmd")
+            return
+        elif len(words) == 2:
+            for cmd in self.endpoints:
+                yield Completion(cmd, start_position=0, display=cmd, style="class:completion.pipe")
+            return
+
+    def _yield_static_commands(self, current_word: str):
+        for cmd in self.static_commands:
+            if cmd.startswith(current_word):
+                yield Completion(
+                    cmd,
+                    start_position=-len(current_word) if current_word else 0,
+                    display=cmd,
+                    style="class:completion.cmd",
+                )
+
+        for cmd in self.datasources + self.shared_datasources:
+            if cmd.startswith(current_word):
+                yield Completion(
+                    cmd,
+                    start_position=-len(current_word) if current_word else 0,
+                    display=cmd,
+                    style="class:completion.datasource",
+                )
+
+        for cmd in self.endpoints + self.pipes:
+            if cmd.startswith(current_word):
+                yield Completion(
+                    cmd,
+                    start_position=-len(current_word) if current_word else 0,
+                    display=cmd,
+                    style="class:completion.pipe",
+                )
+
+
+style = Style.from_dict(
+    {
+        "prompt": "fg:#34D399 bold",
+        "completion.cmd": "fg:#34D399 bg:#111111 bold",
+        "completion.datasource": "fg:#AB49D0 bg:#111111",
+        "completion.pipe": "fg:#FEA827 bg:#111111",
+        "completion.keyword": "fg:#34D399 bg:#111111",
+    }
+)
+
+key_bindings = KeyBindings()
+
+
+@key_bindings.add("c-d")
+def _(event):
+    """
+    Start auto completion. If the menu is showing already, select the next
+    completion.
+    """
+    b = event.app.current_buffer
+    if b.complete_state:
+        b.complete_next()
+    else:
+        b.start_completion(select_first=False)
+
+
+class BuildShell:
+    def __init__(
+        self,
+        folder: str,
+        client: TinyB,
+        datasources: List[str],
+        shared_datasources: List[str],
+        pipes: List[str],
+        endpoints: List[str],
+    ):
+        self.history = self.get_history()
         self.folder = folder
         self.client = client
+        self.datasources = datasources
+        self.shared_datasources = shared_datasources
+        self.pipes = pipes
+        self.endpoints = endpoints
+        self.prompt_message = "\ntb > "
+        self.commands = ["create", "mock", "test", "tb", "select"]
+
+        self.session = PromptSession(
+            completer=DynamicCompleter(self.datasources, self.shared_datasources, self.endpoints, self.pipes),
+            complete_style=CompleteStyle.COLUMN,
+            complete_while_typing=True,
+            history=self.history,
+        )
+
+    def get_history(self):
+        try:
+            history_file = os.path.expanduser("~/.tb_history")
+            return FileHistory(history_file)
+        except Exception:
+            return None
+
+    def run_shell(self):
+        while True:
+            try:
+                user_input = self.session.prompt(
+                    [("class:prompt", self.prompt_message)], style=style, key_bindings=key_bindings
+                )
+                self.handle_input(user_input)
+            except (EOFError, KeyboardInterrupt):
+                sys.exit(0)
+            except CLIException as e:
+                click.echo(str(e))
+            except Exception as e:
+                # Catch-all for unexpected exceptions
+                click.echo(FeedbackManager.error_exception(error=str(e)))
+
+    def handle_input(self, argline):
+        line = argline.strip()
+        if not line:
+            return
 
-    def do_exit(self, arg):
-        sys.exit(0)
+        # Implement the command logic here
+        # Replace do_* methods with equivalent logic:
+        command_parts = line.split(maxsplit=1)
+        cmd = command_parts[0].lower()
+        arg = command_parts[1] if len(command_parts) > 1 else ""
 
-    def do_quit(self, arg):
-        sys.exit(0)
+        if cmd in ["exit", "quit"]:
+            sys.exit(0)
+        elif cmd == "build":
+            self.handle_build(arg)
+        elif cmd == "auth":
+            self.handle_auth(arg)
+        elif cmd == "workspace":
+            self.handle_workspace(arg)
+        elif cmd == "mock":
+            self.handle_mock(arg)
+        elif cmd == "tb":
+            self.handle_tb(arg)
+        else:
+            # Check if it looks like a SQL query or run as a tb command
+            self.default(line)
 
-    def do_build(self, arg):
+    def handle_build(self, arg):
         click.echo(FeedbackManager.error(message=f"'tb {arg}' command is not available in watch mode"))
 
-    def do_auth(self, arg):
+    def handle_auth(self, arg):
         click.echo(FeedbackManager.error(message=f"'tb {arg}' command is not available in watch mode"))
 
-    def do_workspace(self, arg):
+    def handle_workspace(self, arg):
         click.echo(FeedbackManager.error(message=f"'tb {arg}' command is not available in watch mode"))
 
-    def do_mock(self, arg):
+    def handle_mock(self, arg):
         subprocess.run(f"tb mock {arg} --folder {self.folder}", shell=True, text=True)
 
-    def do_tb(self, arg):
+    def handle_tb(self, arg):
         click.echo("")
         arg = arg.strip().lower()
         if arg.startswith("build"):
-            self.do_build(arg)
+            self.handle_build(arg)
         elif arg.startswith("auth"):
-            self.do_auth(arg)
+            self.handle_auth(arg)
         elif arg.startswith("workspace"):
-            self.do_workspace(arg)
+            self.handle_workspace(arg)
         elif arg.startswith("mock"):
-            self.do_mock(arg)
+            self.handle_mock(arg)
         else:
             subprocess.run(f"tb --local {arg}", shell=True, text=True)
 
@@ -59,27 +284,21 @@ class BuildShell(cmd.Cmd):
         if not arg:
             return
         if arg.startswith("with") or arg.startswith("select"):
-            try:
-                self.run_sql(argline)
-            except Exception as e:
-                click.echo(FeedbackManager.error(message=str(e)))
+            self.run_sql(argline)
+        elif len(arg.split()) == 1 and arg in self.endpoints + self.pipes + self.datasources + self.shared_datasources:
+            self.run_sql(f"select * from {arg}")
         else:
             subprocess.run(f"tb --local {arg}", shell=True, text=True)
 
-    def reprint_prompt(self):
-        self.stdout.write(self.prompt)
-        self.stdout.flush()
-
     def run_sql(self, query, rows_limit=20):
         try:
             q = query.strip()
-            if q.startswith("insert"):
+            if q.lower().startswith("insert"):
                 click.echo(FeedbackManager.info_append_data())
                 raise CLIException(FeedbackManager.error_invalid_query())
-            if q.startswith("delete"):
+            if q.lower().startswith("delete"):
                 raise CLIException(FeedbackManager.error_invalid_query())
 
-            # fuck my life
             def run_query_in_thread():
                 loop = asyncio.new_event_loop()
                 asyncio.set_event_loop(loop)
@@ -90,23 +309,23 @@ class BuildShell(cmd.Cmd):
                 finally:
                     loop.close()
 
-            # Run the query in a separate thread
-            import concurrent.futures
-
             with concurrent.futures.ThreadPoolExecutor() as executor:
                 res = executor.submit(run_query_in_thread).result()
 
         except Exception as e:
-            raise CLIException(FeedbackManager.error_exception(error=str(e)))
+            click.echo(FeedbackManager.error_exception(error=str(e)))
 
         if isinstance(res, dict) and "error" in res:
-            raise CLIException(FeedbackManager.error_exception(error=res["error"]))
+            click.echo(FeedbackManager.error_exception(error=res["error"]))
 
         if isinstance(res, dict) and "data" in res and res["data"]:
             print_table_formatted(res, "QUERY")
         else:
             click.echo(FeedbackManager.info_no_rows())
 
+    def reprint_prompt(self):
+        click.echo(f"{bcolors.OKGREEN}{self.prompt_message}{bcolors.ENDC}", nl=False)
+
 
 def print_table_formatted(res: dict, name: str):
     rebuild_colors = [bcolors.FAIL, bcolors.OKBLUE, bcolors.WARNING, bcolors.OKGREEN, bcolors.HEADER]

tinybird/tb/modules/cicd.py

@@ -14,9 +14,7 @@ class Provider(Enum):
     GitLab = 1
 
 
-WORKFLOW_VERSION = "v3.1.0"
-
-DEFAULT_REQUIREMENTS_FILE = "tinybird-cli>=5,<6"
+WORKFLOW_VERSION = "v0.0.1"
 
 GITHUB_CI_YML = """
 name: Tinybird - CI Workflow
@@ -50,6 +48,8 @@ jobs:
         run: curl -LsSf https://api.tinybird.co/static/install.sh | sh
       - name: Build project
         run: tb build
+      - name: Test project
+        run: tb test run
 """
 
 
@@ -64,110 +64,30 @@ stages:
 
 GITLAB_CI_YML = """
 tinybird_ci_workflow:
+  image: ubuntu:latest
   stage: tests
   interruptible: true
   needs: []
   rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"{% if data_project_dir != '.' %}
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
-        - .gitlab/tinybird/*
+        - .gitlab/tinybird/*{% if data_project_dir != '.' %}
         - {{ data_project_dir }}/*
         - {{ data_project_dir }}/**/*{% end %}
   before_script:
+    - apt update && apt install -y curl
     - curl -LsSf https://api.tinybird.co/static/install.sh | sh
   script:
+    - export PATH="$HOME/.local/bin:$PATH"
     - cd $CI_PROJECT_DIR/{{ data_project_dir }}
     - tb build
+    - tb test run
   services:
     - name: tinybirdco/tinybird-local:latest
       alias: tinybird-local
 """
 
 
-EXEC_TEST_SH = """
-#!/usr/bin/env bash
-set -euxo pipefail
-
-export TB_VERSION_WARNING=0
-
-run_test() {
-    t=$1
-    echo "** Running $t **"
-    echo "** $(cat $t)"
-    tmpfile=$(mktemp)
-    retries=0
-    TOTAL_RETRIES=3
-
-    # When appending fixtures, we need to retry in case of the data is not replicated in time
-    while [ $retries -lt $TOTAL_RETRIES ]; do
-        # Run the test and store the output in a temporary file
-        bash $t $2 >$tmpfile
-        exit_code=$?
-        if [ "$exit_code" -eq 0 ]; then
-            # If the test passed, break the loop
-            if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
-                break
-            # If the test failed, increment the retries counter and try again
-            else
-                retries=$((retries+1))
-            fi
-        # If the bash command failed, print an error message and break the loop
-        else
-            break
-        fi
-    done
-
-    if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
-        echo "✅ Test $t passed"
-        rm $tmpfile
-        return 0
-    elif [ $retries -eq $TOTAL_RETRIES ]; then
-        echo "🚨 ERROR: Test $t failed, diff:";
-        diff -B ${t}.result $tmpfile
-        rm $tmpfile
-        return 1
-    else
-        echo "🚨 ERROR: Test $t failed with bash command exit code $?"
-        cat $tmpfile
-        rm $tmpfile
-        return 1
-    fi
-    echo ""
-}
-export -f run_test
-
-fail=0
-find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} || fail=1
-
-if [ $fail == 1 ]; then
-  exit -1;
-fi
-"""
-
-APPEND_FIXTURES_SH = """
-#!/usr/bin/env bash
-set -euxo pipefail
-
-directory="datasources/fixtures"
-extensions=("csv" "ndjson")
-
-absolute_directory=$(realpath "$directory")
-
-for extension in "${extensions[@]}"; do
-  file_list=$(find "$absolute_directory" -type f -name "*.$extension")
-
-  for file_path in $file_list; do
-    file_name=$(basename "$file_path")
-    file_name_without_extension="${file_name%.*}"
-
-    command="tb datasource append $file_name_without_extension datasources/fixtures/$file_name"
-    echo $command
-    $command
-  done
-done
-"""
-
-
 class CICDFile:
     def __init__(
         self,

tinybird/tb/modules/cli.py

@@ -26,8 +26,9 @@ from tinybird.client import (
     DoesNotExistException,
     TinyB,
 )
-from tinybird.config import SUPPORTED_CONNECTORS, VERSION, FeatureFlags, get_config
+from tinybird.config import SUPPORTED_CONNECTORS, FeatureFlags, get_config
 from tinybird.feedback_manager import FeedbackManager
+from tinybird.tb import __cli__
 from tinybird.tb.modules.common import (
     OLDEST_ROLLBACK,
     CatchAuthExceptions,
@@ -70,6 +71,7 @@ __old_click_secho = click.secho
 DEFAULT_PATTERNS: List[Tuple[str, Union[str, Callable[[str], str]]]] = [
     (r"p\.ey[A-Za-z0-9-_\.]+", lambda v: f"{v[:4]}...{v[-8:]}")
 ]
+VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
 
 
 @click.group(cls=CatchAuthExceptions, context_settings={"help_option_names": ["-h", "--help"]})