tinybird 0.0.1.dev4__py3-none-any.whl → 0.0.1.dev6__py3-none-any.whl

tinybird/__cli__.py

@@ -1,8 +1,7 @@
-
-__name__ = 'tinybird'
-__description__ = 'Tinybird Command Line Tool'
-__url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
-__author__ = 'Tinybird'
-__author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev4'
-__revision__ = '388c81a'
+__name__ = "tinybird-cli"
+__description__ = "Tinybird Command Line Tool"
+__url__ = "https://www.tinybird.co/docs/cli/introduction.html"
+__author__ = "Tinybird"
+__author_email__ = "support@tinybird.co"
+__version__ = "x.y.z"
+__revision__ = "xxxxxxxxx"

tinybird/check_pypi.py

@@ -6,7 +6,7 @@ from tinybird.feedback_manager import FeedbackManager
 from tinybird.syncasync import sync_to_async
 from tinybird.tb_cli_modules.common import CLIException, getenv_bool
 
-PYPY_URL = "https://pypi.org/pypi/tinybird-cli/json"
+PYPY_URL = "https://pypi.org/pypi/tinybird/json"
 requests_get = sync_to_async(requests.get, thread_sensitive=False)
 
 

tinybird/feedback_manager.py

@@ -565,9 +565,9 @@ Ready? """
     )
     warning_fixture_not_found = warning_message("** Warning: No fixture found for the datasource {datasource_name}")
     warning_update_version = warning_message(
-        '** UPDATE AVAILABLE: please run "pip install tinybird-cli=={latest_version}" to update or `export TB_VERSION_WARNING=0` to skip the check.'
+        '** UPDATE AVAILABLE: please run "pip install tinybird=={latest_version}" to update or `export TB_VERSION_WARNING=0` to skip the check.'
     )
-    warning_current_version = warning_message("** current: tinybird-cli {current_version}\n")
+    warning_current_version = warning_message("** current: tinybird {current_version}\n")
     warning_confirm_truncate_datasource = prompt_message(
         "Do you want to truncate {datasource}? Once truncated, your data can't be recovered"
     )

tinybird/tb/cli.py

@@ -0,0 +1,28 @@
+import asyncio
+import sys
+
+if sys.platform == "win32":
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+
+import tinybird.tb.modules.auth
+import tinybird.tb.modules.branch
+import tinybird.tb.modules.build
+import tinybird.tb.modules.cli
+import tinybird.tb.modules.common
+import tinybird.tb.modules.connection
+import tinybird.tb.modules.create
+import tinybird.tb.modules.datasource
+import tinybird.tb.modules.fmt
+import tinybird.tb.modules.job
+import tinybird.tb.modules.mock
+import tinybird.tb.modules.pipe
+import tinybird.tb.modules.tag
+import tinybird.tb.modules.test
+import tinybird.tb.modules.token
+import tinybird.tb.modules.workspace
+import tinybird.tb.modules.workspace_members
+
+cli = tinybird.tb.modules.cli.cli
+
+if __name__ == "__main__":
+    cli()

tinybird/{tb_cli_modules → tb/modules}/auth.py

@@ -11,8 +11,8 @@ import humanfriendly.tables
 
 from tinybird.config import get_display_host
 from tinybird.feedback_manager import FeedbackManager
-from tinybird.tb_cli_modules.cli import cli
-from tinybird.tb_cli_modules.common import (
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.common import (
     configure_connector,
     coro,
     echo_safe_humanfriendly_tables_format_smart_table,
@@ -21,9 +21,9 @@ from tinybird.tb_cli_modules.common import (
     try_authenticate,
     try_update_config_with_remote,
 )
-from tinybird.tb_cli_modules.config import CLIConfig, ConfigValueOrigin
-from tinybird.tb_cli_modules.exceptions import CLIAuthException
-from tinybird.tb_cli_modules.regions import Region
+from tinybird.tb.modules.config import CLIConfig, ConfigValueOrigin
+from tinybird.tb.modules.exceptions import CLIAuthException
+from tinybird.tb.modules.regions import Region
 
 
 @cli.group(invoke_without_command=True)

tinybird/{tb_cli_modules → tb/modules}/branch.py

@@ -11,10 +11,9 @@ import aiofiles
 import click
 import yaml
 
-from tinybird.datafile import create_release, wait_job
 from tinybird.feedback_manager import FeedbackManager
-from tinybird.tb_cli_modules.cli import cli
-from tinybird.tb_cli_modules.common import (
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.common import (
     MAIN_BRANCH,
     OLDEST_ROLLBACK,
     coro,
@@ -35,9 +34,11 @@ from tinybird.tb_cli_modules.common import (
     switch_to_workspace_by_user_workspace_data,
     switch_workspace,
     try_update_config_with_remote,
+    wait_job,
 )
-from tinybird.tb_cli_modules.config import CLIConfig
-from tinybird.tb_cli_modules.exceptions import CLIBranchException, CLIException, CLIReleaseException
+from tinybird.tb.modules.config import CLIConfig
+from tinybird.tb.modules.datafile import create_release
+from tinybird.tb.modules.exceptions import CLIBranchException, CLIException, CLIReleaseException
 
 
 @cli.group(hidden=True)

tinybird/{tb_cli_modules → tb/modules}/build.py

@@ -13,7 +13,12 @@ from watchdog.observers import Observer
 import tinybird.context as context
 from tinybird.client import TinyB
 from tinybird.config import FeatureFlags
-from tinybird.datafile import (
+from tinybird.feedback_manager import FeedbackManager, bcolors
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.common import (
+    coro,
+)
+from tinybird.tb.modules.datafile import (
     ParseException,
     folder_build,
     get_project_filenames,
@@ -21,17 +26,10 @@ from tinybird.datafile import (
     parse_datasource,
     parse_pipe,
 )
-from tinybird.feedback_manager import FeedbackManager, bcolors
-from tinybird.tb_cli_modules.cli import cli
-from tinybird.tb_cli_modules.common import (
-    coro,
-)
-from tinybird.tb_cli_modules.config import CLIConfig
-from tinybird.tb_cli_modules.llm import LLM
-from tinybird.tb_cli_modules.local import (
+from tinybird.tb.modules.local import (
     get_tinybird_local_client,
 )
-from tinybird.tb_cli_modules.table import format_table
+from tinybird.tb.modules.table import format_table
 
 
 class FileChangeHandler(FileSystemEventHandler):
@@ -153,22 +151,6 @@ async def build(
             only_pipes=only_pipes,
         )
 
-        for filename in filenames:
-            if filename.endswith(".datasource") and not skip_datasources:
-                ds_path = Path(filename)
-                ds_name = ds_path.stem
-                datasource_content = ds_path.read_text()
-                llm_config = CLIConfig.get_llm_config()
-                llm = LLM(key=llm_config["api_key"])
-                has_json_path = "`json:" in datasource_content
-
-                if has_json_path:
-                    await llm.generate_sql_sample_data(
-                        tb_client=tb_client,
-                        datasource_name=ds_name,
-                        datasource_content=datasource_content,
-                    )
-
         if watch:
             filename = filenames[0]
             if filename.endswith(".pipe"):

tinybird/tb/modules/cicd.py

@@ -0,0 +1,271 @@
+from enum import Enum
+from os import getcwd
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Type, Union
+
+import click
+from tornado.template import Template
+
+from tinybird.feedback_manager import FeedbackManager
+
+
+class Provider(Enum):
+    GitHub = 0
+    GitLab = 1
+
+
+WORKFLOW_VERSION = "v3.1.0"
+
+DEFAULT_REQUIREMENTS_FILE = "tinybird-cli>=5,<6"
+
+GITHUB_CI_YML = """
+name: Tinybird - CI Workflow
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+      - master
+    types: [opened, reopened, labeled, unlabeled, synchronize, closed]{% if data_project_dir != '.' %}
+    paths:
+      - '{{ data_project_dir }}/**'{% end %}
+
+concurrency: ${{! github.workflow }}-${{! github.event.pull_request.number }}
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: '{{ data_project_dir }}'
+    services:
+      tinybird:
+        image: tinybirdco/tinybird-local:latest
+        ports:
+          - 80:80
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Tinybird CLI
+        run: curl -LsSf https://api.tinybird.co/static/install.sh | sh
+      - name: Build project
+        run: tb build
+"""
+
+
+GITLAB_YML = """
+include:
+  - local: .gitlab/tinybird/*.yml
+
+stages:
+  - tests
+"""
+
+
+GITLAB_CI_YML = """
+tinybird_ci_workflow:
+  stage: tests
+  interruptible: true
+  needs: []
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"{% if data_project_dir != '.' %}
+      changes:
+        - .gitlab/tinybird/*
+        - {{ data_project_dir }}/*
+        - {{ data_project_dir }}/**/*{% end %}
+  before_script:
+    - curl -LsSf https://api.tinybird.co/static/install.sh | sh
+  script:
+    - cd $CI_PROJECT_DIR/{{ data_project_dir }}
+    - tb build
+  services:
+    - name: tinybirdco/tinybird-local:latest
+      alias: tinybird-local
+"""
+
+
+EXEC_TEST_SH = """
+#!/usr/bin/env bash
+set -euxo pipefail
+
+export TB_VERSION_WARNING=0
+
+run_test() {
+    t=$1
+    echo "** Running $t **"
+    echo "** $(cat $t)"
+    tmpfile=$(mktemp)
+    retries=0
+    TOTAL_RETRIES=3
+
+    # When appending fixtures, we need to retry in case of the data is not replicated in time
+    while [ $retries -lt $TOTAL_RETRIES ]; do
+        # Run the test and store the output in a temporary file
+        bash $t $2 >$tmpfile
+        exit_code=$?
+        if [ "$exit_code" -eq 0 ]; then
+            # If the test passed, break the loop
+            if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
+                break
+            # If the test failed, increment the retries counter and try again
+            else
+                retries=$((retries+1))
+            fi
+        # If the bash command failed, print an error message and break the loop
+        else
+            break
+        fi
+    done
+
+    if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
+        echo "✅ Test $t passed"
+        rm $tmpfile
+        return 0
+    elif [ $retries -eq $TOTAL_RETRIES ]; then
+        echo "🚨 ERROR: Test $t failed, diff:";
+        diff -B ${t}.result $tmpfile
+        rm $tmpfile
+        return 1
+    else
+        echo "🚨 ERROR: Test $t failed with bash command exit code $?"
+        cat $tmpfile
+        rm $tmpfile
+        return 1
+    fi
+    echo ""
+}
+export -f run_test
+
+fail=0
+find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} || fail=1
+
+if [ $fail == 1 ]; then
+  exit -1;
+fi
+"""
+
+APPEND_FIXTURES_SH = """
+#!/usr/bin/env bash
+set -euxo pipefail
+
+directory="datasources/fixtures"
+extensions=("csv" "ndjson")
+
+absolute_directory=$(realpath "$directory")
+
+for extension in "${extensions[@]}"; do
+  file_list=$(find "$absolute_directory" -type f -name "*.$extension")
+
+  for file_path in $file_list; do
+    file_name=$(basename "$file_path")
+    file_name_without_extension="${file_name%.*}"
+
+    command="tb datasource append $file_name_without_extension datasources/fixtures/$file_name"
+    echo $command
+    $command
+  done
+done
+"""
+
+
+class CICDFile:
+    def __init__(
+        self,
+        template: str,
+        file_name: str,
+        dir_path: Optional[str] = None,
+        warning_message: Optional[str] = None,
+    ):
+        self.template = template
+        self.file_name = file_name
+        self.dir_path = dir_path
+        self.warning_message = warning_message
+
+    @property
+    def full_path(self) -> str:
+        return f"{self.dir_path}/{self.file_name}" if self.dir_path else self.file_name
+
+
+class CICDGeneratorBase:
+    cicd_files: List[CICDFile] = []
+
+    def __call__(self, path: str, params: Dict[str, Any]):
+        for cicd_file in self.cicd_files:
+            if cicd_file.dir_path:
+                Path(f"{path}/{cicd_file.dir_path}").mkdir(parents=True, exist_ok=True)
+            content = Template(cicd_file.template).generate(**params)
+            if Path(f"{path}/{cicd_file.full_path}").exists():
+                continue
+            with open(f"{path}/{cicd_file.full_path}", "wb") as f:
+                f.write(content)
+            click.echo(FeedbackManager.info_cicd_file_generated(file_path=cicd_file.full_path))
+            if cicd_file.warning_message is not None:
+                return FeedbackManager.warning_for_cicd_file(
+                    file_name=cicd_file.file_name, warning_message=cicd_file.warning_message.format(**params)
+                )
+
+    def is_already_generated(self, path: str) -> bool:
+        for cicd_file in self.cicd_files:
+            if cicd_file.file_name and Path(f"{path}/{cicd_file.full_path}").exists():
+                return True
+        return False
+
+    @classmethod
+    def build_generator(cls, provider: str) -> Union["GitHubCICDGenerator", "GitLabCICDGenerator"]:
+        builder: Dict[str, Union[Type[GitHubCICDGenerator], Type[GitLabCICDGenerator]]] = {
+            Provider.GitHub.name: GitHubCICDGenerator,
+            Provider.GitLab.name: GitLabCICDGenerator,
+        }
+        return builder[provider]()
+
+
+class GitHubCICDGenerator(CICDGeneratorBase):
+    cicd_files = [
+        CICDFile(
+            template=GITHUB_CI_YML,
+            file_name="tinybird-ci.yml",
+            dir_path=".github/workflows",
+        ),
+    ]
+
+
+class GitLabCICDGenerator(CICDGeneratorBase):
+    cicd_files = [
+        CICDFile(
+            template=GITLAB_YML,
+            file_name=".gitlab-ci.yml",
+            dir_path=".",
+        ),
+        CICDFile(
+            template=GITLAB_CI_YML,
+            file_name="tinybird-ci.yml",
+            dir_path=".gitlab/tinybird",
+            warning_message="Make sure to import the file in your .gitlab-ci.yml file, e.g., `include: '.gitlab/tinybird/*.yml'`.",
+        ),
+    ]
+
+
+async def init_cicd(
+    path: Optional[str] = None,
+    data_project_dir: Optional[str] = None,
+):
+    for provider in Provider:
+        path = path if path else getcwd()
+        data_project_dir = data_project_dir if data_project_dir else "."
+        generator = CICDGeneratorBase.build_generator(provider.name)
+        params = {
+            "data_project_dir": data_project_dir,
+            "workflow_version": WORKFLOW_VERSION,
+        }
+        warning_message = generator(path, params)
+        if warning_message:
+            click.echo(warning_message)
+
+
+async def check_cicd_exists(path: Optional[str] = None) -> Optional[Provider]:
+    path = path if path else getcwd()
+    for provider in Provider:
+        generator = CICDGeneratorBase.build_generator(provider.name)
+        if generator.is_already_generated(path):
+            return provider
+    return None

tinybird/{tb_cli_modules → tb/modules}/cli.py

@@ -10,14 +10,12 @@ import pprint
 import re
 import shutil
 import sys
-from datetime import datetime
 from os import environ, getcwd
 from pathlib import Path
 from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
 
 import click
 import humanfriendly
-import requests
 from click import Context
 from packaging import version
 
@@ -30,7 +28,28 @@ from tinybird.client import (
     TinyB,
 )
 from tinybird.config import CURRENT_VERSION, SUPPORTED_CONNECTORS, VERSION, FeatureFlags, get_config
-from tinybird.datafile import (
+from tinybird.feedback_manager import FeedbackManager
+from tinybird.tb.modules.cicd import check_cicd_exists, init_cicd
+from tinybird.tb.modules.common import (
+    OLDEST_ROLLBACK,
+    CatchAuthExceptions,
+    CLIException,
+    _get_tb_client,
+    coro,
+    create_tb_client,
+    echo_safe_format_table,
+    folder_init,
+    get_current_main_workspace,
+    getenv_bool,
+    is_major_semver,
+    is_post_semver,
+    load_connector_config,
+    remove_release,
+    try_update_config_with_remote,
+    wait_job,
+)
+from tinybird.tb.modules.config import CLIConfig
+from tinybird.tb.modules.datafile import (
     AlreadyExistsException,
     CLIGitRelease,
     CLIGitReleaseException,
@@ -47,30 +66,8 @@ from tinybird.datafile import (
     parse_datasource,
     parse_pipe,
     parse_token,
-    wait_job,
 )
-from tinybird.feedback_manager import FeedbackManager
-from tinybird.tb_cli_modules.cicd import check_cicd_exists, init_cicd
-from tinybird.tb_cli_modules.common import (
-    OLDEST_ROLLBACK,
-    CatchAuthExceptions,
-    CLIException,
-    _get_tb_client,
-    coro,
-    create_tb_client,
-    echo_safe_humanfriendly_tables_format_smart_table,
-    folder_init,
-    get_current_main_workspace,
-    getenv_bool,
-    is_major_semver,
-    is_post_semver,
-    load_connector_config,
-    remove_release,
-    try_update_config_with_remote,
-)
-from tinybird.tb_cli_modules.config import CLIConfig
-from tinybird.tb_cli_modules.prompts import sample_data_prompt
-from tinybird.tb_cli_modules.telemetry import add_telemetry_event
+from tinybird.tb.modules.telemetry import add_telemetry_event
 
 __old_click_echo = click.echo
 __old_click_secho = click.secho
@@ -927,7 +924,7 @@ async def sql(
             dd = []
             for d in res["data"]:
                 dd.append(d.values())
-            echo_safe_humanfriendly_tables_format_smart_table(dd, column_names=res["data"][0].keys())
+            echo_safe_format_table(dd, columns=res["meta"])
     else:
         click.echo(FeedbackManager.info_no_rows())
 
@@ -1572,67 +1569,3 @@ async def deploy(
         raise
     except Exception as e:
         raise CLIException(str(e))
-
-
-@cli.command()
-@click.argument("datasource_file", type=click.Path(exists=True))
-@click.option("--count", type=int, default=10, help="Number of events to send")
-@click.option("--model", type=str, default=None, help="Model to use for data generation")
-@click.option("--print-data", is_flag=True, default=False, help="Print the data being sent")
-@click.pass_context
-def load_sample_data(ctx: Context, datasource_file: str, count: int, model: Optional[str], print_data: bool) -> None:
-    """Load sample data into a datasource.
-
-    Args:
-        ctx: Click context object
-        datasource_file: Path to the datasource file to load sample data into
-    """
-    import llm
-
-    try:
-        # TODO(eclbg): allow passing a datasource name instead of a file
-        datasource_path = Path(datasource_file)
-        if datasource_path.suffix != ".datasource":
-            raise CLIException(FeedbackManager.error_file_extension(filename=datasource_file))
-
-        datasource_name = datasource_path.stem
-
-        response = requests.get("http://localhost:80/tokens")
-        token = response.json()["workspace_admin_token"]
-
-        with open(datasource_file) as f:
-            content = f.read()
-            schema_start = next(i for i, line in enumerate(content.splitlines()) if line.strip().startswith("SCHEMA >"))
-            schema_end = next(
-                i
-                for i, line in enumerate(content.splitlines()[schema_start + 1 :], schema_start + 1)
-                if not line.strip()
-            )
-            schema = "\n".join(content.splitlines()[schema_start:schema_end])
-        llm_model = llm.get_model(model)
-        click.echo(f"Using model: {model}")
-        prompt = sample_data_prompt.format(current_datetime=datetime.now().isoformat(), row_count=count)
-        # prompt = sample_data_with_errors_prompt.format(current_datetime=datetime.now().isoformat())  # This prompt will generate data with errors
-        full_prompt = prompt + "\n\n" + schema
-        sent_events = 0
-        while sent_events < count:
-            click.echo(f"Generating data for '{datasource_name}'")
-            data = llm_model.prompt(full_prompt)
-
-            click.echo(f"Sending data to '{datasource_name}'")
-            headers = {"Authorization": f"Bearer {token}"}
-            if print_data:
-                click.echo(f"Data: {data}")
-            response = requests.post(
-                f"http://localhost:80/v0/events?name={datasource_name}",
-                data=data,
-                headers=headers,
-            )
-            if response.status_code not in (200, 202):
-                raise CLIException(f"Failed to send data: {response.text}")
-            click.echo(f"Response: {response.text}")
-            sent_events += 10
-            click.echo(f"Sent 10 events to datasource '{datasource_name}'")
-
-    except Exception as e:
-        raise CLIException(FeedbackManager.error_exception(error=str(e)))