tinybird 0.0.1.dev5__py3-none-any.whl → 0.0.1.dev7__py3-none-any.whl

tinybird/__cli__.py

@@ -1,8 +1,7 @@
-
-__name__ = 'tinybird'
-__description__ = 'Tinybird Command Line Tool'
-__url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
-__author__ = 'Tinybird'
-__author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev5'
-__revision__ = 'b69dd20'
+__name__ = "tinybird-cli"
+__description__ = "Tinybird Command Line Tool"
+__url__ = "https://www.tinybird.co/docs/cli/introduction.html"
+__author__ = "Tinybird"
+__author_email__ = "support@tinybird.co"
+__version__ = "x.y.z"
+__revision__ = "xxxxxxxxx"

tinybird/tb/cli.py

@@ -0,0 +1,28 @@
+import asyncio
+import sys
+
+if sys.platform == "win32":
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+
+import tinybird.tb.modules.auth
+import tinybird.tb.modules.branch
+import tinybird.tb.modules.build
+import tinybird.tb.modules.cli
+import tinybird.tb.modules.common
+import tinybird.tb.modules.connection
+import tinybird.tb.modules.create
+import tinybird.tb.modules.datasource
+import tinybird.tb.modules.fmt
+import tinybird.tb.modules.job
+import tinybird.tb.modules.mock
+import tinybird.tb.modules.pipe
+import tinybird.tb.modules.tag
+import tinybird.tb.modules.test
+import tinybird.tb.modules.token
+import tinybird.tb.modules.workspace
+import tinybird.tb.modules.workspace_members
+
+cli = tinybird.tb.modules.cli.cli
+
+if __name__ == "__main__":
+    cli()

tinybird/{tb_cli_modules → tb/modules}/auth.py

@@ -11,8 +11,8 @@ import humanfriendly.tables
 
 from tinybird.config import get_display_host
 from tinybird.feedback_manager import FeedbackManager
-from tinybird.tb_cli_modules.cli import cli
-from tinybird.tb_cli_modules.common import (
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.common import (
     configure_connector,
     coro,
     echo_safe_humanfriendly_tables_format_smart_table,
@@ -21,9 +21,9 @@ from tinybird.tb_cli_modules.common import (
     try_authenticate,
     try_update_config_with_remote,
 )
-from tinybird.tb_cli_modules.config import CLIConfig, ConfigValueOrigin
-from tinybird.tb_cli_modules.exceptions import CLIAuthException
-from tinybird.tb_cli_modules.regions import Region
+from tinybird.tb.modules.config import CLIConfig, ConfigValueOrigin
+from tinybird.tb.modules.exceptions import CLIAuthException
+from tinybird.tb.modules.regions import Region
 
 
 @cli.group(invoke_without_command=True)

tinybird/{tb_cli_modules → tb/modules}/branch.py

@@ -4,17 +4,15 @@
 # - But please, **do not** interleave utility functions and command definitions.
 
 import os
-from os import getcwd
 from typing import List, Optional, Tuple
 
 import aiofiles
 import click
 import yaml
 
-from tinybird.datafile import create_release, wait_job
 from tinybird.feedback_manager import FeedbackManager
-from tinybird.tb_cli_modules.cli import cli
-from tinybird.tb_cli_modules.common import (
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.common import (
     MAIN_BRANCH,
     OLDEST_ROLLBACK,
     coro,
@@ -35,9 +33,10 @@ from tinybird.tb_cli_modules.common import (
     switch_to_workspace_by_user_workspace_data,
     switch_workspace,
     try_update_config_with_remote,
+    wait_job,
 )
-from tinybird.tb_cli_modules.config import CLIConfig
-from tinybird.tb_cli_modules.exceptions import CLIBranchException, CLIException, CLIReleaseException
+from tinybird.tb.modules.config import CLIConfig
+from tinybird.tb.modules.exceptions import CLIBranchException, CLIException, CLIReleaseException
 
 
 @cli.group(hidden=True)
@@ -129,25 +128,6 @@ set -euxo pipefail
     click.echo(FeedbackManager.info_release_generated(semver=semver))
 
 
-@release.command(name="create", short_help="Create a new Release in deploying status")
-@click.option(
-    "--semver",
-    is_flag=False,
-    required=True,
-    type=str,
-    help="Semver of the new Release. Example: 1.0.0",
-)
-@coro
-async def release_create(semver: str) -> None:
-    click.echo(FeedbackManager.warning_deprecated_releases())
-    config = CLIConfig.get_project_config()
-    _ = await try_update_config_with_remote(config, only_if_needed=True)
-
-    client = config.get_client()
-    folder = getcwd()
-    await create_release(client, config, semver, folder)
-
-
 @release.command(name="promote", short_help="Promotes to live status a preview Release")
 @click.option(
     "--semver", required=True, type=str, help="Semver of a preview Release to promote to live. Example: 1.0.0"

tinybird/{tb_cli_modules → tb/modules}/build.py

@@ -3,7 +3,7 @@ import os
 import random
 import time
 from pathlib import Path
-from typing import Any, Awaitable, Callable, Dict, List, Union
+from typing import Any, Awaitable, Callable, List, Union
 
 import click
 import humanfriendly
@@ -13,23 +13,18 @@ from watchdog.observers import Observer
 import tinybird.context as context
 from tinybird.client import TinyB
 from tinybird.config import FeatureFlags
-from tinybird.datafile import (
-    ParseException,
-    folder_build,
-    get_project_filenames,
-    has_internal_datafiles,
-    parse_datasource,
-    parse_pipe,
-)
 from tinybird.feedback_manager import FeedbackManager, bcolors
-from tinybird.tb_cli_modules.cli import cli
-from tinybird.tb_cli_modules.common import (
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.common import (
     coro,
 )
-from tinybird.tb_cli_modules.local import (
-    get_tinybird_local_client,
-)
-from tinybird.tb_cli_modules.table import format_table
+from tinybird.tb.modules.datafile.build import folder_build
+from tinybird.tb.modules.datafile.common import get_project_filenames, has_internal_datafiles
+from tinybird.tb.modules.datafile.exceptions import ParseException
+from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
+from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
+from tinybird.tb.modules.local import get_tinybird_local_client
+from tinybird.tb.modules.table import format_table
 
 
 class FileChangeHandler(FileSystemEventHandler):
@@ -116,9 +111,6 @@ async def build(
     context.disable_template_security_validation.set(True)
     is_internal = has_internal_datafiles(folder)
     tb_client = get_tinybird_local_client()
-    workspaces: List[Dict[str, Any]] = (await tb_client.user_workspaces_and_branches()).get("workspaces", [])
-    datasources: List[Dict[str, Any]] = await tb_client.datasources()
-    pipes: List[Dict[str, Any]] = await tb_client.pipes(dependencies=True)
 
     def check_filenames(filenames: List[str]):
         parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
@@ -142,9 +134,6 @@ async def build(
         check_filenames(filenames=filenames)
         await folder_build(
             tb_client,
-            workspaces,
-            datasources,
-            pipes,
             filenames,
             ignore_sql_errors=ignore_sql_errors,
             is_internal=is_internal,

tinybird/tb/modules/cicd.py

@@ -0,0 +1,271 @@
+from enum import Enum
+from os import getcwd
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Type, Union
+
+import click
+from tornado.template import Template
+
+from tinybird.feedback_manager import FeedbackManager
+
+
+class Provider(Enum):
+    GitHub = 0
+    GitLab = 1
+
+
+WORKFLOW_VERSION = "v3.1.0"
+
+DEFAULT_REQUIREMENTS_FILE = "tinybird-cli>=5,<6"
+
+GITHUB_CI_YML = """
+name: Tinybird - CI Workflow
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+      - master
+    types: [opened, reopened, labeled, unlabeled, synchronize, closed]{% if data_project_dir != '.' %}
+    paths:
+      - '{{ data_project_dir }}/**'{% end %}
+
+concurrency: ${{! github.workflow }}-${{! github.event.pull_request.number }}
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: '{{ data_project_dir }}'
+    services:
+      tinybird:
+        image: tinybirdco/tinybird-local:latest
+        ports:
+          - 80:80
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Tinybird CLI
+        run: curl -LsSf https://api.tinybird.co/static/install.sh | sh
+      - name: Build project
+        run: tb build
+"""
+
+
+GITLAB_YML = """
+include:
+  - local: .gitlab/tinybird/*.yml
+
+stages:
+  - tests
+"""
+
+
+GITLAB_CI_YML = """
+tinybird_ci_workflow:
+  stage: tests
+  interruptible: true
+  needs: []
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"{% if data_project_dir != '.' %}
+      changes:
+        - .gitlab/tinybird/*
+        - {{ data_project_dir }}/*
+        - {{ data_project_dir }}/**/*{% end %}
+  before_script:
+    - curl -LsSf https://api.tinybird.co/static/install.sh | sh
+  script:
+    - cd $CI_PROJECT_DIR/{{ data_project_dir }}
+    - tb build
+  services:
+    - name: tinybirdco/tinybird-local:latest
+      alias: tinybird-local
+"""
+
+
+EXEC_TEST_SH = """
+#!/usr/bin/env bash
+set -euxo pipefail
+
+export TB_VERSION_WARNING=0
+
+run_test() {
+    t=$1
+    echo "** Running $t **"
+    echo "** $(cat $t)"
+    tmpfile=$(mktemp)
+    retries=0
+    TOTAL_RETRIES=3
+
+    # When appending fixtures, we need to retry in case of the data is not replicated in time
+    while [ $retries -lt $TOTAL_RETRIES ]; do
+        # Run the test and store the output in a temporary file
+        bash $t $2 >$tmpfile
+        exit_code=$?
+        if [ "$exit_code" -eq 0 ]; then
+            # If the test passed, break the loop
+            if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
+                break
+            # If the test failed, increment the retries counter and try again
+            else
+                retries=$((retries+1))
+            fi
+        # If the bash command failed, print an error message and break the loop
+        else
+            break
+        fi
+    done
+
+    if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
+        echo "✅ Test $t passed"
+        rm $tmpfile
+        return 0
+    elif [ $retries -eq $TOTAL_RETRIES ]; then
+        echo "🚨 ERROR: Test $t failed, diff:";
+        diff -B ${t}.result $tmpfile
+        rm $tmpfile
+        return 1
+    else
+        echo "🚨 ERROR: Test $t failed with bash command exit code $?"
+        cat $tmpfile
+        rm $tmpfile
+        return 1
+    fi
+    echo ""
+}
+export -f run_test
+
+fail=0
+find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} || fail=1
+
+if [ $fail == 1 ]; then
+  exit -1;
+fi
+"""
+
+APPEND_FIXTURES_SH = """
+#!/usr/bin/env bash
+set -euxo pipefail
+
+directory="datasources/fixtures"
+extensions=("csv" "ndjson")
+
+absolute_directory=$(realpath "$directory")
+
+for extension in "${extensions[@]}"; do
+  file_list=$(find "$absolute_directory" -type f -name "*.$extension")
+
+  for file_path in $file_list; do
+    file_name=$(basename "$file_path")
+    file_name_without_extension="${file_name%.*}"
+
+    command="tb datasource append $file_name_without_extension datasources/fixtures/$file_name"
+    echo $command
+    $command
+  done
+done
+"""
+
+
+class CICDFile:
+    def __init__(
+        self,
+        template: str,
+        file_name: str,
+        dir_path: Optional[str] = None,
+        warning_message: Optional[str] = None,
+    ):
+        self.template = template
+        self.file_name = file_name
+        self.dir_path = dir_path
+        self.warning_message = warning_message
+
+    @property
+    def full_path(self) -> str:
+        return f"{self.dir_path}/{self.file_name}" if self.dir_path else self.file_name
+
+
+class CICDGeneratorBase:
+    cicd_files: List[CICDFile] = []
+
+    def __call__(self, path: str, params: Dict[str, Any]):
+        for cicd_file in self.cicd_files:
+            if cicd_file.dir_path:
+                Path(f"{path}/{cicd_file.dir_path}").mkdir(parents=True, exist_ok=True)
+            content = Template(cicd_file.template).generate(**params)
+            if Path(f"{path}/{cicd_file.full_path}").exists():
+                continue
+            with open(f"{path}/{cicd_file.full_path}", "wb") as f:
+                f.write(content)
+            click.echo(FeedbackManager.info_cicd_file_generated(file_path=cicd_file.full_path))
+            if cicd_file.warning_message is not None:
+                return FeedbackManager.warning_for_cicd_file(
+                    file_name=cicd_file.file_name, warning_message=cicd_file.warning_message.format(**params)
+                )
+
+    def is_already_generated(self, path: str) -> bool:
+        for cicd_file in self.cicd_files:
+            if cicd_file.file_name and Path(f"{path}/{cicd_file.full_path}").exists():
+                return True
+        return False
+
+    @classmethod
+    def build_generator(cls, provider: str) -> Union["GitHubCICDGenerator", "GitLabCICDGenerator"]:
+        builder: Dict[str, Union[Type[GitHubCICDGenerator], Type[GitLabCICDGenerator]]] = {
+            Provider.GitHub.name: GitHubCICDGenerator,
+            Provider.GitLab.name: GitLabCICDGenerator,
+        }
+        return builder[provider]()
+
+
+class GitHubCICDGenerator(CICDGeneratorBase):
+    cicd_files = [
+        CICDFile(
+            template=GITHUB_CI_YML,
+            file_name="tinybird-ci.yml",
+            dir_path=".github/workflows",
+        ),
+    ]
+
+
+class GitLabCICDGenerator(CICDGeneratorBase):
+    cicd_files = [
+        CICDFile(
+            template=GITLAB_YML,
+            file_name=".gitlab-ci.yml",
+            dir_path=".",
+        ),
+        CICDFile(
+            template=GITLAB_CI_YML,
+            file_name="tinybird-ci.yml",
+            dir_path=".gitlab/tinybird",
+            warning_message="Make sure to import the file in your .gitlab-ci.yml file, e.g., `include: '.gitlab/tinybird/*.yml'`.",
+        ),
+    ]
+
+
+async def init_cicd(
+    path: Optional[str] = None,
+    data_project_dir: Optional[str] = None,
+):
+    for provider in Provider:
+        path = path if path else getcwd()
+        data_project_dir = data_project_dir if data_project_dir else "."
+        generator = CICDGeneratorBase.build_generator(provider.name)
+        params = {
+            "data_project_dir": data_project_dir,
+            "workflow_version": WORKFLOW_VERSION,
+        }
+        warning_message = generator(path, params)
+        if warning_message:
+            click.echo(warning_message)
+
+
+async def check_cicd_exists(path: Optional[str] = None) -> Optional[Provider]:
+    path = path if path else getcwd()
+    for provider in Provider:
+        generator = CICDGeneratorBase.build_generator(provider.name)
+        if generator.is_already_generated(path):
+            return provider
+    return None

tinybird/{tb_cli_modules → tb/modules}/cli.py

@@ -24,32 +24,11 @@ from tinybird.client import (
     AuthException,
     AuthNoTokenException,
     DoesNotExistException,
-    OperationCanNotBePerformed,
     TinyB,
 )
 from tinybird.config import CURRENT_VERSION, SUPPORTED_CONNECTORS, VERSION, FeatureFlags, get_config
-from tinybird.datafile import (
-    AlreadyExistsException,
-    CLIGitRelease,
-    CLIGitReleaseException,
-    Datafile,
-    ParseException,
-    build_graph,
-    create_release,
-    diff_command,
-    folder_pull,
-    folder_push,
-    get_project_filenames,
-    get_resource_versions,
-    has_internal_datafiles,
-    parse_datasource,
-    parse_pipe,
-    parse_token,
-    wait_job,
-)
 from tinybird.feedback_manager import FeedbackManager
-from tinybird.tb_cli_modules.cicd import check_cicd_exists, init_cicd
-from tinybird.tb_cli_modules.common import (
+from tinybird.tb.modules.common import (
     OLDEST_ROLLBACK,
     CatchAuthExceptions,
     CLIException,
@@ -65,9 +44,25 @@ from tinybird.tb_cli_modules.common import (
     load_connector_config,
     remove_release,
     try_update_config_with_remote,
+    wait_job,
+)
+from tinybird.tb.modules.config import CLIConfig
+from tinybird.tb.modules.datafile.build import build_graph, folder_push
+from tinybird.tb.modules.datafile.common import (
+    Datafile,
+    get_project_filenames,
+    get_resource_versions,
+    has_internal_datafiles,
+)
+from tinybird.tb.modules.datafile.diff import diff_command
+from tinybird.tb.modules.datafile.exceptions import (
+    AlreadyExistsException,
+    ParseException,
 )
-from tinybird.tb_cli_modules.config import CLIConfig
-from tinybird.tb_cli_modules.telemetry import add_telemetry_event
+from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
+from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
+from tinybird.tb.modules.datafile.pull import folder_pull
+from tinybird.tb.modules.telemetry import add_telemetry_event
 
 __old_click_echo = click.echo
 __old_click_secho = click.secho
@@ -323,118 +318,8 @@ async def init(
 
     await folder_init(client, folder, generate_datasources, generate_releases=True, force=force)
 
-    if not (git or override_commit or cicd):
-        return
-
-    sync_git = git or override_commit
-    cli_git_release = None
     error = False
     final_response = None
-    try:
-        cli_git_release = CLIGitRelease(path=folder)
-    except CLIGitReleaseException:
-        raise CLIGitReleaseException(FeedbackManager.error_no_git_repo_for_init(repo_path=folder))
-
-    if sync_git:
-        if not cli_git_release.is_main_branch() and not override_commit:
-            raise CLIGitReleaseException(FeedbackManager.error_no_git_main_branch())
-
-        if not cli_git_release.is_dottinyb_ignored():
-            raise CLIGitReleaseException(
-                FeedbackManager.error_dottinyb_not_ignored(git_working_dir=f"{cli_git_release.working_dir()}/")
-            )
-        else:
-            click.echo(FeedbackManager.info_dottinyb_already_ignored())
-
-        if (
-            os.path.exists(f"{cli_git_release.working_dir()}/.diff_tmp")
-            and not cli_git_release.is_dotdifftemp_ignored()
-        ):
-            raise CLIGitReleaseException(
-                FeedbackManager.error_dotdiff_not_ignored(git_working_dir=f"{cli_git_release.working_dir()}/")
-            )
-        else:
-            click.echo(FeedbackManager.info_dotdifftemp_already_ignored())
-
-        if "release" not in current_ws:
-            raise CLIGitReleaseException(FeedbackManager.error_no_correct_token_for_init())
-
-        # If we have a release and we are not overriding the commit, we check if we have a release already
-        elif current_ws.get("release") and not override_commit:
-            final_response = FeedbackManager.error_release_already_set(
-                workspace=current_ws["name"], commit=current_ws["release"]["commit"]
-            )
-            error = True
-
-        elif override_commit:
-            if click.confirm(
-                FeedbackManager.prompt_init_git_release_force(
-                    current_commit=current_ws["release"]["commit"], new_commit=override_commit
-                )
-            ):
-                try:
-                    release = await cli_git_release.update_release(client, current_ws, override_commit)
-                except Exception as exc:
-                    raise CLIGitReleaseException(FeedbackManager.error_exception(error=str(exc)))
-
-                final_response = FeedbackManager.success_init_git_release(
-                    workspace_name=current_ws["name"], release_commit=release["commit"]
-                )
-            else:
-                return
-
-        else:
-            click.echo(FeedbackManager.info_no_git_release_yet(workspace=current_ws["name"]))
-            click.echo(FeedbackManager.info_diff_resources_for_git_init())
-            changed = await diff_command(
-                [], True, client, with_print=False, verbose=False, clean_up=True, progress_bar=True
-            )
-            changed = {
-                k: v
-                for k, v in changed.items()
-                if v is not None and (not ignore_remote or v not in ["remote", "shared"])
-            }
-            if changed:
-                tb_pull_command = "tb pull --force"
-                click.echo(FeedbackManager.warning_git_release_init_with_diffs())
-                if click.confirm(FeedbackManager.prompt_init_git_release_pull(pull_command=tb_pull_command)):
-                    await folder_pull(client, folder, auto=True, match=None, force=True)
-
-                else:
-                    raise CLIGitReleaseException(
-                        FeedbackManager.error_diff_resources_for_git_init(
-                            workspace=current_ws["name"], pull_command=tb_pull_command
-                        )
-                    )
-            else:
-                click.echo(FeedbackManager.info_git_release_init_without_diffs(workspace=current_ws["name"]))
-            if cli_git_release.is_dirty_to_init():
-                raise CLIGitReleaseException(
-                    FeedbackManager.error_commit_changes_to_init_release(
-                        path=cli_git_release.path, git_output=cli_git_release.status()
-                    )
-                )
-            try:
-                release = await cli_git_release.update_release(client, current_ws)
-            except Exception as exc:
-                raise CLIGitReleaseException(FeedbackManager.error_exception(error=str(exc)))
-
-            final_response = FeedbackManager.success_init_git_release(
-                workspace_name=current_ws["name"], release_commit=release["commit"]
-            )
-
-    if not override_commit:
-        cicd_provider = await check_cicd_exists(cli_git_release.working_dir())
-        if not force and cicd_provider:
-            if cicd:
-                final_response = FeedbackManager.error_cicd_already_exists(provider=cicd_provider.name)
-                error = True
-            else:
-                click.echo(FeedbackManager.info_cicd_already_exists(provider=cicd_provider.name))
-        else:
-            if cicd:
-                data_project_dir = os.path.relpath(folder, cli_git_release.working_dir())
-                await init_cicd(client, path=cli_git_release.working_dir(), data_project_dir=data_project_dir)
 
     if final_response:
         if error:
@@ -452,7 +337,7 @@ def check(filenames: List[str], debug: bool) -> None:
         filenames = get_project_filenames(".")
 
     def process(filenames: Iterable):
-        parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource, ".token": parse_token}
+        parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
         incl_suffix = ".incl"
         try:
             for filename in filenames:
@@ -1473,11 +1358,6 @@ async def deploy(
                 new_release = True
 
         if new_release:
-            if not dry_run:
-                try:
-                    await create_release(client, config, semver)
-                except OperationCanNotBePerformed as e:
-                    raise CLIException(FeedbackManager.error_exception(error=str(e)))
             release_created = True
             fork_downstream = True
             # allows TB_CHECK_BACKFILL_REQUIRED=0 so it is not checked