tinybird 0.0.1.dev6__py3-none-any.whl → 0.0.1.dev7__py3-none-any.whl

tinybird/tb/modules/branch.py

@@ -4,7 +4,6 @@
 # - But please, **do not** interleave utility functions and command definitions.
 
 import os
-from os import getcwd
 from typing import List, Optional, Tuple
 
 import aiofiles
@@ -37,7 +36,6 @@ from tinybird.tb.modules.common import (
     wait_job,
 )
 from tinybird.tb.modules.config import CLIConfig
-from tinybird.tb.modules.datafile import create_release
 from tinybird.tb.modules.exceptions import CLIBranchException, CLIException, CLIReleaseException
 
 
@@ -130,25 +128,6 @@ set -euxo pipefail
     click.echo(FeedbackManager.info_release_generated(semver=semver))
 
 
-@release.command(name="create", short_help="Create a new Release in deploying status")
-@click.option(
-    "--semver",
-    is_flag=False,
-    required=True,
-    type=str,
-    help="Semver of the new Release. Example: 1.0.0",
-)
-@coro
-async def release_create(semver: str) -> None:
-    click.echo(FeedbackManager.warning_deprecated_releases())
-    config = CLIConfig.get_project_config()
-    _ = await try_update_config_with_remote(config, only_if_needed=True)
-
-    client = config.get_client()
-    folder = getcwd()
-    await create_release(client, config, semver, folder)
-
-
 @release.command(name="promote", short_help="Promotes to live status a preview Release")
 @click.option(
     "--semver", required=True, type=str, help="Semver of a preview Release to promote to live. Example: 1.0.0"

tinybird/tb/modules/build.py

@@ -3,7 +3,7 @@ import os
 import random
 import time
 from pathlib import Path
-from typing import Any, Awaitable, Callable, Dict, List, Union
+from typing import Any, Awaitable, Callable, List, Union
 
 import click
 import humanfriendly
@@ -18,17 +18,12 @@ from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import (
     coro,
 )
-from tinybird.tb.modules.datafile import (
-    ParseException,
-    folder_build,
-    get_project_filenames,
-    has_internal_datafiles,
-    parse_datasource,
-    parse_pipe,
-)
-from tinybird.tb.modules.local import (
-    get_tinybird_local_client,
-)
+from tinybird.tb.modules.datafile.build import folder_build
+from tinybird.tb.modules.datafile.common import get_project_filenames, has_internal_datafiles
+from tinybird.tb.modules.datafile.exceptions import ParseException
+from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
+from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
+from tinybird.tb.modules.local import get_tinybird_local_client
 from tinybird.tb.modules.table import format_table
 
 
@@ -116,9 +111,6 @@ async def build(
     context.disable_template_security_validation.set(True)
     is_internal = has_internal_datafiles(folder)
     tb_client = get_tinybird_local_client()
-    workspaces: List[Dict[str, Any]] = (await tb_client.user_workspaces_and_branches()).get("workspaces", [])
-    datasources: List[Dict[str, Any]] = await tb_client.datasources()
-    pipes: List[Dict[str, Any]] = await tb_client.pipes(dependencies=True)
 
     def check_filenames(filenames: List[str]):
         parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
@@ -142,9 +134,6 @@ async def build(
         check_filenames(filenames=filenames)
         await folder_build(
             tb_client,
-            workspaces,
-            datasources,
-            pipes,
             filenames,
             ignore_sql_errors=ignore_sql_errors,
             is_internal=is_internal,

tinybird/tb/modules/cli.py

@@ -24,12 +24,10 @@ from tinybird.client import (
     AuthException,
     AuthNoTokenException,
     DoesNotExistException,
-    OperationCanNotBePerformed,
     TinyB,
 )
 from tinybird.config import CURRENT_VERSION, SUPPORTED_CONNECTORS, VERSION, FeatureFlags, get_config
 from tinybird.feedback_manager import FeedbackManager
-from tinybird.tb.modules.cicd import check_cicd_exists, init_cicd
 from tinybird.tb.modules.common import (
     OLDEST_ROLLBACK,
     CatchAuthExceptions,
@@ -49,24 +47,21 @@ from tinybird.tb.modules.common import (
     wait_job,
 )
 from tinybird.tb.modules.config import CLIConfig
-from tinybird.tb.modules.datafile import (
-    AlreadyExistsException,
-    CLIGitRelease,
-    CLIGitReleaseException,
+from tinybird.tb.modules.datafile.build import build_graph, folder_push
+from tinybird.tb.modules.datafile.common import (
     Datafile,
-    ParseException,
-    build_graph,
-    create_release,
-    diff_command,
-    folder_pull,
-    folder_push,
     get_project_filenames,
     get_resource_versions,
     has_internal_datafiles,
-    parse_datasource,
-    parse_pipe,
-    parse_token,
 )
+from tinybird.tb.modules.datafile.diff import diff_command
+from tinybird.tb.modules.datafile.exceptions import (
+    AlreadyExistsException,
+    ParseException,
+)
+from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
+from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
+from tinybird.tb.modules.datafile.pull import folder_pull
 from tinybird.tb.modules.telemetry import add_telemetry_event
 
 __old_click_echo = click.echo
@@ -323,118 +318,8 @@ async def init(
 
     await folder_init(client, folder, generate_datasources, generate_releases=True, force=force)
 
-    if not (git or override_commit or cicd):
-        return
-
-    sync_git = git or override_commit
-    cli_git_release = None
     error = False
     final_response = None
-    try:
-        cli_git_release = CLIGitRelease(path=folder)
-    except CLIGitReleaseException:
-        raise CLIGitReleaseException(FeedbackManager.error_no_git_repo_for_init(repo_path=folder))
-
-    if sync_git:
-        if not cli_git_release.is_main_branch() and not override_commit:
-            raise CLIGitReleaseException(FeedbackManager.error_no_git_main_branch())
-
-        if not cli_git_release.is_dottinyb_ignored():
-            raise CLIGitReleaseException(
-                FeedbackManager.error_dottinyb_not_ignored(git_working_dir=f"{cli_git_release.working_dir()}/")
-            )
-        else:
-            click.echo(FeedbackManager.info_dottinyb_already_ignored())
-
-        if (
-            os.path.exists(f"{cli_git_release.working_dir()}/.diff_tmp")
-            and not cli_git_release.is_dotdifftemp_ignored()
-        ):
-            raise CLIGitReleaseException(
-                FeedbackManager.error_dotdiff_not_ignored(git_working_dir=f"{cli_git_release.working_dir()}/")
-            )
-        else:
-            click.echo(FeedbackManager.info_dotdifftemp_already_ignored())
-
-        if "release" not in current_ws:
-            raise CLIGitReleaseException(FeedbackManager.error_no_correct_token_for_init())
-
-        # If we have a release and we are not overriding the commit, we check if we have a release already
-        elif current_ws.get("release") and not override_commit:
-            final_response = FeedbackManager.error_release_already_set(
-                workspace=current_ws["name"], commit=current_ws["release"]["commit"]
-            )
-            error = True
-
-        elif override_commit:
-            if click.confirm(
-                FeedbackManager.prompt_init_git_release_force(
-                    current_commit=current_ws["release"]["commit"], new_commit=override_commit
-                )
-            ):
-                try:
-                    release = await cli_git_release.update_release(client, current_ws, override_commit)
-                except Exception as exc:
-                    raise CLIGitReleaseException(FeedbackManager.error_exception(error=str(exc)))
-
-                final_response = FeedbackManager.success_init_git_release(
-                    workspace_name=current_ws["name"], release_commit=release["commit"]
-                )
-            else:
-                return
-
-        else:
-            click.echo(FeedbackManager.info_no_git_release_yet(workspace=current_ws["name"]))
-            click.echo(FeedbackManager.info_diff_resources_for_git_init())
-            changed = await diff_command(
-                [], True, client, with_print=False, verbose=False, clean_up=True, progress_bar=True
-            )
-            changed = {
-                k: v
-                for k, v in changed.items()
-                if v is not None and (not ignore_remote or v not in ["remote", "shared"])
-            }
-            if changed:
-                tb_pull_command = "tb pull --force"
-                click.echo(FeedbackManager.warning_git_release_init_with_diffs())
-                if click.confirm(FeedbackManager.prompt_init_git_release_pull(pull_command=tb_pull_command)):
-                    await folder_pull(client, folder, auto=True, match=None, force=True)
-
-                else:
-                    raise CLIGitReleaseException(
-                        FeedbackManager.error_diff_resources_for_git_init(
-                            workspace=current_ws["name"], pull_command=tb_pull_command
-                        )
-                    )
-            else:
-                click.echo(FeedbackManager.info_git_release_init_without_diffs(workspace=current_ws["name"]))
-            if cli_git_release.is_dirty_to_init():
-                raise CLIGitReleaseException(
-                    FeedbackManager.error_commit_changes_to_init_release(
-                        path=cli_git_release.path, git_output=cli_git_release.status()
-                    )
-                )
-            try:
-                release = await cli_git_release.update_release(client, current_ws)
-            except Exception as exc:
-                raise CLIGitReleaseException(FeedbackManager.error_exception(error=str(exc)))
-
-            final_response = FeedbackManager.success_init_git_release(
-                workspace_name=current_ws["name"], release_commit=release["commit"]
-            )
-
-    if not override_commit:
-        cicd_provider = await check_cicd_exists(cli_git_release.working_dir())
-        if not force and cicd_provider:
-            if cicd:
-                final_response = FeedbackManager.error_cicd_already_exists(provider=cicd_provider.name)
-                error = True
-            else:
-                click.echo(FeedbackManager.info_cicd_already_exists(provider=cicd_provider.name))
-        else:
-            if cicd:
-                data_project_dir = os.path.relpath(folder, cli_git_release.working_dir())
-                await init_cicd(client, path=cli_git_release.working_dir(), data_project_dir=data_project_dir)
 
     if final_response:
         if error:
@@ -452,7 +337,7 @@ def check(filenames: List[str], debug: bool) -> None:
         filenames = get_project_filenames(".")
 
     def process(filenames: Iterable):
-        parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource, ".token": parse_token}
+        parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
         incl_suffix = ".incl"
         try:
             for filename in filenames:
@@ -1473,11 +1358,6 @@ async def deploy(
                 new_release = True
 
         if new_release:
-            if not dry_run:
-                try:
-                    await create_release(client, config, semver)
-                except OperationCanNotBePerformed as e:
-                    raise CLIException(FeedbackManager.error_exception(error=str(e)))
             release_created = True
             fork_downstream = True
             # allows TB_CHECK_BACKFILL_REQUIRED=0 so it is not checked

tinybird/tb/modules/common.py

@@ -27,6 +27,7 @@ import click.formatting
 import humanfriendly
 import humanfriendly.tables
 import pyperclip
+import requests
 from click import Context
 from click._termui_impl import ProgressBar
 from humanfriendly.tables import format_pretty_table
@@ -60,7 +61,7 @@ if TYPE_CHECKING:
 
 from tinybird.feedback_manager import FeedbackManager, warning_message
 from tinybird.git_settings import DEFAULT_TINYENV_FILE
-from tinybird.syncasync import async_to_sync
+from tinybird.syncasync import async_to_sync, sync_to_async
 from tinybird.tb.modules.cicd import APPEND_FIXTURES_SH, DEFAULT_REQUIREMENTS_FILE, EXEC_TEST_SH
 from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.exceptions import (
@@ -160,10 +161,17 @@ def normalize_datasource_name(s: str) -> str:
 
 
 def generate_datafile(
-    datafile: str, filename: str, data: Optional[bytes], force: Optional[bool] = False, _format: Optional[str] = "csv"
+    datafile: str,
+    filename: str,
+    data: Optional[bytes],
+    force: Optional[bool] = False,
+    _format: Optional[str] = "csv",
+    parent_dir: Optional[str] = None,
 ):
     p = Path(filename)
     base = Path("datasources")
+    if parent_dir:
+        base = Path(parent_dir) / base
     datasource_name = normalize_datasource_name(p.stem)
     if not base.exists():
         base = Path()
@@ -2096,3 +2104,7 @@ def get_ca_pem_content(ca_pem: Optional[str], filename: Optional[str] = None) ->
         raise CLIConnectionException(FeedbackManager.error_connection_invalid_ca_pem())
 
     return ca_pem_content
+
+
+requests_get = sync_to_async(requests.get, thread_sensitive=False)
+requests_delete = sync_to_async(requests.delete, thread_sensitive=False)

tinybird/tb/modules/create.py

@@ -1,7 +1,7 @@
 import os
 from os import getcwd
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Optional
 
 import click
 from click import Context
@@ -12,7 +12,7 @@ from tinybird.tb.modules.cicd import init_cicd
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import _generate_datafile, coro, generate_datafile, push_data
 from tinybird.tb.modules.config import CLIConfig
-from tinybird.tb.modules.datafile import folder_build
+from tinybird.tb.modules.datafile.build import folder_build
 from tinybird.tb.modules.exceptions import CLIDatasourceException
 from tinybird.tb.modules.llm import LLM
 from tinybird.tb.modules.local import (
@@ -55,15 +55,7 @@ async def create(
         click.echo(FeedbackManager.gray(message="Creating new project structure..."))
         await project_create(tb_client, data, prompt, folder)
         click.echo(FeedbackManager.success(message="✓ Scaffolding completed!\n"))
-        workspaces: List[Dict[str, Any]] = (await tb_client.user_workspaces()).get("workspaces", [])
-        datasources = await tb_client.datasources()
-        pipes = await tb_client.pipes(dependencies=True)
-        await folder_build(
-            tb_client,
-            workspaces,
-            datasources,
-            pipes,
-        )
+        await folder_build(tb_client, folder=folder)
 
         await init_cicd(data_project_dir=os.path.relpath(folder))
 
@@ -102,8 +94,10 @@ async def project_create(
         except FileExistsError:
             click.echo(FeedbackManager.info_path_created(path=x))
 
-    def generate_pipe_file(name: str, content: str):
+    def generate_pipe_file(name: str, content: str, parent_dir: Optional[str] = None):
         base = Path("endpoints")
+        if parent_dir:
+            base = Path(parent_dir) / base
         if not base.exists():
             base = Path()
         f = base / (f"{name}.pipe")
@@ -168,8 +162,10 @@ SQL >
     LIMIT 5
 TYPE ENDPOINT
 """
-        generate_datafile(events_ds, filename="events.datasource", data=None, _format="ndjson", force=force)
-        generate_pipe_file("top_airlines", top_airlines)
+        generate_datafile(
+            events_ds, filename="events.datasource", data=None, _format="ndjson", force=force, parent_dir=folder
+        )
+        generate_pipe_file("top_airlines", top_airlines, parent_dir=folder)
 
 
 async def append_datasource(