tinybird 0.0.1.dev9__tar.gz → 0.0.1.dev10__tar.gz

{tinybird-0.0.1.dev9 → tinybird-0.0.1.dev10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird
-Version: 0.0.1.dev9
+Version: 0.0.1.dev10
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev9 → tinybird-0.0.1.dev10}/tinybird/feedback_manager.py

@@ -792,7 +792,7 @@ Ready? """
     info_diff_resources_for_git_init = info_message(
         "** Checking diffs between remote Workspace and local. Hint: use 'tb diff' to check if your Data Project and Workspace synced"
     )
-    info_cicd_file_generated = info_message("** File {file_path} generated for CI/CD")
+    info_cicd_file_generated = info_message("✓ {file_path}")
     info_available_git_providers = info_message("** List of available providers:")
     info_git_release_init_without_diffs = info_message("** No diffs detected for '{workspace}'")
     info_deployment_detecting_changes_header = info_message("\n** Detecting changes from last commit ...")

{tinybird-0.0.1.dev9 → tinybird-0.0.1.dev10}/tinybird/tb/modules/build.py

@@ -1,7 +1,9 @@
 import asyncio
+import cmd
 import os
 import random
 import subprocess
+import sys
 import threading
 import time
 from pathlib import Path
@@ -16,38 +18,80 @@ import tinybird.context as context
 from tinybird.client import TinyB
 from tinybird.config import FeatureFlags
 from tinybird.feedback_manager import FeedbackManager, bcolors
-from tinybird.syncasync import sync_to_async
 from tinybird.tb.modules.cli import cli
-from tinybird.tb.modules.common import (
-    coro,
-)
+from tinybird.tb.modules.common import coro, push_data
 from tinybird.tb.modules.datafile.build import folder_build
-from tinybird.tb.modules.datafile.common import get_project_filenames, has_internal_datafiles
+from tinybird.tb.modules.datafile.common import get_project_filenames, get_project_fixtures, has_internal_datafiles
 from tinybird.tb.modules.datafile.exceptions import ParseException
+from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
 from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
 from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 from tinybird.tb.modules.local import get_tinybird_local_client
 from tinybird.tb.modules.table import format_table
 
 
+class BuildShell(cmd.Cmd):
+    prompt = "\n\001\033[1;32m\002TB > \001\033[0m\002"
+
+    def __init__(self, folder: str):
+        super().__init__()
+        self.folder = folder
+
+    def do_exit(self, arg):
+        sys.exit(0)
+
+    def do_quit(self, arg):
+        sys.exit(0)
+
+    def default(self, argline):
+        click.echo("")
+        if argline.startswith("tb build"):
+            click.echo(FeedbackManager.error(message="Build command is already running"))
+        else:
+            arg_stripped = argline.strip().lower()
+            if not arg_stripped:
+                return
+            if arg_stripped.startswith("tb"):
+                extra_args = f" --folder {self.folder}" if arg_stripped.startswith("tb mock") else ""
+                subprocess.run(arg_stripped + extra_args, shell=True, text=True)
+            elif arg_stripped.startswith("with") or arg_stripped.startswith("select"):
+                subprocess.run(f'tb sql "{arg_stripped}"', shell=True, text=True)
+            elif arg_stripped.startswith("mock "):
+                subprocess.run(f"tb {arg_stripped} --folder {self.folder}", shell=True, text=True)
+            else:
+                click.echo(FeedbackManager.error(message="Invalid command"))
+
+    def reprint_prompt(self):
+        self.stdout.write(self.prompt)
+        self.stdout.flush()
+
+
 class FileChangeHandler(FileSystemEventHandler):
     def __init__(self, filenames: List[str], process: Callable[[List[str]], None]):
         self.filenames = filenames
         self.process = process
 
     def on_modified(self, event: Any) -> None:
-        if not event.is_directory and any(event.src_path.endswith(ext) for ext in [".datasource", ".pipe"]):
+        if not event.is_directory and any(event.src_path.endswith(ext) for ext in [".datasource", ".pipe", ".ndjson"]):
             filename = event.src_path.split("/")[-1]
-            click.echo(FeedbackManager.highlight(message=f"\n⟲ Changes detected in {filename}\n"))
+            click.echo(FeedbackManager.highlight(message=f"\n\n⟲ Changes detected in {filename}\n"))
             try:
                 self.process([event.src_path])
             except Exception as e:
                 click.echo(FeedbackManager.error_exception(error=e))
 
+    def on_created(self, event: Any) -> None:
+        click.echo(FeedbackManager.highlight(message=f"\n\n⟲ Changes detected in {event.src_path}\n"))
+        try:
+            self.process([event.src_path])
+        except Exception as e:
+            click.echo(FeedbackManager.error_exception(error=e))
+
 
 def watch_files(
     filenames: List[str],
     process: Union[Callable[[List[str]], None], Callable[[List[str]], Awaitable[None]]],
+    shell: BuildShell,
 ) -> None:
     # Handle both sync and async process functions
     async def process_wrapper(files: List[str]) -> None:
@@ -61,8 +105,9 @@ def watch_files(
         elapsed_time = time_end - time_start
         click.echo(
             FeedbackManager.success(message="\n✓ ")
-            + FeedbackManager.gray(message=f"Rebuild completed in {elapsed_time:.1f}s\n")
+            + FeedbackManager.gray(message=f"Rebuild completed in {elapsed_time:.1f}s")
         )
+        shell.reprint_prompt()
 
     event_handler = FileChangeHandler(filenames, lambda f: asyncio.run(process_wrapper(f)))
     observer = Observer()
@@ -113,7 +158,9 @@ async def build(
     ignore_sql_errors = FeatureFlags.ignore_sql_errors()
     context.disable_template_security_validation.set(True)
     is_internal = has_internal_datafiles(folder)
-    tb_client = get_tinybird_local_client()
+
+    folder_path = os.path.abspath(folder)
+    tb_client = await get_tinybird_local_client(folder_path)
 
     def check_filenames(filenames: List[str]):
         parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
@@ -134,29 +181,53 @@ async def build(
             parser(filename)
 
     async def process(filenames: List[str], watch: bool = False, only_pipes: bool = False):
-        check_filenames(filenames=filenames)
-        await folder_build(
-            tb_client,
-            filenames,
-            ignore_sql_errors=ignore_sql_errors,
-            is_internal=is_internal,
-            only_pipes=only_pipes,
-        )
+        datafiles = [f for f in filenames if f.endswith(".datasource") or f.endswith(".pipe")]
+        if len(datafiles) > 0:
+            check_filenames(filenames=datafiles)
+            await folder_build(
+                tb_client,
+                filenames=datafiles,
+                ignore_sql_errors=ignore_sql_errors,
+                is_internal=is_internal,
+                only_pipes=only_pipes,
+            )
+
+        filename = filenames[0]
+        if filename.endswith(".ndjson"):
+            fixture_path = Path(filename)
+            name = "_".join(fixture_path.stem.split("_")[:-1])
+            ds_path = Path(folder) / "datasources" / f"{name}.datasource"
+            if ds_path.exists():
+                await append_datasource({}, tb_client, name, str(fixture_path), silent=True)
 
         if watch:
-            filename = filenames[0]
-            if filename.endswith(".pipe"):
-                await build_and_print_pipe(tb_client, filename)
-
-    filenames = get_project_filenames(folder)
+            if filename.endswith(".datasource"):
+                ds_path = Path(filename)
+                name = build_fixture_name(filename, ds_path.stem, ds_path.read_text())
+                fixture_path = get_fixture_dir() / f"{name}.ndjson"
+                if fixture_path.exists():
+                    await append_datasource({}, tb_client, ds_path.stem, str(fixture_path), silent=True)
+            if not filename.endswith(".ndjson"):
+                await build_and_print_resource(tb_client, filename)
+
+    datafiles = get_project_filenames(folder)
+    fixtures = get_project_fixtures(folder)
+    filenames = datafiles + fixtures
 
     async def build_once(filenames: List[str]):
         try:
-            click.echo("⚡ Building project...")
+            click.echo("⚡ Building project...\n")
             time_start = time.time()
             await process(filenames=filenames, watch=False, only_pipes=skip_datasources)
             time_end = time.time()
             elapsed_time = time_end - time_start
+            for filename in filenames:
+                if filename.endswith(".datasource"):
+                    ds_path = Path(filename)
+                    name = build_fixture_name(filename, ds_path.stem, ds_path.read_text())
+                    fixture_path = get_fixture_dir() / f"{name}.ndjson"
+                    if fixture_path.exists():
+                        await append_datasource({}, tb_client, ds_path.stem, str(fixture_path), silent=True)
             click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s\n"))
         except Exception as e:
             click.echo(FeedbackManager.error(message=str(e)))
@@ -164,31 +235,21 @@ async def build(
     await build_once(filenames)
 
     if watch:
-        click.echo(FeedbackManager.highlight(message="◎ Watching for changes...\n"))
-        watcher_thread = threading.Thread(target=watch_files, args=(filenames, process), daemon=True)
+        shell = BuildShell(folder=folder)
+        click.echo(FeedbackManager.highlight(message="◎ Watching for changes..."))
+        watcher_thread = threading.Thread(target=watch_files, args=(filenames, process, shell), daemon=True)
         watcher_thread.start()
+        shell.cmdloop()
 
-        # Main CLI loop
-        while True:
-            user_input = click.prompt("", prompt_suffix="")
-            if user_input.lower() == "exit":
-                break
 
-            if "tb build" in user_input:
-                click.echo(FeedbackManager.error(message="Build command is already running"))
-            else:
-                # Process the user command
-                await sync_to_async(subprocess.run, thread_sensitive=True)(user_input, shell=True, text=True)
-
-            click.echo(FeedbackManager.highlight(message="\n◎ Watching for changes...\n"))
-
-
-async def build_and_print_pipe(tb_client: TinyB, filename: str):
+async def build_and_print_resource(tb_client: TinyB, filename: str):
     rebuild_colors = [bcolors.FAIL, bcolors.OKBLUE, bcolors.WARNING, bcolors.OKGREEN, bcolors.HEADER]
     rebuild_index = random.randint(0, len(rebuild_colors) - 1)
     rebuild_color = rebuild_colors[rebuild_index % len(rebuild_colors)]
-    pipe_name = Path(filename).stem
-    res = await tb_client.query(f"SELECT * FROM {pipe_name} FORMAT JSON", pipeline=pipe_name)
+    resource_path = Path(filename)
+    name = resource_path.stem
+    pipeline = name if filename.endswith(".pipe") else None
+    res = await tb_client.query(f"SELECT * FROM {name} FORMAT JSON", pipeline=pipeline)
     data = []
     limit = 5
     for d in res["data"][:5]:
@@ -197,7 +258,6 @@ async def build_and_print_pipe(tb_client: TinyB, filename: str):
     row_count = res.get("rows", 0)
     stats = res.get("statistics", {})
     elapsed = stats.get("elapsed", 0)
-    node_name = "endpoint"
     cols = len(meta)
     try:
 
@@ -207,7 +267,7 @@ async def build_and_print_pipe(tb_client: TinyB, filename: str):
         table = format_table(data, meta)
         colored_char = print_message("│", rebuild_color)
         table_with_marker = "\n".join(f"{colored_char} {line}" for line in table.split("\n"))
-        click.echo(f"\n{colored_char} {print_message('⚡', rebuild_color)} Running {pipe_name} → {node_name}")
+        click.echo(f"\n{colored_char} {print_message('⚡', rebuild_color)} Running {name}")
         click.echo(colored_char)
         click.echo(table_with_marker)
         click.echo(colored_char)
@@ -225,3 +285,24 @@ async def build_and_print_pipe(tb_client: TinyB, filename: str):
             click.echo("------------")
         else:
             raise exc
+
+
+async def append_datasource(
+    ctx: click.Context,
+    tb_client: TinyB,
+    datasource_name: str,
+    url: str,
+    silent: bool = False,
+):
+    await push_data(
+        ctx,
+        tb_client,
+        datasource_name,
+        url,
+        connector=None,
+        sql=None,
+        mode="append",
+        ignore_empty=False,
+        concurrency=1,
+        silent=silent,
+    )

{tinybird-0.0.1.dev9 → tinybird-0.0.1.dev10}/tinybird/tb/modules/cicd.py

@@ -240,7 +240,6 @@ class GitLabCICDGenerator(CICDGeneratorBase):
             template=GITLAB_CI_YML,
             file_name="tinybird-ci.yml",
             dir_path=".gitlab/tinybird",
-            warning_message="Make sure to import the file in your .gitlab-ci.yml file, e.g., `include: '.gitlab/tinybird/*.yml'`.",
         ),
     ]
 

{tinybird-0.0.1.dev9 → tinybird-0.0.1.dev10}/tinybird/tb/modules/cli.py

@@ -158,9 +158,6 @@ async def cli(
 
         latest_version = await CheckPypi().get_latest_version()
 
-        if "x.y.z" in CURRENT_VERSION:
-            click.echo(FeedbackManager.warning_development_cli())
-
         if "x.y.z" not in CURRENT_VERSION and latest_version != CURRENT_VERSION:
             click.echo(FeedbackManager.warning_update_version(latest_version=latest_version))
             click.echo(FeedbackManager.warning_current_version(current_version=CURRENT_VERSION))

{tinybird-0.0.1.dev9 → tinybird-0.0.1.dev10}/tinybird/tb/modules/common.py

@@ -11,6 +11,7 @@ import os
 import re
 import socket
 import sys
+import time
 import uuid
 from contextlib import closing
 from copy import deepcopy
@@ -166,12 +167,12 @@ def generate_datafile(
     data: Optional[bytes],
     force: Optional[bool] = False,
     _format: Optional[str] = "csv",
-    parent_dir: Optional[str] = None,
+    folder: Optional[str] = None,
 ):
     p = Path(filename)
     base = Path("datasources")
-    if parent_dir:
-        base = Path(parent_dir) / base
+    if folder:
+        base = Path(folder) / base
     datasource_name = normalize_datasource_name(p.stem)
     if not base.exists():
         base = Path()
@@ -179,7 +180,7 @@ def generate_datafile(
     if not f.exists() or force:
         with open(f"{f}", "w") as ds_file:
             ds_file.write(datafile)
-        click.echo(FeedbackManager.info_file_created(file=f))
+        click.echo(FeedbackManager.info_file_created(file=str(f)))
 
         if data and (base / "fixtures").exists():
             # Generating a fixture for Parquet files is not so trivial, since Parquet format
@@ -1078,6 +1079,7 @@ async def push_data(
     replace_options=None,
     ignore_empty: bool = False,
     concurrency: int = 1,
+    silent: bool = False,
 ):
     if url and type(url) is tuple:
         url = url[0]
@@ -1088,7 +1090,8 @@ async def push_data(
             raise CLIException(FeedbackManager.error_connector_not_configured(connector=connector))
         else:
             _connector: "Connector" = ctx.obj[connector]
-            click.echo(FeedbackManager.info_starting_export_process(connector=connector))
+            if not silent:
+                click.echo(FeedbackManager.info_starting_export_process(connector=connector))
             try:
                 url = _connector.export_to_gcs(sql, datasource_name, mode)
             except ConnectorNothingToLoad as e:
@@ -1116,7 +1119,8 @@ async def push_data(
     cb.First = True  # type: ignore[attr-defined]
     cb.prev_done = 0  # type: ignore[attr-defined]
 
-    click.echo(FeedbackManager.gray(message=f"\nImporting data to {datasource_name} Data Source..."))
+    if not silent:
+        click.echo(FeedbackManager.gray(message=f"\nImporting data to {datasource_name}..."))
 
     if isinstance(url, list):
         urls = url
@@ -1187,15 +1191,14 @@ async def push_data(
     except Exception as e:
         raise CLIException(FeedbackManager.error_exception(error=e))
     else:
-        if mode == "append" and parser and parser != "clickhouse":
-            click.echo(FeedbackManager.success_appended_rows(appended_rows=appended_rows))
+        if not silent:
+            if mode == "append" and parser and parser != "clickhouse":
+                click.echo(FeedbackManager.success_appended_rows(appended_rows=appended_rows))
 
-        if mode == "replace":
-            click.echo(FeedbackManager.success_replaced_datasource(datasource=datasource_name))
-        else:
-            click.echo(FeedbackManager.highlight(message="» 2.57m rows x 9 cols in 852.04ms"))
+            if mode == "replace":
+                click.echo(FeedbackManager.success_replaced_datasource(datasource=datasource_name))
 
-        click.echo(FeedbackManager.success_progress_blocks())
+            click.echo(FeedbackManager.success_progress_blocks())
 
     finally:
         try:
@@ -2108,3 +2111,22 @@ def get_ca_pem_content(ca_pem: Optional[str], filename: Optional[str] = None) ->
 
 requests_get = sync_to_async(requests.get, thread_sensitive=False)
 requests_delete = sync_to_async(requests.delete, thread_sensitive=False)
+
+
+def format_data_to_ndjson(data: List[Dict[str, Any]]) -> str:
+    return "\n".join([json.dumps(row) for row in data])
+
+
+async def send_batch_events(
+    client: TinyB, datasource_name: str, data: List[Dict[str, Any]], batch_size: int = 10
+) -> None:
+    rows = len(data)
+    time_start = time.time()
+    for i in range(0, rows, batch_size):
+        batch = data[i : i + batch_size]
+        ndjson_data = format_data_to_ndjson(batch)
+        await client.datasource_events(datasource_name, ndjson_data)
+    time_end = time.time()
+    elapsed_time = time_end - time_start
+    cols = len(data[0].keys()) if len(data) > 0 else 0
+    click.echo(FeedbackManager.highlight(message=f"» {rows} rows x {cols} cols in {elapsed_time:.1f}s"))

tinybird-0.0.1.dev10/tinybird/tb/modules/create.py

@@ -0,0 +1,176 @@
+import os
+import subprocess
+from os import getcwd
+from pathlib import Path
+from typing import Optional
+
+import click
+from click import Context
+
+from tinybird.client import TinyB
+from tinybird.feedback_manager import FeedbackManager
+from tinybird.tb.modules.cicd import init_cicd
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.common import _generate_datafile, coro, generate_datafile
+from tinybird.tb.modules.config import CLIConfig
+from tinybird.tb.modules.datafile.fixture import build_fixture_name, persist_fixture
+from tinybird.tb.modules.exceptions import CLIException
+from tinybird.tb.modules.llm import LLM
+from tinybird.tb.modules.local import get_tinybird_local_client
+
+
+@cli.command()
+@click.option(
+    "--data",
+    type=click.Path(exists=True),
+    default=None,
+    help="Initial data to be used to create the project",
+)
+@click.option(
+    "--prompt",
+    type=str,
+    default=None,
+    help="Prompt to be used to create the project",
+)
+@click.option(
+    "--folder",
+    default=None,
+    type=click.Path(exists=True, file_okay=False),
+    help="Folder where datafiles will be placed",
+)
+@click.option("--rows", type=int, default=10, help="Number of events to send")
+@click.pass_context
+@coro
+async def create(
+    ctx: Context,
+    data: Optional[str],
+    prompt: Optional[str],
+    folder: Optional[str],
+    rows: int,
+) -> None:
+    """Initialize a new project."""
+    folder = folder or getcwd()
+    try:
+        tb_client = await get_tinybird_local_client(os.path.abspath(folder))
+        click.echo(FeedbackManager.gray(message="Creating new project structure..."))
+        await project_create(tb_client, data, prompt, folder)
+        click.echo(FeedbackManager.success(message="✓ Scaffolding completed!\n"))
+
+        click.echo(FeedbackManager.gray(message="\nCreating CI/CD files for GitHub and GitLab..."))
+        init_git(folder)
+        await init_cicd(data_project_dir=os.path.relpath(folder))
+        click.echo(FeedbackManager.success(message="✓ Done!\n"))
+
+        click.echo(FeedbackManager.gray(message="Building fixtures..."))
+
+        if data:
+            ds_name = os.path.basename(data.split(".")[0])
+            data_content = Path(data).read_text()
+            datasource_path = Path(folder) / "datasources" / f"{ds_name}.datasource"
+            fixture_name = build_fixture_name(datasource_path.absolute(), ds_name, datasource_path.read_text())
+            click.echo(FeedbackManager.info(message=f"✓ /fixtures/{ds_name}"))
+            persist_fixture(fixture_name, data_content)
+        elif prompt:
+            datasource_files = [f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")]
+            for datasource_file in datasource_files:
+                datasource_path = Path(folder) / "datasources" / datasource_file
+                llm_config = CLIConfig.get_llm_config()
+                llm = LLM(key=llm_config["api_key"])
+                datasource_name = datasource_path.stem
+                datasource_content = datasource_path.read_text()
+                has_json_path = "`json:" in datasource_content
+                if has_json_path:
+                    sql = await llm.generate_sql_sample_data(tb_client, datasource_name, datasource_content, rows)
+                    result = await tb_client.query(f"{sql} FORMAT JSON")
+                    data = result.get("data", [])
+                    fixture_name = build_fixture_name(datasource_path.absolute(), datasource_name, datasource_content)
+                    persist_fixture(fixture_name, data)
+                    click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}"))
+
+        click.echo(FeedbackManager.success(message="✓ Done!\n"))
+
+        click.echo(FeedbackManager.success(message="✓ Tinybird local is ready!"))
+    except Exception as e:
+        click.echo(FeedbackManager.error(message=f"Error: {str(e)}"))
+
+
+async def project_create(
+    client: TinyB,
+    data: Optional[str],
+    prompt: Optional[str],
+    folder: str,
+):
+    project_paths = ["datasources", "endpoints", "materializations", "copies", "sinks", "fixtures"]
+    force = True
+    for x in project_paths:
+        try:
+            f = Path(folder) / x
+            f.mkdir()
+        except FileExistsError:
+            pass
+        click.echo(FeedbackManager.info_path_created(path=x))
+
+    if data:
+        path = Path(folder) / data
+        format = path.suffix.lstrip(".")
+        try:
+            await _generate_datafile(str(path), client, format=format, force=force)
+        except Exception as e:
+            click.echo(FeedbackManager.error(message=f"Ersssssror: {str(e)}"))
+        name = data.split(".")[0]
+        generate_pipe_file(
+            f"{name}_endpoint",
+            f"""
+NODE endpoint
+SQL >
+    SELECT * from {name}
+TYPE ENDPOINT
+            """,
+            folder,
+        )
+    elif prompt:
+        try:
+            llm_config = CLIConfig.get_llm_config()
+            llm = LLM(key=llm_config["api_key"])
+            result = await llm.create_project(prompt)
+            for ds in result.datasources:
+                content = ds.content.replace("```", "")
+                generate_datafile(
+                    content, filename=f"{ds.name}.datasource", data=None, _format="ndjson", force=force, folder=folder
+                )
+
+            for pipe in result.pipes:
+                content = pipe.content.replace("```", "")
+                generate_pipe_file(pipe.name, content, folder)
+        except Exception as e:
+            click.echo(FeedbackManager.error(message=f"Error: {str(e)}"))
+
+
+def init_git(folder: str):
+    try:
+        path = Path(folder)
+        gitignore_file = path / ".gitignore"
+        git_folder = path / ".git"
+        if not git_folder.exists():
+            subprocess.run(["git", "init"], cwd=path, check=True, capture_output=True)
+
+        if gitignore_file.exists():
+            content = gitignore_file.read_text()
+            if ".tinyb" not in content:
+                gitignore_file.write_text(content + "\n.tinyb\n")
+        else:
+            gitignore_file.write_text(".tinyb\n")
+
+        click.echo(FeedbackManager.info_file_created(file=".gitignore"))
+    except Exception as e:
+        raise CLIException(f"Error initializing Git: {e}")
+
+
+def generate_pipe_file(name: str, content: str, folder: str):
+    base = Path(folder) / "endpoints"
+    if not base.exists():
+        base = Path()
+    f = base / (f"{name}.pipe")
+    with open(f"{f}", "w") as file:
+        file.write(content)
+    click.echo(FeedbackManager.info_file_created(file=f.relative_to(folder)))

{tinybird-0.0.1.dev9 → tinybird-0.0.1.dev10}/tinybird/tb/modules/datafile/build.py

@@ -32,7 +32,6 @@ from tinybird.tb.modules.datafile.common import (
     INTERNAL_TABLES,
     ON_DEMAND,
     PREVIEW_CONNECTOR_SERVICES,
-    TB_LOCAL_WORKSPACE_NAME,
     CopyModes,
     CopyParameters,
     DataFileExtensions,
@@ -58,6 +57,7 @@ async def folder_build(
     only_pipes: bool = False,
     is_vendor: bool = False,
     current_ws: Optional[Dict[str, Any]] = None,
+    local_ws: Optional[Dict[str, Any]] = None,
     workspaces: Optional[List[Dict[str, Any]]] = None,
 ):
     if only_pipes:
@@ -117,8 +117,9 @@ async def folder_build(
         user_client.token = user_token
 
     vendor_workspaces = []
-    user_workspaces = await user_client.user_workspaces()
+
     if vendor_path.exists() and not is_vendor:
+        user_workspaces = await user_client.user_workspaces()
         for x in vendor_path.iterdir():
             if x.is_dir() and x.name not in existing_workspaces:
                 if user_token:
@@ -133,15 +134,23 @@ async def folder_build(
                 workspace_lib_paths.append((x.name, x))
 
     workspaces: List[Dict[str, Any]] = (await user_client.user_workspaces()).get("workspaces", [])
-    local_ws = next((ws for ws in workspaces if ws["name"] == TB_LOCAL_WORKSPACE_NAME), {})
+
+    if not is_vendor:
+        local_workspace = await tb_client.workspace_info()
+        local_ws_id = local_workspace.get("id")
+        local_ws = next((ws for ws in workspaces if ws["id"] == local_ws_id), {})
+
     current_ws: Dict[str, Any] = current_ws or local_ws
+
     for vendor_ws in [ws for ws in workspaces if ws["name"] in [ws["name"] for ws in vendor_workspaces]]:
         ws_client = deepcopy(tb_client)
         ws_client.token = vendor_ws["token"]
         shared_ws_path = Path(folder) / "vendor" / vendor_ws["name"]
 
         if shared_ws_path.exists() and not is_vendor:
-            await folder_build(ws_client, folder=shared_ws_path.as_posix(), is_vendor=True, current_ws=vendor_ws)
+            await folder_build(
+                ws_client, folder=shared_ws_path.as_posix(), is_vendor=True, current_ws=vendor_ws, local_ws=local_ws
+            )
 
     datasources: List[Dict[str, Any]] = await tb_client.datasources()
     pipes: List[Dict[str, Any]] = await tb_client.pipes(dependencies=True)

{tinybird-0.0.1.dev9 → tinybird-0.0.1.dev10}/tinybird/tb/modules/datafile/build_pipe.py

@@ -256,7 +256,6 @@ async def new_pipe(
                     raise click.ClickException(FeedbackManager.error_creating_pipe(error=e))
             else:
                 token_name = tk["token_name"]
-                click.echo(FeedbackManager.info_create_not_found_token(token=token_name))
                 try:
                     r = await tb_client.create_token(
                         token_name, [f"PIPES:{tk['permissions']}:{p['name']}"], "P", p["name"]

{tinybird-0.0.1.dev9 → tinybird-0.0.1.dev10}/tinybird/tb/modules/datafile/common.py

@@ -662,6 +662,18 @@ def get_project_filenames(folder: str, with_vendor=False) -> List[str]:
     ]
     if with_vendor:
         folders.append(f"{folder}/vendor/**/**/*.datasource")
+
+    filenames: List[str] = []
+    for x in folders:
+        filenames += glob.glob(x)
+    return filenames
+
+
+def get_project_fixtures(folder: str) -> List[str]:
+    folders: List[str] = [
+        f"{folder}/fixtures/*.ndjson",
+        f"{folder}/fixtures/*.csv",
+    ]
     filenames: List[str] = []
     for x in folders:
         filenames += glob.glob(x)