tinybird 0.0.1.dev289__tar.gz → 0.0.1.dev291__tar.gz

{tinybird-0.0.1.dev289 → tinybird-0.0.1.dev291}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev289
+Version: 0.0.1.dev291
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev289 → tinybird-0.0.1.dev291}/tinybird/datafile/common.py

@@ -1870,7 +1870,7 @@ def parse(
 
         for entries in args:
             # In case they specify multiple workspaces, handle both line-separated and comma-separated values
-            lines = entries.splitlines()
+            lines = _unquote(entries).splitlines()
             for line in lines:
                 # Split by comma and strip whitespace from each workspace name
                 workspaces = [workspace.strip().rstrip(",") for workspace in line.split(",") if workspace.strip()]
@@ -1957,7 +1957,7 @@ def parse(
             "import_query": assign_var("import_query"),  # Deprecated, BQ and SFK
             "import_table_arn": assign_var("import_table_arn"),  # Only for DynamoDB
             "import_export_bucket": assign_var("import_export_bucket"),  # For DynamoDB
-            "shared_with": shared_with,  # Not supported yet
+            "shared_with": shared_with,
             "export_service": export_service,  # Deprecated
             "forward_query": sql("forward_query"),
             "backfill": assign_var("backfill", allowed_values={"skip"}),

{tinybird-0.0.1.dev289 → tinybird-0.0.1.dev291}/tinybird/sql.py

@@ -7,6 +7,9 @@ from typing import Any, Dict, Iterable, List, Optional
 
 valid_chars_name: str = string.ascii_letters + string.digits + "._`*<>+-'"
 valid_chars_fn: str = valid_chars_name + "[](),=!?:/ \n\t\r"
+# Use sets for O(1) membership checks in hot loops
+_VALID_CHARS_NAME_SET = set(valid_chars_name)
+_VALID_CHARS_FN_SET = set(valid_chars_fn)
 
 INDEX_WHITELIST = ["minmax", "set", "bloom_filter", "ngrambf_v1", "tokenbf_v1"]
 INDEX_SUPPORTED_TYPES = {
@@ -28,6 +31,15 @@ INDEX_SUPPORTED_TYPES = {
     "ngrambf_v1": ["String", "FixedString", "Map"],
 }
 
+# Precompiled regexes used across functions to avoid recompilation overhead
+_RE_FORMAT = re.compile(r"\s+format\s+(\w+)\s*$", re.IGNORECASE)
+_RE_REMOVE_FORMAT = re.compile(r"\s+(format)\s+(\w+)\s*$", re.IGNORECASE)
+_RE_TRY_FIX_NULLABLE_SAF = re.compile(r"SimpleAggregateFunction\((\w+),\s*(?!(?:Nullable))([\w,.()]+)\)")
+_RE_INDEX_ENTRY = re.compile(
+    r"(\w+)\s+([\w\s*\[\]\*\(\),\'\"-><.]+)\s+TYPE\s+(\w+)(?:\(([\w\s*.,]+)\))?(?:\s+GRANULARITY\s+(\d+))?"
+)
+_RE_REPLICATED_MT = re.compile(r"Replicated(.*)MergeTree\(([^\)]*)\)(.*)")
+
 
 @dataclass
 class TableIndex:
@@ -137,10 +149,9 @@ def get_format(sql: str) -> Optional[str]:
     >>> get_format('select * from test formAt JSON')
     'JSON'
     """
-    FORMAT_RE = r"\s+format\s+(\w+)\s*$"
     sql = sql.strip()
-    format = re.findall(FORMAT_RE, sql, re.IGNORECASE)
-    return format[0] if format else None
+    match = _RE_FORMAT.findall(sql)
+    return match[0] if match else None
 
 
 def get_format_group(sql: str) -> str:
@@ -151,10 +162,9 @@ def get_format_group(sql: str) -> str:
     >>> get_format_group('select * from test formAt JSON')
     ' formAt JSON'
     """
-    FORMAT_RE = r"\s+format\s+(\w+)\s*$"
     sql = sql.strip()
-    format = re.search(FORMAT_RE, sql, re.IGNORECASE)
-    return format.group() if format else ""
+    match = _RE_FORMAT.search(sql)
+    return match.group() if match else ""
 
 
 def wrap_finalize_aggregation(sql: str, describe_result: Dict[str, Any], fm_group: Optional[str] = None) -> str:
@@ -186,9 +196,8 @@ def remove_format(sql: str) -> str:
     >>> remove_format('select * from test formAt JSON')
     'select * from test'
     """
-    FORMAT_RE = r"\s+(format)\s+(\w+)\s*$"
     sql = sql.strip()
-    return re.sub(FORMAT_RE, "", sql, flags=re.IGNORECASE)
+    return _RE_REMOVE_FORMAT.sub("", sql)
 
 
 def col_name(name: str, backquotes: bool = True) -> str:
@@ -220,7 +229,7 @@ def try_to_fix_nullable_in_simple_aggregating_function(t: str) -> Optional[str]:
     # as it is done with other aggregate functions.
     # If not, the aggregation could return incorrect results.
     result = None
-    if match := re.search(r"SimpleAggregateFunction\((\w+),\s*(?!(?:Nullable))([\w,.()]+)\)", t):
+    if match := _RE_TRY_FIX_NULLABLE_SAF.search(t):
         fn = match.group(1)
         inner_type = match.group(2)
         result = f"SimpleAggregateFunction({fn}, Nullable({inner_type}))"
@@ -342,10 +351,7 @@ def parse_indexes_structure(indexes: Optional[List[str]]) -> List[TableIndex]:
         if index.count("TYPE") != 1:
             raise ValueError("invalid INDEX format. Usage: `name expr TYPE type_full GRANULARITY granularity`")
 
-        match = re.match(
-            r"(\w+)\s+([\w\s*\[\]\*\(\),\'\"-><.]+)\s+TYPE\s+(\w+)(?:\(([\w\s*.,]+)\))?(?:\s+GRANULARITY\s+(\d+))?",
-            index,
-        )
+        match = _RE_INDEX_ENTRY.match(index)
         if match:
             index_name, a, index_type, value, granularity = match.groups()
             index_expr = f"{index_type}({value})" if value else index_type
@@ -552,7 +558,7 @@ def clean_comments(schema_to_clean: str) -> str:
             if i + 1 < len(line) and line[i] == "-" and line[i + 1] == "-" and not inside_json_path:
                 return line[:i].strip()
 
-            if not inside_json_path and line[i:].startswith("`json:"):
+            if not inside_json_path and line.startswith("`json:", i):
                 inside_json_path = True
             elif inside_json_path and line[i] == "`":
                 inside_json_path = False
@@ -562,12 +568,12 @@ def clean_comments(schema_to_clean: str) -> str:
     if schema_to_clean is None:
         return schema_to_clean
 
-    cleaned_schema = ""
+    out_lines: List[str] = []
     for line in schema_to_clean.splitlines():
         cleaned_line = clean_line_comments(line)
         if cleaned_line:
-            cleaned_schema += cleaned_line + "\n"
-    return cleaned_schema.strip()
+            out_lines.append(cleaned_line)
+    return "\n".join(out_lines).strip()
 
 
 SyntaxExpr = namedtuple("SyntaxExpr", ["name", "regex"])
@@ -608,9 +614,9 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
 
     # Find the first SyntaxExpr in lookup that matches the schema at the current offset
     def lookahead_matches(lookup: Iterable) -> Optional[SyntaxExpr]:
+        # Use substring to preserve semantics of patterns anchored to end ($)
         s = schema[i:]
-        match = next((x for x in lookup if x.regex.match(s)), None)
-        return match
+        return next((x for x in lookup if x.regex.match(s)), None)
 
     def advance_single_char() -> None:
         nonlocal i, line, pos
@@ -647,7 +653,7 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
                 c = schema[i]
                 if c in " \t\r\n":
                     return schema[begin:i]
-                if c not in valid_chars_name:
+                if c not in _VALID_CHARS_NAME_SET:
                     raise ValueError(
                         format_parse_error(schema, i, pos, "wrong value, please check the schema syntax", line=line)
                     )
@@ -677,7 +683,9 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
                 context_stack.append("(")
             elif context is None and lookahead_matches(lookup):
                 return schema[begin:i].strip(" \t\r\n")
-            elif (context is None and c not in valid_chars_fn) or (context == "(" and c not in valid_chars_fn):
+            elif (context is None and c not in _VALID_CHARS_FN_SET) or (
+                context == "(" and c not in _VALID_CHARS_FN_SET
+            ):
                 raise ValueError(
                     format_parse_error(schema, i, pos, "wrong value, please check the schema syntax", line=line)
                 )
@@ -884,7 +892,7 @@ def engine_replicated_to_local(engine: str) -> str:
     if "Replicated" not in engine:
         return engine
 
-    return re.sub(r"Replicated(.*)MergeTree\(([^\)]*)\)(.*)", _replace, engine.strip())
+    return _RE_REPLICATED_MT.sub(_replace, engine.strip())
 
 
 def engine_patch_replicated_engine(engine: str, engine_full: Optional[str], new_table_name: str) -> Optional[str]:

{tinybird-0.0.1.dev289 → tinybird-0.0.1.dev291}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev289'
-__revision__ = 'ac20837'
+__version__ = '0.0.1.dev291'
+__revision__ = '87da698'

{tinybird-0.0.1.dev289 → tinybird-0.0.1.dev291}/tinybird/tb/modules/agent/agent.py

@@ -23,6 +23,7 @@ from tinybird.tb.modules.agent.banner import display_banner
 from tinybird.tb.modules.agent.command_agent import CommandAgent
 from tinybird.tb.modules.agent.compactor import compact_messages
 from tinybird.tb.modules.agent.explore_agent import ExploreAgent
+from tinybird.tb.modules.agent.file_agent import FileAgent
 from tinybird.tb.modules.agent.memory import (
     clear_history,
     clear_messages,
@@ -185,6 +186,16 @@ class TinybirdAgent:
             workspace_id=workspace_id,
             project=self.project,
         )
+        self.file_agent = FileAgent(
+            dangerously_skip_permissions=self.dangerously_skip_permissions,
+            prompt_mode=prompt_mode,
+            thinking_animation=self.thinking_animation,
+            token=self.token,
+            user_token=self.user_token,
+            host=self.host,
+            workspace_id=workspace_id,
+            project=self.project,
+        )
 
         @self.agent.tool
         def manage_tests(ctx: RunContext[TinybirdAgentContext], task: str) -> str:
@@ -251,6 +262,19 @@ class TinybirdAgent:
             result = self.mock_agent.run(user_input, deps=ctx.deps, usage=ctx.usage)
             return result.output or "No result returned"
 
+        @self.agent.tool
+        def manage_files(ctx: RunContext[TinybirdAgentContext], task: str) -> str:
+            """List file/folders and read any type of file in the current directory. Use this tool when you need to list or read non Tinybird project files (e.g. .txt, .md).
+
+            Args:
+                task (str): The task to solve. Required.
+
+            Returns:
+                str: The result of the task.
+            """
+            result = self.file_agent.run(task, deps=ctx.deps, usage=ctx.usage)
+            return result.output or "No result returned"
+
         @self.agent.instructions
         def get_local_host(ctx: RunContext[TinybirdAgentContext]) -> str:
             return f"""

tinybird-0.0.1.dev291/tinybird/tb/modules/agent/file_agent.py

@@ -0,0 +1,64 @@
+from pydantic_ai import Agent, Tool
+from pydantic_ai.messages import ModelMessage
+from pydantic_ai.usage import Usage
+
+from tinybird.tb.modules.agent.animations import ThinkingAnimation
+from tinybird.tb.modules.agent.models import create_model
+from tinybird.tb.modules.agent.prompts import tone_and_style_instructions
+from tinybird.tb.modules.agent.tools.file import list_files, read_file
+from tinybird.tb.modules.agent.utils import TinybirdAgentContext
+from tinybird.tb.modules.project import Project
+
+
+class FileAgent:
+    def __init__(
+        self,
+        token: str,
+        user_token: str,
+        host: str,
+        workspace_id: str,
+        project: Project,
+        dangerously_skip_permissions: bool,
+        prompt_mode: bool,
+        thinking_animation: ThinkingAnimation,
+    ):
+        self.token = token
+        self.user_token = user_token
+        self.host = host
+        self.workspace_id = workspace_id
+        self.dangerously_skip_permissions = dangerously_skip_permissions or prompt_mode
+        self.project = project
+        self.thinking_animation = thinking_animation
+        self.messages: list[ModelMessage] = []
+        self.agent = Agent(
+            model=create_model(user_token, host, workspace_id),
+            deps_type=TinybirdAgentContext,
+            instructions=[
+                """
+You are part of Tinybird Code, an agentic CLI that can help users to work with Tinybird.                 
+You are a sub-agent of the main Tinybird Code agent. You are responsible for managing files.
+You can do the following:
+- List files in a directory.
+- Read a file.
+
+IMPORTANT: Use the `list_files` tool always before `read_file` to get the correct file path.
+""",
+                tone_and_style_instructions,
+            ],
+            tools=[
+                Tool(list_files, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+                Tool(read_file, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+            ],
+        )
+
+    def run(self, task: str, deps: TinybirdAgentContext, usage: Usage):
+        result = self.agent.run_sync(
+            task,
+            deps=deps,
+            usage=usage,
+            message_history=self.messages,
+            model=create_model(self.user_token, self.host, self.workspace_id, run_id=deps.run_id),
+        )
+        new_messages = result.new_messages()
+        self.messages.extend(new_messages)
+        return result

{tinybird-0.0.1.dev289 → tinybird-0.0.1.dev291}/tinybird/tb/modules/agent/tools/append.py

@@ -24,7 +24,7 @@ def append_file(
     Args:
         datasource_name: Name of the datasource to append fixture to
         fixture_pathname: Path to the fixture file to append
-        cloud: Whether to append the fixture to the cloud or local environment. If None (user didn't specify), will ask user to clarify. Defaults to local (False) in dangerous skip permissions mode.
+        cloud: Whether to append the fixture to the cloud or local environment. If None (user didn't specify in a previous step), will ask user to clarify. Defaults to local (False) in dangerous skip permissions mode.
 
     Returns:
         str: Message indicating the success or failure of the appending
@@ -54,6 +54,7 @@ def append_file(
 
         cloud_or_local = "Cloud" if cloud else "Local"
         active_plan = ctx.deps.get_plan() is not None and not cloud
+        ctx.deps.thinking_animation.stop()
         confirmation = show_confirmation(
             title=f"Append fixture {fixture_pathname} to datasource '{datasource_name}' in Tinybird {cloud_or_local}?",
             skip_confirmation=ctx.deps.dangerously_skip_permissions or active_plan,

tinybird-0.0.1.dev291/tinybird/tb/modules/agent/tools/file.py

@@ -0,0 +1,82 @@
+from glob import glob
+from pathlib import Path
+
+import click
+from pydantic_ai import RunContext
+
+from tinybird.tb.modules.agent.utils import TinybirdAgentContext
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+
+
+def list_files(
+    ctx: RunContext[TinybirdAgentContext],
+    folder_path: str = ".",
+    search_pattern: str = "*",
+) -> str:
+    """Lists all files and folders in a directory.
+
+    Args:
+        folder_path (str): The path to the directory to list. If not provided, it will list the files in the current directory. Default: "."
+        search_pattern (str): The pattern to search for in the files and folders. Default: "*"
+    Returns:
+        str: The list of files in the directory.
+    """
+    try:
+        ctx.deps.thinking_animation.stop()
+        if folder_path:
+            path = Path(folder_path)
+        else:
+            path = Path.cwd()
+
+        display_folder = "current directory" if (folder_path == "." or not folder_path) else folder_path
+
+        click.echo(
+            FeedbackManager.highlight(message=f"» Searching in {display_folder} with pattern '{search_pattern}'")
+        )
+
+        if not path.exists():
+            ctx.deps.thinking_animation.start()
+            return f"Folder {folder_path} does not exist"
+
+        result = glob(f"{path}/**/{search_pattern}", recursive=True)
+        folders = [f for f in result if Path(f).is_dir()]
+        files = [f for f in result if Path(f).is_file()]
+
+        ctx.deps.thinking_animation.start()
+        return f"Folders: {folders}\nFiles: {files}"
+    except Exception as e:
+        ctx.deps.thinking_animation.start()
+        return f"Error listing files: {e}"
+
+
+def read_file(ctx: RunContext[TinybirdAgentContext], file_path: str) -> str:
+    """Reads a file from the local filesystem. The file_path parameter must be an absolute path, not a relative path. Any lines longer than 2000 characters will be truncated.
+
+    Args:
+        file_path (str): The path to the file to read.
+
+    Returns:
+        str: The content of the file.
+    """
+    try:
+        ctx.deps.thinking_animation.stop()
+        click.echo(FeedbackManager.highlight(message=f"» Reading file {file_path}"))
+        path = Path(file_path)
+
+        if not path.exists():
+            ctx.deps.thinking_animation.start()
+            return f"File {file_path} does not exist"
+
+        content = path.read_text()
+        chunks = content.split("\n")
+
+        if len(chunks) > 2000:
+            first_2000 = "\n".join(chunks[0:2000])
+            ctx.deps.thinking_animation.start()
+            return f"This file is too large to be read by the agent. Showing first 2000 lines instead:\n\n{first_2000}"
+
+        ctx.deps.thinking_animation.start()
+        return content
+    except Exception as e:
+        ctx.deps.thinking_animation.start()
+        return f"Error reading file: {e}"

{tinybird-0.0.1.dev289 → tinybird-0.0.1.dev291}/tinybird/tb/modules/cli.py

@@ -212,16 +212,39 @@ def cli(
 
 
 @cli.command(hidden=True)
+@click.option("--only-vendored", is_flag=True, default=False, help="Only update vendored files")
 @click.option("-f", "--force", is_flag=True, default=False, help="Override existing files")
 @click.option("--fmt", is_flag=True, default=False, help="Format files before saving")
 @click.pass_context
-def pull(ctx: Context, force: bool, fmt: bool) -> None:
+def pull(ctx: Context, only_vendored: bool, force: bool, fmt: bool) -> None:
     """Retrieve latest versions for project files from Tinybird."""
 
     client = ctx.ensure_object(dict)["client"]
     project = ctx.ensure_object(dict)["project"]
 
-    return folder_pull(client, project.path, force, fmt=fmt)
+    if only_vendored:
+        force = True
+
+    written_files = folder_pull(client, project.path, force, only_vendored=only_vendored, fmt=fmt)
+
+    if only_vendored:
+        for_user_to_delete = set(project.get_vendored_files()) - set(written_files)
+        if for_user_to_delete:
+            # TODO(eclbg): this prints the full path of the files. Let's print the relative path from the project root
+            display_paths = []
+            for full_path in for_user_to_delete:
+                try:
+                    display_paths.append(str(Path(full_path).relative_to(project.path)))
+                except:
+                    display_paths.append(full_path)
+            click.echo(
+                FeedbackManager.warning(
+                    message=(
+                        f"This workspace no longer has access to the following files: {display_paths}. "
+                        "Please remove them manually to be able to deploy."
+                    )
+                )
+            )
 
 
 @cli.command()
@@ -392,7 +415,7 @@ def create_ctx_client(
     if not command or command in commands_without_ctx_client:
         return None
 
-    commands_always_cloud = ["pull", "infra"]
+    commands_always_cloud = ["infra"]
     commands_always_local = ["build", "dev"]
     command_always_test = ["test"]
 

{tinybird-0.0.1.dev289 → tinybird-0.0.1.dev291}/tinybird/tb/modules/datafile/diff.py

@@ -121,12 +121,12 @@ def diff_command(
     if filenames:
         if len(filenames) == 1:
             filenames = [filenames[0], *get_project_filenames(filenames[0])]
-        folder_pull(client, target_dir, True, verbose=False)
+        folder_pull(client, target_dir, True, only_vendored=False, verbose=False)
     else:
         filenames = get_project_filenames(".")
         if verbose:
             click.echo("Saving remote resources in .diff_tmp folder.\n")
-        folder_pull(client, target_dir, True, verbose=verbose, progress_bar=progress_bar)
+        folder_pull(client, target_dir, True, verbose=verbose, only_vendored=False, progress_bar=progress_bar)
 
     remote_datasources: List[Dict[str, Any]] = client.datasources()
     remote_pipes: List[Dict[str, Any]] = client.pipes()

{tinybird-0.0.1.dev289 → tinybird-0.0.1.dev291}/tinybird/tb/modules/datafile/pull.py

@@ -13,10 +13,11 @@ def folder_pull(
     client: TinyB,
     folder: str,
     force: bool,
+    only_vendored: bool,
     verbose: bool = True,
     progress_bar: bool = False,
     fmt: bool = False,
-):
+) -> list[str]:
     def get_file_folder(extension: str, resource_type: Optional[str]):
         if extension == "datasource":
             return "datasources"
@@ -40,8 +41,8 @@ def folder_pull(
         get_resource_function: str,
         progress_bar: bool = False,
         fmt: bool = False,
-    ):
-        def write_resource(k: dict[str, Any]):
+    ) -> list[Path]:
+        def write_resource(k: dict[str, Any]) -> Optional[Path]:
             name = f"{k['name']}.{extension}"
             try:
                 resource = getattr(client, get_resource_function)(k["name"])
@@ -72,59 +73,71 @@ def folder_pull(
                     with open(f, "w") as fd:
                         if resource_to_write:
                             fd.write(resource_to_write)
+                            return f
                 else:
                     if verbose:
                         click.echo(FeedbackManager.info_skip_already_exists())
+                return None
             except Exception as e:
                 raise click.ClickException(FeedbackManager.error_exception(error=e))
 
         if progress_bar:
+            written_files = []
             with click.progressbar(resources, label=f"Pulling {extension}s") as resources:  # type: ignore
                 for k in resources:
-                    write_resource(k)
+                    written_file = write_resource(k)
+                    if written_file:
+                        written_files.append(written_file)
         else:
-            tasks = [write_resource(k) for k in resources]
-            _gather_with_concurrency(5, *tasks)
+            written_files = list(filter(None, (write_resource(k) for k in resources)))
+
+        return written_files
 
     try:
         datasources = client.datasources()
         pipes = client.pipes()
         connections = client.connections()
 
-        write_files(
-            resources=datasources,
-            extension="datasource",
-            get_resource_function="datasource_file",
-            progress_bar=progress_bar,
-            fmt=fmt,
-        )
-        write_files(
-            resources=pipes,
-            extension="pipe",
-            get_resource_function="pipe_file",
-            progress_bar=progress_bar,
-            fmt=fmt,
-        )
-        write_files(
-            resources=connections,
-            extension="connection",
-            get_resource_function="connection_file",
-            progress_bar=progress_bar,
-            fmt=fmt,
-        )
-        return
+        if only_vendored:
+            written_files = write_files(
+                resources=[ds for ds in datasources if "shared_from" in ds],
+                extension="datasource",
+                get_resource_function="datasource_file",
+                progress_bar=progress_bar,
+                fmt=fmt,
+            )
+        else:
+            written_files = []
+            written_files.extend(
+                write_files(
+                    resources=datasources,
+                    extension="datasource",
+                    get_resource_function="datasource_file",
+                    progress_bar=progress_bar,
+                    fmt=fmt,
+                )
+            )
+            written_files.extend(
+                write_files(
+                    resources=pipes,
+                    extension="pipe",
+                    get_resource_function="pipe_file",
+                    progress_bar=progress_bar,
+                    fmt=fmt,
+                )
+            )
+            written_files.extend(
+                write_files(
+                    resources=connections,
+                    extension="connection",
+                    get_resource_function="connection_file",
+                    progress_bar=progress_bar,
+                    fmt=fmt,
+                )
+            )
+        return [str(wf) for wf in written_files]
 
     except AuthNoTokenException:
         raise
     except Exception as e:
         raise click.ClickException(FeedbackManager.error_pull(error=str(e)))
-
-
-def _gather_with_concurrency(n, *tasks):
-    results = []
-    for task in tasks:
-        if callable(task):
-            results.append(task())
-        else:
-            results.append(task)
-    return results

{tinybird-0.0.1.dev289 → tinybird-0.0.1.dev291}/tinybird/tb/modules/deployment_common.py

@@ -222,6 +222,7 @@ def create_deployment(
     # TODO: This code is duplicated in build_server.py
     # Should be refactored to be shared
     MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
+    MULTIPART_BOUNDARY_DATA_PROJECT_VENDORED = "data_project_vendored://"
     DATAFILE_TYPE_TO_CONTENT_TYPE = {
         ".datasource": "text/plain",
         ".pipe": "text/plain",
@@ -248,6 +249,13 @@ def create_deployment(
         with open(file_path, "rb") as fd:
             content_type = DATAFILE_TYPE_TO_CONTENT_TYPE.get(Path(file_path).suffix, "application/unknown")
             files.append((MULTIPART_BOUNDARY_DATA_PROJECT, (relative_path, fd.read().decode("utf-8"), content_type)))
+    for file_path in project.get_vendored_files():
+        relative_path = Path(file_path).relative_to(project.path).as_posix()
+        with open(file_path, "rb") as fd:
+            content_type = DATAFILE_TYPE_TO_CONTENT_TYPE.get(Path(file_path).suffix, "application/unknown")
+            files.append(
+                (MULTIPART_BOUNDARY_DATA_PROJECT_VENDORED, (relative_path, fd.read().decode("utf-8"), content_type))
+            )
 
     deployment = None
     try:

{tinybird-0.0.1.dev289 → tinybird-0.0.1.dev291}/tinybird/tb/modules/project.py

@@ -37,11 +37,13 @@ class Project:
     def has_deeper_level(self) -> bool:
         """Check if there are folders with depth greater than max_depth in project path.
 
+        Does not consider the vendor directory.
+
         Returns:
             bool: True if there are folders deeper than max_depth, False otherwise
         """
         for obj in glob.glob(f"{self.path}{'/*' * (self.max_depth - 1)}/*", recursive=False):
-            if Path(obj).is_dir():
+            if Path(obj).is_dir() and self.vendor_path not in obj:
                 return True
         return False
 
@@ -54,6 +56,13 @@ class Project:
                 project_files.append(project_file)
         return project_files
 
+    def get_vendored_files(self) -> List[str]:
+        vendored_files: List[str] = []
+        for extension in self.extensions:
+            for level in range(3):
+                vendored_files.extend(glob.glob(f"{self.vendor_path}{'/*' * level}/*.{extension}", recursive=True))
+        return vendored_files
+
     def get_fixture_files(self) -> List[str]:
         fixture_files: List[str] = []
         for extension in [

{tinybird-0.0.1.dev289 → tinybird-0.0.1.dev291}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev289
+Version: 0.0.1.dev291
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird