tinybird 0.0.1.dev289__py3-none-any.whl → 0.0.1.dev291__py3-none-any.whl

tinybird/datafile/common.py

@@ -1870,7 +1870,7 @@ def parse(
 
         for entries in args:
             # In case they specify multiple workspaces, handle both line-separated and comma-separated values
-            lines = entries.splitlines()
+            lines = _unquote(entries).splitlines()
             for line in lines:
                 # Split by comma and strip whitespace from each workspace name
                 workspaces = [workspace.strip().rstrip(",") for workspace in line.split(",") if workspace.strip()]
@@ -1957,7 +1957,7 @@ def parse(
             "import_query": assign_var("import_query"),  # Deprecated, BQ and SFK
             "import_table_arn": assign_var("import_table_arn"),  # Only for DynamoDB
             "import_export_bucket": assign_var("import_export_bucket"),  # For DynamoDB
-            "shared_with": shared_with,  # Not supported yet
+            "shared_with": shared_with,
             "export_service": export_service,  # Deprecated
             "forward_query": sql("forward_query"),
             "backfill": assign_var("backfill", allowed_values={"skip"}),

tinybird/sql.py

@@ -7,6 +7,9 @@ from typing import Any, Dict, Iterable, List, Optional
 
 valid_chars_name: str = string.ascii_letters + string.digits + "._`*<>+-'"
 valid_chars_fn: str = valid_chars_name + "[](),=!?:/ \n\t\r"
+# Use sets for O(1) membership checks in hot loops
+_VALID_CHARS_NAME_SET = set(valid_chars_name)
+_VALID_CHARS_FN_SET = set(valid_chars_fn)
 
 INDEX_WHITELIST = ["minmax", "set", "bloom_filter", "ngrambf_v1", "tokenbf_v1"]
 INDEX_SUPPORTED_TYPES = {
@@ -28,6 +31,15 @@ INDEX_SUPPORTED_TYPES = {
     "ngrambf_v1": ["String", "FixedString", "Map"],
 }
 
+# Precompiled regexes used across functions to avoid recompilation overhead
+_RE_FORMAT = re.compile(r"\s+format\s+(\w+)\s*$", re.IGNORECASE)
+_RE_REMOVE_FORMAT = re.compile(r"\s+(format)\s+(\w+)\s*$", re.IGNORECASE)
+_RE_TRY_FIX_NULLABLE_SAF = re.compile(r"SimpleAggregateFunction\((\w+),\s*(?!(?:Nullable))([\w,.()]+)\)")
+_RE_INDEX_ENTRY = re.compile(
+    r"(\w+)\s+([\w\s*\[\]\*\(\),\'\"-><.]+)\s+TYPE\s+(\w+)(?:\(([\w\s*.,]+)\))?(?:\s+GRANULARITY\s+(\d+))?"
+)
+_RE_REPLICATED_MT = re.compile(r"Replicated(.*)MergeTree\(([^\)]*)\)(.*)")
+
 
 @dataclass
 class TableIndex:
@@ -137,10 +149,9 @@ def get_format(sql: str) -> Optional[str]:
     >>> get_format('select * from test formAt JSON')
     'JSON'
     """
-    FORMAT_RE = r"\s+format\s+(\w+)\s*$"
     sql = sql.strip()
-    format = re.findall(FORMAT_RE, sql, re.IGNORECASE)
-    return format[0] if format else None
+    match = _RE_FORMAT.findall(sql)
+    return match[0] if match else None
 
 
 def get_format_group(sql: str) -> str:
@@ -151,10 +162,9 @@ def get_format_group(sql: str) -> str:
     >>> get_format_group('select * from test formAt JSON')
     ' formAt JSON'
     """
-    FORMAT_RE = r"\s+format\s+(\w+)\s*$"
     sql = sql.strip()
-    format = re.search(FORMAT_RE, sql, re.IGNORECASE)
-    return format.group() if format else ""
+    match = _RE_FORMAT.search(sql)
+    return match.group() if match else ""
 
 
 def wrap_finalize_aggregation(sql: str, describe_result: Dict[str, Any], fm_group: Optional[str] = None) -> str:
@@ -186,9 +196,8 @@ def remove_format(sql: str) -> str:
     >>> remove_format('select * from test formAt JSON')
     'select * from test'
     """
-    FORMAT_RE = r"\s+(format)\s+(\w+)\s*$"
     sql = sql.strip()
-    return re.sub(FORMAT_RE, "", sql, flags=re.IGNORECASE)
+    return _RE_REMOVE_FORMAT.sub("", sql)
 
 
 def col_name(name: str, backquotes: bool = True) -> str:
@@ -220,7 +229,7 @@ def try_to_fix_nullable_in_simple_aggregating_function(t: str) -> Optional[str]:
     # as it is done with other aggregate functions.
     # If not, the aggregation could return incorrect results.
     result = None
-    if match := re.search(r"SimpleAggregateFunction\((\w+),\s*(?!(?:Nullable))([\w,.()]+)\)", t):
+    if match := _RE_TRY_FIX_NULLABLE_SAF.search(t):
         fn = match.group(1)
         inner_type = match.group(2)
         result = f"SimpleAggregateFunction({fn}, Nullable({inner_type}))"
@@ -342,10 +351,7 @@ def parse_indexes_structure(indexes: Optional[List[str]]) -> List[TableIndex]:
         if index.count("TYPE") != 1:
             raise ValueError("invalid INDEX format. Usage: `name expr TYPE type_full GRANULARITY granularity`")
 
-        match = re.match(
-            r"(\w+)\s+([\w\s*\[\]\*\(\),\'\"-><.]+)\s+TYPE\s+(\w+)(?:\(([\w\s*.,]+)\))?(?:\s+GRANULARITY\s+(\d+))?",
-            index,
-        )
+        match = _RE_INDEX_ENTRY.match(index)
         if match:
             index_name, a, index_type, value, granularity = match.groups()
             index_expr = f"{index_type}({value})" if value else index_type
@@ -552,7 +558,7 @@ def clean_comments(schema_to_clean: str) -> str:
             if i + 1 < len(line) and line[i] == "-" and line[i + 1] == "-" and not inside_json_path:
                 return line[:i].strip()
 
-            if not inside_json_path and line[i:].startswith("`json:"):
+            if not inside_json_path and line.startswith("`json:", i):
                 inside_json_path = True
             elif inside_json_path and line[i] == "`":
                 inside_json_path = False
@@ -562,12 +568,12 @@ def clean_comments(schema_to_clean: str) -> str:
     if schema_to_clean is None:
         return schema_to_clean
 
-    cleaned_schema = ""
+    out_lines: List[str] = []
     for line in schema_to_clean.splitlines():
         cleaned_line = clean_line_comments(line)
         if cleaned_line:
-            cleaned_schema += cleaned_line + "\n"
-    return cleaned_schema.strip()
+            out_lines.append(cleaned_line)
+    return "\n".join(out_lines).strip()
 
 
 SyntaxExpr = namedtuple("SyntaxExpr", ["name", "regex"])
@@ -608,9 +614,9 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
 
     # Find the first SyntaxExpr in lookup that matches the schema at the current offset
     def lookahead_matches(lookup: Iterable) -> Optional[SyntaxExpr]:
+        # Use substring to preserve semantics of patterns anchored to end ($)
         s = schema[i:]
-        match = next((x for x in lookup if x.regex.match(s)), None)
-        return match
+        return next((x for x in lookup if x.regex.match(s)), None)
 
     def advance_single_char() -> None:
         nonlocal i, line, pos
@@ -647,7 +653,7 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
                 c = schema[i]
                 if c in " \t\r\n":
                     return schema[begin:i]
-                if c not in valid_chars_name:
+                if c not in _VALID_CHARS_NAME_SET:
                     raise ValueError(
                         format_parse_error(schema, i, pos, "wrong value, please check the schema syntax", line=line)
                     )
@@ -677,7 +683,9 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
                 context_stack.append("(")
             elif context is None and lookahead_matches(lookup):
                 return schema[begin:i].strip(" \t\r\n")
-            elif (context is None and c not in valid_chars_fn) or (context == "(" and c not in valid_chars_fn):
+            elif (context is None and c not in _VALID_CHARS_FN_SET) or (
+                context == "(" and c not in _VALID_CHARS_FN_SET
+            ):
                 raise ValueError(
                     format_parse_error(schema, i, pos, "wrong value, please check the schema syntax", line=line)
                 )
@@ -884,7 +892,7 @@ def engine_replicated_to_local(engine: str) -> str:
     if "Replicated" not in engine:
         return engine
 
-    return re.sub(r"Replicated(.*)MergeTree\(([^\)]*)\)(.*)", _replace, engine.strip())
+    return _RE_REPLICATED_MT.sub(_replace, engine.strip())
 
 
 def engine_patch_replicated_engine(engine: str, engine_full: Optional[str], new_table_name: str) -> Optional[str]:

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev289'
-__revision__ = 'ac20837'
+__version__ = '0.0.1.dev291'
+__revision__ = '87da698'

tinybird/tb/modules/agent/agent.py

@@ -23,6 +23,7 @@ from tinybird.tb.modules.agent.banner import display_banner
 from tinybird.tb.modules.agent.command_agent import CommandAgent
 from tinybird.tb.modules.agent.compactor import compact_messages
 from tinybird.tb.modules.agent.explore_agent import ExploreAgent
+from tinybird.tb.modules.agent.file_agent import FileAgent
 from tinybird.tb.modules.agent.memory import (
     clear_history,
     clear_messages,
@@ -185,6 +186,16 @@ class TinybirdAgent:
             workspace_id=workspace_id,
             project=self.project,
         )
+        self.file_agent = FileAgent(
+            dangerously_skip_permissions=self.dangerously_skip_permissions,
+            prompt_mode=prompt_mode,
+            thinking_animation=self.thinking_animation,
+            token=self.token,
+            user_token=self.user_token,
+            host=self.host,
+            workspace_id=workspace_id,
+            project=self.project,
+        )
 
         @self.agent.tool
         def manage_tests(ctx: RunContext[TinybirdAgentContext], task: str) -> str:
@@ -251,6 +262,19 @@ class TinybirdAgent:
             result = self.mock_agent.run(user_input, deps=ctx.deps, usage=ctx.usage)
             return result.output or "No result returned"
 
+        @self.agent.tool
+        def manage_files(ctx: RunContext[TinybirdAgentContext], task: str) -> str:
+            """List file/folders and read any type of file in the current directory. Use this tool when you need to list or read non Tinybird project files (e.g. .txt, .md).
+
+            Args:
+                task (str): The task to solve. Required.
+
+            Returns:
+                str: The result of the task.
+            """
+            result = self.file_agent.run(task, deps=ctx.deps, usage=ctx.usage)
+            return result.output or "No result returned"
+
         @self.agent.instructions
         def get_local_host(ctx: RunContext[TinybirdAgentContext]) -> str:
             return f"""

tinybird/tb/modules/agent/file_agent.py

@@ -0,0 +1,64 @@
+from pydantic_ai import Agent, Tool
+from pydantic_ai.messages import ModelMessage
+from pydantic_ai.usage import Usage
+
+from tinybird.tb.modules.agent.animations import ThinkingAnimation
+from tinybird.tb.modules.agent.models import create_model
+from tinybird.tb.modules.agent.prompts import tone_and_style_instructions
+from tinybird.tb.modules.agent.tools.file import list_files, read_file
+from tinybird.tb.modules.agent.utils import TinybirdAgentContext
+from tinybird.tb.modules.project import Project
+
+
+class FileAgent:
+    def __init__(
+        self,
+        token: str,
+        user_token: str,
+        host: str,
+        workspace_id: str,
+        project: Project,
+        dangerously_skip_permissions: bool,
+        prompt_mode: bool,
+        thinking_animation: ThinkingAnimation,
+    ):
+        self.token = token
+        self.user_token = user_token
+        self.host = host
+        self.workspace_id = workspace_id
+        self.dangerously_skip_permissions = dangerously_skip_permissions or prompt_mode
+        self.project = project
+        self.thinking_animation = thinking_animation
+        self.messages: list[ModelMessage] = []
+        self.agent = Agent(
+            model=create_model(user_token, host, workspace_id),
+            deps_type=TinybirdAgentContext,
+            instructions=[
+                """
+You are part of Tinybird Code, an agentic CLI that can help users to work with Tinybird.                 
+You are a sub-agent of the main Tinybird Code agent. You are responsible for managing files.
+You can do the following:
+- List files in a directory.
+- Read a file.
+
+IMPORTANT: Use the `list_files` tool always before `read_file` to get the correct file path.
+""",
+                tone_and_style_instructions,
+            ],
+            tools=[
+                Tool(list_files, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+                Tool(read_file, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+            ],
+        )
+
+    def run(self, task: str, deps: TinybirdAgentContext, usage: Usage):
+        result = self.agent.run_sync(
+            task,
+            deps=deps,
+            usage=usage,
+            message_history=self.messages,
+            model=create_model(self.user_token, self.host, self.workspace_id, run_id=deps.run_id),
+        )
+        new_messages = result.new_messages()
+        self.messages.extend(new_messages)
+        return result

tinybird/tb/modules/agent/tools/append.py

@@ -24,7 +24,7 @@ def append_file(
     Args:
         datasource_name: Name of the datasource to append fixture to
         fixture_pathname: Path to the fixture file to append
-        cloud: Whether to append the fixture to the cloud or local environment. If None (user didn't specify), will ask user to clarify. Defaults to local (False) in dangerous skip permissions mode.
+        cloud: Whether to append the fixture to the cloud or local environment. If None (user didn't specify in a previous step), will ask user to clarify. Defaults to local (False) in dangerous skip permissions mode.
 
     Returns:
         str: Message indicating the success or failure of the appending
@@ -54,6 +54,7 @@ def append_file(
 
         cloud_or_local = "Cloud" if cloud else "Local"
         active_plan = ctx.deps.get_plan() is not None and not cloud
+        ctx.deps.thinking_animation.stop()
         confirmation = show_confirmation(
             title=f"Append fixture {fixture_pathname} to datasource '{datasource_name}' in Tinybird {cloud_or_local}?",
             skip_confirmation=ctx.deps.dangerously_skip_permissions or active_plan,

tinybird/tb/modules/agent/tools/file.py

@@ -0,0 +1,82 @@
+from glob import glob
+from pathlib import Path
+
+import click
+from pydantic_ai import RunContext
+
+from tinybird.tb.modules.agent.utils import TinybirdAgentContext
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+
+
+def list_files(
+    ctx: RunContext[TinybirdAgentContext],
+    folder_path: str = ".",
+    search_pattern: str = "*",
+) -> str:
+    """Lists all files and folders in a directory.
+
+    Args:
+        folder_path (str): The path to the directory to list. If not provided, it will list the files in the current directory. Default: "."
+        search_pattern (str): The pattern to search for in the files and folders. Default: "*"
+    Returns:
+        str: The list of files in the directory.
+    """
+    try:
+        ctx.deps.thinking_animation.stop()
+        if folder_path:
+            path = Path(folder_path)
+        else:
+            path = Path.cwd()
+
+        display_folder = "current directory" if (folder_path == "." or not folder_path) else folder_path
+
+        click.echo(
+            FeedbackManager.highlight(message=f"» Searching in {display_folder} with pattern '{search_pattern}'")
+        )
+
+        if not path.exists():
+            ctx.deps.thinking_animation.start()
+            return f"Folder {folder_path} does not exist"
+
+        result = glob(f"{path}/**/{search_pattern}", recursive=True)
+        folders = [f for f in result if Path(f).is_dir()]
+        files = [f for f in result if Path(f).is_file()]
+
+        ctx.deps.thinking_animation.start()
+        return f"Folders: {folders}\nFiles: {files}"
+    except Exception as e:
+        ctx.deps.thinking_animation.start()
+        return f"Error listing files: {e}"
+
+
+def read_file(ctx: RunContext[TinybirdAgentContext], file_path: str) -> str:
+    """Reads a file from the local filesystem. The file_path parameter must be an absolute path, not a relative path. Any lines longer than 2000 characters will be truncated.
+
+    Args:
+        file_path (str): The path to the file to read.
+
+    Returns:
+        str: The content of the file.
+    """
+    try:
+        ctx.deps.thinking_animation.stop()
+        click.echo(FeedbackManager.highlight(message=f"» Reading file {file_path}"))
+        path = Path(file_path)
+
+        if not path.exists():
+            ctx.deps.thinking_animation.start()
+            return f"File {file_path} does not exist"
+
+        content = path.read_text()
+        chunks = content.split("\n")
+
+        if len(chunks) > 2000:
+            first_2000 = "\n".join(chunks[0:2000])
+            ctx.deps.thinking_animation.start()
+            return f"This file is too large to be read by the agent. Showing first 2000 lines instead:\n\n{first_2000}"
+
+        ctx.deps.thinking_animation.start()
+        return content
+    except Exception as e:
+        ctx.deps.thinking_animation.start()
+        return f"Error reading file: {e}"

tinybird/tb/modules/cli.py

@@ -212,16 +212,39 @@ def cli(
 
 
 @cli.command(hidden=True)
+@click.option("--only-vendored", is_flag=True, default=False, help="Only update vendored files")
 @click.option("-f", "--force", is_flag=True, default=False, help="Override existing files")
 @click.option("--fmt", is_flag=True, default=False, help="Format files before saving")
 @click.pass_context
-def pull(ctx: Context, force: bool, fmt: bool) -> None:
+def pull(ctx: Context, only_vendored: bool, force: bool, fmt: bool) -> None:
     """Retrieve latest versions for project files from Tinybird."""
 
     client = ctx.ensure_object(dict)["client"]
     project = ctx.ensure_object(dict)["project"]
 
-    return folder_pull(client, project.path, force, fmt=fmt)
+    if only_vendored:
+        force = True
+
+    written_files = folder_pull(client, project.path, force, only_vendored=only_vendored, fmt=fmt)
+
+    if only_vendored:
+        for_user_to_delete = set(project.get_vendored_files()) - set(written_files)
+        if for_user_to_delete:
+            # TODO(eclbg): this prints the full path of the files. Let's print the relative path from the project root
+            display_paths = []
+            for full_path in for_user_to_delete:
+                try:
+                    display_paths.append(str(Path(full_path).relative_to(project.path)))
+                except:
+                    display_paths.append(full_path)
+            click.echo(
+                FeedbackManager.warning(
+                    message=(
+                        f"This workspace no longer has access to the following files: {display_paths}. "
+                        "Please remove them manually to be able to deploy."
+                    )
+                )
+            )
 
 
 @cli.command()
@@ -392,7 +415,7 @@ def create_ctx_client(
     if not command or command in commands_without_ctx_client:
         return None
 
-    commands_always_cloud = ["pull", "infra"]
+    commands_always_cloud = ["infra"]
     commands_always_local = ["build", "dev"]
     command_always_test = ["test"]
 

tinybird/tb/modules/datafile/diff.py

@@ -121,12 +121,12 @@ def diff_command(
     if filenames:
         if len(filenames) == 1:
             filenames = [filenames[0], *get_project_filenames(filenames[0])]
-        folder_pull(client, target_dir, True, verbose=False)
+        folder_pull(client, target_dir, True, only_vendored=False, verbose=False)
     else:
         filenames = get_project_filenames(".")
         if verbose:
             click.echo("Saving remote resources in .diff_tmp folder.\n")
-        folder_pull(client, target_dir, True, verbose=verbose, progress_bar=progress_bar)
+        folder_pull(client, target_dir, True, verbose=verbose, only_vendored=False, progress_bar=progress_bar)
 
     remote_datasources: List[Dict[str, Any]] = client.datasources()
     remote_pipes: List[Dict[str, Any]] = client.pipes()

tinybird/tb/modules/datafile/pull.py

@@ -13,10 +13,11 @@ def folder_pull(
     client: TinyB,
     folder: str,
     force: bool,
+    only_vendored: bool,
     verbose: bool = True,
     progress_bar: bool = False,
     fmt: bool = False,
-):
+) -> list[str]:
     def get_file_folder(extension: str, resource_type: Optional[str]):
         if extension == "datasource":
             return "datasources"
@@ -40,8 +41,8 @@ def folder_pull(
         get_resource_function: str,
         progress_bar: bool = False,
         fmt: bool = False,
-    ):
-        def write_resource(k: dict[str, Any]):
+    ) -> list[Path]:
+        def write_resource(k: dict[str, Any]) -> Optional[Path]:
             name = f"{k['name']}.{extension}"
             try:
                 resource = getattr(client, get_resource_function)(k["name"])
@@ -72,59 +73,71 @@ def folder_pull(
                     with open(f, "w") as fd:
                         if resource_to_write:
                             fd.write(resource_to_write)
+                            return f
                 else:
                     if verbose:
                         click.echo(FeedbackManager.info_skip_already_exists())
+                return None
             except Exception as e:
                 raise click.ClickException(FeedbackManager.error_exception(error=e))
 
         if progress_bar:
+            written_files = []
             with click.progressbar(resources, label=f"Pulling {extension}s") as resources:  # type: ignore
                 for k in resources:
-                    write_resource(k)
+                    written_file = write_resource(k)
+                    if written_file:
+                        written_files.append(written_file)
         else:
-            tasks = [write_resource(k) for k in resources]
-            _gather_with_concurrency(5, *tasks)
+            written_files = list(filter(None, (write_resource(k) for k in resources)))
+
+        return written_files
 
     try:
         datasources = client.datasources()
         pipes = client.pipes()
         connections = client.connections()
 
-        write_files(
-            resources=datasources,
-            extension="datasource",
-            get_resource_function="datasource_file",
-            progress_bar=progress_bar,
-            fmt=fmt,
-        )
-        write_files(
-            resources=pipes,
-            extension="pipe",
-            get_resource_function="pipe_file",
-            progress_bar=progress_bar,
-            fmt=fmt,
-        )
-        write_files(
-            resources=connections,
-            extension="connection",
-            get_resource_function="connection_file",
-            progress_bar=progress_bar,
-            fmt=fmt,
-        )
-        return
+        if only_vendored:
+            written_files = write_files(
+                resources=[ds for ds in datasources if "shared_from" in ds],
+                extension="datasource",
+                get_resource_function="datasource_file",
+                progress_bar=progress_bar,
+                fmt=fmt,
+            )
+        else:
+            written_files = []
+            written_files.extend(
+                write_files(
+                    resources=datasources,
+                    extension="datasource",
+                    get_resource_function="datasource_file",
+                    progress_bar=progress_bar,
+                    fmt=fmt,
+                )
+            )
+            written_files.extend(
+                write_files(
+                    resources=pipes,
+                    extension="pipe",
+                    get_resource_function="pipe_file",
+                    progress_bar=progress_bar,
+                    fmt=fmt,
+                )
+            )
+            written_files.extend(
+                write_files(
+                    resources=connections,
+                    extension="connection",
+                    get_resource_function="connection_file",
+                    progress_bar=progress_bar,
+                    fmt=fmt,
+                )
+            )
+        return [str(wf) for wf in written_files]
 
     except AuthNoTokenException:
         raise
     except Exception as e:
         raise click.ClickException(FeedbackManager.error_pull(error=str(e)))
-
-
-def _gather_with_concurrency(n, *tasks):
-    results = []
-    for task in tasks:
-        if callable(task):
-            results.append(task())
-        else:
-            results.append(task)
-    return results

tinybird/tb/modules/deployment_common.py

@@ -222,6 +222,7 @@ def create_deployment(
     # TODO: This code is duplicated in build_server.py
     # Should be refactored to be shared
     MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
+    MULTIPART_BOUNDARY_DATA_PROJECT_VENDORED = "data_project_vendored://"
     DATAFILE_TYPE_TO_CONTENT_TYPE = {
         ".datasource": "text/plain",
         ".pipe": "text/plain",
@@ -248,6 +249,13 @@ def create_deployment(
         with open(file_path, "rb") as fd:
             content_type = DATAFILE_TYPE_TO_CONTENT_TYPE.get(Path(file_path).suffix, "application/unknown")
             files.append((MULTIPART_BOUNDARY_DATA_PROJECT, (relative_path, fd.read().decode("utf-8"), content_type)))
+    for file_path in project.get_vendored_files():
+        relative_path = Path(file_path).relative_to(project.path).as_posix()
+        with open(file_path, "rb") as fd:
+            content_type = DATAFILE_TYPE_TO_CONTENT_TYPE.get(Path(file_path).suffix, "application/unknown")
+            files.append(
+                (MULTIPART_BOUNDARY_DATA_PROJECT_VENDORED, (relative_path, fd.read().decode("utf-8"), content_type))
+            )
 
     deployment = None
     try:

tinybird/tb/modules/project.py

@@ -37,11 +37,13 @@ class Project:
     def has_deeper_level(self) -> bool:
         """Check if there are folders with depth greater than max_depth in project path.
 
+        Does not consider the vendor directory.
+
         Returns:
             bool: True if there are folders deeper than max_depth, False otherwise
         """
         for obj in glob.glob(f"{self.path}{'/*' * (self.max_depth - 1)}/*", recursive=False):
-            if Path(obj).is_dir():
+            if Path(obj).is_dir() and self.vendor_path not in obj:
                 return True
         return False
 
@@ -54,6 +56,13 @@ class Project:
                 project_files.append(project_file)
         return project_files
 
+    def get_vendored_files(self) -> List[str]:
+        vendored_files: List[str] = []
+        for extension in self.extensions:
+            for level in range(3):
+                vendored_files.extend(glob.glob(f"{self.vendor_path}{'/*' * level}/*.{extension}", recursive=True))
+        return vendored_files
+
     def get_fixture_files(self) -> List[str]:
         fixture_files: List[str] = []
         for extension in [

{tinybird-0.0.1.dev289.dist-info → tinybird-0.0.1.dev291.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev289
+Version: 0.0.1.dev291
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev289.dist-info → tinybird-0.0.1.dev291.dist-info}/RECORD

@@ -5,7 +5,7 @@ tinybird/feedback_manager.py,sha256=XY8d83pRlq-LH7xHMApkaEebfXEWLjDzrGe1prpcTHE,
 tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
 tinybird/prompts.py,sha256=HoDv9TxPiP8v2XoGTWYxP133dK9CEbXVv4XE5IT339c,45483
 tinybird/service_datasources.py,sha256=o6Az3T6OvcChR_7GXRu7sJH173KzGg1Gv-dNd0bI_vY,46085
-tinybird/sql.py,sha256=UZJLop6zA9tTPEaS-Fq7M-QyzmC5uV_tIeXZzkjnhso,48299
+tinybird/sql.py,sha256=7pkwQMUVy0CH5zFgLMZyCx1BeaJSQYC05EmOb9vO3Ls,48694
 tinybird/sql_template.py,sha256=kaF5pi-f2JiWSYXEF8JsU1OIxvdu2homHnw4MYjq0n8,101953
 tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
 tinybird/sql_toolset.py,sha256=Y2_fQrbk5GSNoRncAmRS_snpV61XQbiOy4uYkeF6pr4,19767
@@ -13,12 +13,12 @@ tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
 tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
 tinybird/ch_utils/constants.py,sha256=v5-nkXHUhysu4i9Z4WVv0-sBbh6xSYUH5q5xHSY2xTI,4194
 tinybird/ch_utils/engine.py,sha256=4X1B-iuhdW_mxKnX_m3iCsxgP9RPVgR75g7yH1vsJ6A,40851
-tinybird/datafile/common.py,sha256=Cj-CyO6TKa9_31Mao5qQffIDDtoLnKfe93cubGF5S-Y,105670
+tinybird/datafile/common.py,sha256=hr9Qv4Xot21l2dW6xKPa5_JUu8pnGuDNnVpDf8pgthc,105659
 tinybird/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
 tinybird/datafile/parse_connection.py,sha256=tRyn2Rpr1TeWet5BXmMoQgaotbGdYep1qiTak_OqC5E,1825
 tinybird/datafile/parse_datasource.py,sha256=ssW8QeFSgglVFi3sDZj_HgkJiTJ2069v2JgqnH3CkDE,1825
 tinybird/datafile/parse_pipe.py,sha256=xf4m0Tw44QWJzHzAm7Z7FwUoUUtr7noMYjU1NiWnX0k,3880
-tinybird/tb/__cli__.py,sha256=mQDawLTa-oQDyZZZYyjnBrWWRtaViQL0ZaN2lmv4e1E,247
+tinybird/tb/__cli__.py,sha256=Etfq1IZSoQiHmnt8KFW5i_w4w_P9rnn-X56chF8OH3o,247
 tinybird/tb/check_pypi.py,sha256=Gp0HkHHDFMSDL6nxKlOY51z7z1Uv-2LRexNTZSHHGmM,552
 tinybird/tb/cli.py,sha256=FdDFEIayjmsZEVsVSSvRiVYn_FHOVg_zWQzchnzfWho,1008
 tinybird/tb/client.py,sha256=IQRaInDjOwr9Fzaz3_xXc3aUGqh94tM2lew7IZbB9eM,53733
@@ -26,7 +26,7 @@ tinybird/tb/config.py,sha256=mhMTGnMB5KcxGoh3dewIr2Jjsa6pHE183gCPAQWyp6o,3973
 tinybird/tb/modules/build.py,sha256=lSCbmrqDxqa8bG009qgp2nNRpQW7JWgSen7UQNCfLVE,7657
 tinybird/tb/modules/build_common.py,sha256=KGCtyY1JGexXXhcMVFI3e-gRavmRfCoZnfGPSTYHGxk,12962
 tinybird/tb/modules/cicd.py,sha256=0KLKccha9IP749QvlXBmzdWv1On3mFwMY4DUcJlBxiE,7326
-tinybird/tb/modules/cli.py,sha256=QpEGFMtmwqnJRaK1KAtKkLsKHeJ39VHU5wFkDJJiCnk,16748
+tinybird/tb/modules/cli.py,sha256=-hkwg2ZIWPtldkLkgdY7RYWacOJtkMx58eqqYzMoMfw,17793
 tinybird/tb/modules/common.py,sha256=zY0PFDQVWjhu2MivBtUmcXtAlv7imgO-RG_PUK37XvI,83659
 tinybird/tb/modules/config.py,sha256=gK7rgaWTDd4ZKCrNEg_Uemr26EQjqWt6TjyQKujxOws,11462
 tinybird/tb/modules/connection.py,sha256=axp8Fny1_4PSLJGN4UF6WygyRbQtM3Lbt6thxHKTxzw,17790
@@ -34,7 +34,7 @@ tinybird/tb/modules/copy.py,sha256=dPZkcIDvxjJrlQUIvToO0vsEEEs4EYumbNV77-BzNoU,4
 tinybird/tb/modules/create.py,sha256=pJxHXG69c9Z_21s-7VuJ3RZOF_nJU51LEwiAkvI3dZY,23251
 tinybird/tb/modules/datasource.py,sha256=kDFHdxckTnRosk2829icfltQvlJd8EY5c9oWB5eS5Xo,41797
 tinybird/tb/modules/deployment.py,sha256=v0layOmG0IMnuXc3RT39mpGfa5M8yPlrL9F089fJFCo,15964
-tinybird/tb/modules/deployment_common.py,sha256=8Cc0VyKthmTnULiTKgciPyOGtf1kaRghC3Q00bZJbD4,19897
+tinybird/tb/modules/deployment_common.py,sha256=ZX27oUw7u1ld20qiXH69j7m0DImOc5nckABZ37BPAn4,20406
 tinybird/tb/modules/deprecations.py,sha256=rrszC1f_JJeJ8mUxGoCxckQTJFBCR8wREf4XXXN-PRc,4507
 tinybird/tb/modules/dev_server.py,sha256=57FCKuWpErwYUYgHspYDkLWEm9F4pbvVOtMrFXX1fVU,10129
 tinybird/tb/modules/endpoint.py,sha256=ksRj6mfDb9Xv63PhTkV_uKSosgysHElqagg3RTt21Do,11958
@@ -55,7 +55,7 @@ tinybird/tb/modules/mock.py,sha256=ET8sRpmXnQsd2sSJXH_KCdREU1_XQgkORru6T357Akc,3
 tinybird/tb/modules/mock_common.py,sha256=72yKp--Zo40hrycUtiajSRW2BojOsgOZFqUorQ_KQ3w,2279
 tinybird/tb/modules/open.py,sha256=LYiuO8Z1I9O_v6pv58qpUCWFD6BT00BdeO21fRa4I4Y,1315
 tinybird/tb/modules/pipe.py,sha256=xPKtezhnWZ6k_g82r4XpgKslofhuIxb_PvynH4gdUzI,2393
-tinybird/tb/modules/project.py,sha256=_P6wUOFTY87ixE06SjfYIROMoVr6oipi6qHu-RAHLD8,7356
+tinybird/tb/modules/project.py,sha256=tP6XuG8dR8QCA4pojlXZ7Am6EfS2h3Xnzw5lA3sdtlY,7747
 tinybird/tb/modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb/modules/secret.py,sha256=9BIdh2PZDAbY2wRbf4ZDvkEltygztz1RMxgDmY1D0LI,3521
 tinybird/tb/modules/secret_common.py,sha256=8mEXOdNFrTuVWu7rgZ5-C8v-rB1EAdZQUHIyBIgwISI,1578
@@ -70,12 +70,13 @@ tinybird/tb/modules/watch.py,sha256=No0bK1M1_3CYuMaIgylxf7vYFJ72lTJe3brz6xQ-mJo,
 tinybird/tb/modules/workspace.py,sha256=tCP1zZMwBhLRGm22TGfpSd4cHvQLAS1o_azIXv_r6uw,11172
 tinybird/tb/modules/workspace_members.py,sha256=5JdkJgfuEwbq-t6vxkBhYwgsiTDxF790wsa6Xfif9nk,8608
 tinybird/tb/modules/agent/__init__.py,sha256=i3oe3vDIWWPaicdCM0zs7D7BJ1W0k7th93ooskHAV00,54
-tinybird/tb/modules/agent/agent.py,sha256=wxLG84pYfNdQPB1TI6Nkv_8Y_bO5lmOX3ht87gnmrwY,36315
+tinybird/tb/modules/agent/agent.py,sha256=uVZ0yGOk4uRtF74mPHIYjawIDPSCweb5HCC7eLVuf2k,37339
 tinybird/tb/modules/agent/animations.py,sha256=4WOC5_2BracttmMCrV0H91tXfWcUzQHBUaIJc5FA7tE,3490
 tinybird/tb/modules/agent/banner.py,sha256=l6cO5Fi7lbVKp-GsBP8jf3IkjOWxg2jpAt9NBCy0WR8,4085
 tinybird/tb/modules/agent/command_agent.py,sha256=0Z08rQsir59zQAr-kkOvsKIFpIBsBSTGJJ1VgqqF5WA,3654
 tinybird/tb/modules/agent/compactor.py,sha256=BK5AxZFhrp3xWnsRnYaleiYoIWtVNc-_m650Hsopt8g,13841
 tinybird/tb/modules/agent/explore_agent.py,sha256=gyD5uV5TJwV24eeQiSwhkgfNPb4mtbeH7t2qSdoc18U,4100
+tinybird/tb/modules/agent/file_agent.py,sha256=vE8VChVbG_-uyeBtp3I6QrXeNaPaVlVoNn4c9cGTAJA,2466
 tinybird/tb/modules/agent/memory.py,sha256=vBewB_64L_wHoT4tLT6UX2uxcHwSY880QZ26F9rPqXs,3793
 tinybird/tb/modules/agent/mock_agent.py,sha256=zbAZfAqdSLUtMr2VqO0erWpzjT2F1tTcuYjvHb-gvbA,8023
 tinybird/tb/modules/agent/models.py,sha256=eokO8XlY-kVJOsbqiVporGUAOCyKAXCO5xgTEK9SM6Y,2208
@@ -84,13 +85,14 @@ tinybird/tb/modules/agent/testing_agent.py,sha256=AtwtJViH7805i7djyBgDb7SSUtDyJn
 tinybird/tb/modules/agent/utils.py,sha256=alvRNtpPPRZP0sNdOXbzX3uQZHqqeh1daIAg3aquq24,32015
 tinybird/tb/modules/agent/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tinybird/tb/modules/agent/tools/analyze.py,sha256=CR5LXg4fou-zYEksqnjpJ0icvxJVoKnTctoI1NRvqCM,3873
-tinybird/tb/modules/agent/tools/append.py,sha256=8UsGMzmv7GYbzp0gerOBqpxxocDbous_haSuwS3zuGU,8222
+tinybird/tb/modules/agent/tools/append.py,sha256=cF-1XYmgTXI3dfMQ_nEpOKo6duJGe2GOg64rXNnS9gw,8284
 tinybird/tb/modules/agent/tools/build.py,sha256=Hm-xDAP9ckMiKquT-DmDg5H0yxZefLOaWKANyoVSaEQ,846
 tinybird/tb/modules/agent/tools/datafile.py,sha256=kTob7G2TwCwIgwom0rERgXQ13rgPtZv3_ByLnrvpIdU,10881
 tinybird/tb/modules/agent/tools/deploy.py,sha256=6Vmm0lCG8XKE2iUF_ZJrOqXbTFhoe3anPzYCFehQ3_E,2027
 tinybird/tb/modules/agent/tools/deploy_check.py,sha256=pE3d9TPtXVKZjYbU0G6ORAGI86lN5K_4JKUriClERbM,1229
 tinybird/tb/modules/agent/tools/diff_resource.py,sha256=_9xHcDzCTKk_E1wKQbuktVqV6U9sA0kqYaBxWvtliX0,2613
 tinybird/tb/modules/agent/tools/execute_query.py,sha256=DL2jsZ0jaEqFIkGoiWfR-IUAwsgoF0D-_JUhq7xe4gA,9145
+tinybird/tb/modules/agent/tools/file.py,sha256=_strhF1bnFFui3pVGUi2dFdcKzMyYzOD59lKeSHqG7o,2855
 tinybird/tb/modules/agent/tools/get_endpoint_stats.py,sha256=r2FrXg1L1s_Llr1tPdJ6k_gu6qw7qLsAXOkbz3eTk1g,2307
 tinybird/tb/modules/agent/tools/get_openapi_definition.py,sha256=4TIMO2XzHBMhpt9zIWRfjjPZbThT8r_iPS4CVHcItE0,2904
 tinybird/tb/modules/agent/tools/mock.py,sha256=Seo4WcYNLL1-SmPXutoaX94_pfOdIb47JXo8dHtUVhg,7106
@@ -103,14 +105,14 @@ tinybird/tb/modules/datafile/build.py,sha256=NFKBrusFLU0WJNCXePAFWiEDuTaXpwc0lHl
 tinybird/tb/modules/datafile/build_common.py,sha256=2yNdxe49IMA9wNvl25NemY2Iaz8L66snjOdT64dm1is,4511
 tinybird/tb/modules/datafile/build_datasource.py,sha256=Ra8pVQBDafbFRUKlhpgohhTsRyp_ADKZJVG8Gd69idY,17227
 tinybird/tb/modules/datafile/build_pipe.py,sha256=-mpzYcTEdH313R-FEDYuDOtAbZ5zehynzJwmtsfy16w,11262
-tinybird/tb/modules/datafile/diff.py,sha256=vCwBxUtQskmZSu-UxscqdgFkEJ9i3t1U1F6RrX9wRrA,6674
+tinybird/tb/modules/datafile/diff.py,sha256=RxkBkuSAPUZF7slWe0yaaYLea4ZDZ-qfEscVAT9xvWU,6716
 tinybird/tb/modules/datafile/fixture.py,sha256=gftYWeweeQDFK3cHgUmSOfltNjZVOuMt8v7WTMLhGBw,1579
 tinybird/tb/modules/datafile/format_common.py,sha256=5w6GX_8RDFueW9AtDfWpc1OicEu8wWdrjUBdCEidCQk,1951
 tinybird/tb/modules/datafile/format_datasource.py,sha256=X5N65vDpprFKaWaxkSwFnHaNzYIxVhG1Sya4BARTd2Q,6138
 tinybird/tb/modules/datafile/format_pipe.py,sha256=_64RGD8XI-gpQBMzhymUAVGMhpQK6zar_irDrJFQrHY,7378
 tinybird/tb/modules/datafile/pipe_checker.py,sha256=dxsCQoA6ruxg1fvF6sMLFowpjaqws8lUQcM7XyhgZPE,24609
 tinybird/tb/modules/datafile/playground.py,sha256=47q4XcvDhclHWYv7oFNEqmF1T5NXd9GPWkrT6Av2VhY,56343
-tinybird/tb/modules/datafile/pull.py,sha256=BB-JSAPDRL8WotaSo73PHBDdAKBx4ZM-YeZCVcXdRTQ,4311
+tinybird/tb/modules/datafile/pull.py,sha256=6mn3xqLxb6KQDcnqq63AaPzku9XsSOpscfcdsHyvvsY,5101
 tinybird/tb/modules/tinyunit/tinyunit.py,sha256=sJeBnKDHcUNn8t0sdsKB_M2w8IdzeeQ1PzJuleeQ2-E,11695
 tinybird/tb/modules/tinyunit/tinyunit_lib.py,sha256=hGh1ZaXC1af7rKnX7222urkj0QJMhMWclqMy59dOqwE,1922
 tinybird/tb_cli_modules/cicd.py,sha256=0lMkb6CVOFZl5HOwgY8mK4T4mgI7O8335UngLXtCc-c,13851
@@ -119,8 +121,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev289.dist-info/METADATA,sha256=ENjdQSJQH3QgYbGGAJddSYazWCBQ-ZJbpW6SyEqFGKE,1845
-tinybird-0.0.1.dev289.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev289.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev289.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev289.dist-info/RECORD,,
+tinybird-0.0.1.dev291.dist-info/METADATA,sha256=WCSw6JMv71AZt3gonZC3XeWThiDtGWEynv8GH1jSKhE,1845
+tinybird-0.0.1.dev291.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev291.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev291.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev291.dist-info/RECORD,,