tinybird 0.0.1.dev290__py3-none-any.whl → 0.0.1.dev292__py3-none-any.whl

tinybird/context.py

@@ -3,10 +3,11 @@ from typing import TYPE_CHECKING
 
 # Avoid circular import error
 if TYPE_CHECKING:
-    from tinybird.user import User
+    from hfi.hfi_workspace_data import HfiWorkspaceData
+
 
 workspace_id: ContextVar[str] = ContextVar("workspace_id")
-workspace: ContextVar["User"] = ContextVar("workspace")
+hfi_workspace_data: ContextVar["HfiWorkspaceData"] = ContextVar("hfi_workspace_data")
 table_id: ContextVar[str] = ContextVar("table_id")
 hfi_frequency: ContextVar[float] = ContextVar("hfi_frequency")
 hfi_frequency_gatherer: ContextVar[float] = ContextVar("hfi_frequency_gatherer")

tinybird/datafile/common.py

@@ -1870,7 +1870,7 @@ def parse(
 
         for entries in args:
             # In case they specify multiple workspaces, handle both line-separated and comma-separated values
-            lines = entries.splitlines()
+            lines = _unquote(entries).splitlines()
             for line in lines:
                 # Split by comma and strip whitespace from each workspace name
                 workspaces = [workspace.strip().rstrip(",") for workspace in line.split(",") if workspace.strip()]
@@ -1957,7 +1957,7 @@ def parse(
             "import_query": assign_var("import_query"),  # Deprecated, BQ and SFK
             "import_table_arn": assign_var("import_table_arn"),  # Only for DynamoDB
             "import_export_bucket": assign_var("import_export_bucket"),  # For DynamoDB
-            "shared_with": shared_with,  # Not supported yet
+            "shared_with": shared_with,
             "export_service": export_service,  # Deprecated
             "forward_query": sql("forward_query"),
             "backfill": assign_var("backfill", allowed_values={"skip"}),

tinybird/sql.py

@@ -7,6 +7,9 @@ from typing import Any, Dict, Iterable, List, Optional
 
 valid_chars_name: str = string.ascii_letters + string.digits + "._`*<>+-'"
 valid_chars_fn: str = valid_chars_name + "[](),=!?:/ \n\t\r"
+# Use sets for O(1) membership checks in hot loops
+_VALID_CHARS_NAME_SET = set(valid_chars_name)
+_VALID_CHARS_FN_SET = set(valid_chars_fn)
 
 INDEX_WHITELIST = ["minmax", "set", "bloom_filter", "ngrambf_v1", "tokenbf_v1"]
 INDEX_SUPPORTED_TYPES = {
@@ -28,6 +31,15 @@ INDEX_SUPPORTED_TYPES = {
     "ngrambf_v1": ["String", "FixedString", "Map"],
 }
 
+# Precompiled regexes used across functions to avoid recompilation overhead
+_RE_FORMAT = re.compile(r"\s+format\s+(\w+)\s*$", re.IGNORECASE)
+_RE_REMOVE_FORMAT = re.compile(r"\s+(format)\s+(\w+)\s*$", re.IGNORECASE)
+_RE_TRY_FIX_NULLABLE_SAF = re.compile(r"SimpleAggregateFunction\((\w+),\s*(?!(?:Nullable))([\w,.()]+)\)")
+_RE_INDEX_ENTRY = re.compile(
+    r"(\w+)\s+([\w\s*\[\]\*\(\),\'\"-><.]+)\s+TYPE\s+(\w+)(?:\(([\w\s*.,]+)\))?(?:\s+GRANULARITY\s+(\d+))?"
+)
+_RE_REPLICATED_MT = re.compile(r"Replicated(.*)MergeTree\(([^\)]*)\)(.*)")
+
 
 @dataclass
 class TableIndex:
@@ -137,10 +149,9 @@ def get_format(sql: str) -> Optional[str]:
     >>> get_format('select * from test formAt JSON')
     'JSON'
     """
-    FORMAT_RE = r"\s+format\s+(\w+)\s*$"
     sql = sql.strip()
-    format = re.findall(FORMAT_RE, sql, re.IGNORECASE)
-    return format[0] if format else None
+    match = _RE_FORMAT.findall(sql)
+    return match[0] if match else None
 
 
 def get_format_group(sql: str) -> str:
@@ -151,10 +162,9 @@ def get_format_group(sql: str) -> str:
     >>> get_format_group('select * from test formAt JSON')
     ' formAt JSON'
     """
-    FORMAT_RE = r"\s+format\s+(\w+)\s*$"
     sql = sql.strip()
-    format = re.search(FORMAT_RE, sql, re.IGNORECASE)
-    return format.group() if format else ""
+    match = _RE_FORMAT.search(sql)
+    return match.group() if match else ""
 
 
 def wrap_finalize_aggregation(sql: str, describe_result: Dict[str, Any], fm_group: Optional[str] = None) -> str:
@@ -186,9 +196,8 @@ def remove_format(sql: str) -> str:
     >>> remove_format('select * from test formAt JSON')
     'select * from test'
     """
-    FORMAT_RE = r"\s+(format)\s+(\w+)\s*$"
     sql = sql.strip()
-    return re.sub(FORMAT_RE, "", sql, flags=re.IGNORECASE)
+    return _RE_REMOVE_FORMAT.sub("", sql)
 
 
 def col_name(name: str, backquotes: bool = True) -> str:
@@ -220,7 +229,7 @@ def try_to_fix_nullable_in_simple_aggregating_function(t: str) -> Optional[str]:
     # as it is done with other aggregate functions.
     # If not, the aggregation could return incorrect results.
     result = None
-    if match := re.search(r"SimpleAggregateFunction\((\w+),\s*(?!(?:Nullable))([\w,.()]+)\)", t):
+    if match := _RE_TRY_FIX_NULLABLE_SAF.search(t):
         fn = match.group(1)
         inner_type = match.group(2)
         result = f"SimpleAggregateFunction({fn}, Nullable({inner_type}))"
@@ -342,10 +351,7 @@ def parse_indexes_structure(indexes: Optional[List[str]]) -> List[TableIndex]:
         if index.count("TYPE") != 1:
             raise ValueError("invalid INDEX format. Usage: `name expr TYPE type_full GRANULARITY granularity`")
 
-        match = re.match(
-            r"(\w+)\s+([\w\s*\[\]\*\(\),\'\"-><.]+)\s+TYPE\s+(\w+)(?:\(([\w\s*.,]+)\))?(?:\s+GRANULARITY\s+(\d+))?",
-            index,
-        )
+        match = _RE_INDEX_ENTRY.match(index)
         if match:
             index_name, a, index_type, value, granularity = match.groups()
             index_expr = f"{index_type}({value})" if value else index_type
@@ -552,7 +558,7 @@ def clean_comments(schema_to_clean: str) -> str:
             if i + 1 < len(line) and line[i] == "-" and line[i + 1] == "-" and not inside_json_path:
                 return line[:i].strip()
 
-            if not inside_json_path and line[i:].startswith("`json:"):
+            if not inside_json_path and line.startswith("`json:", i):
                 inside_json_path = True
             elif inside_json_path and line[i] == "`":
                 inside_json_path = False
@@ -562,12 +568,12 @@ def clean_comments(schema_to_clean: str) -> str:
     if schema_to_clean is None:
         return schema_to_clean
 
-    cleaned_schema = ""
+    out_lines: List[str] = []
     for line in schema_to_clean.splitlines():
         cleaned_line = clean_line_comments(line)
         if cleaned_line:
-            cleaned_schema += cleaned_line + "\n"
-    return cleaned_schema.strip()
+            out_lines.append(cleaned_line)
+    return "\n".join(out_lines).strip()
 
 
 SyntaxExpr = namedtuple("SyntaxExpr", ["name", "regex"])
@@ -608,9 +614,9 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
 
     # Find the first SyntaxExpr in lookup that matches the schema at the current offset
     def lookahead_matches(lookup: Iterable) -> Optional[SyntaxExpr]:
+        # Use substring to preserve semantics of patterns anchored to end ($)
         s = schema[i:]
-        match = next((x for x in lookup if x.regex.match(s)), None)
-        return match
+        return next((x for x in lookup if x.regex.match(s)), None)
 
     def advance_single_char() -> None:
         nonlocal i, line, pos
@@ -647,7 +653,7 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
                 c = schema[i]
                 if c in " \t\r\n":
                     return schema[begin:i]
-                if c not in valid_chars_name:
+                if c not in _VALID_CHARS_NAME_SET:
                     raise ValueError(
                         format_parse_error(schema, i, pos, "wrong value, please check the schema syntax", line=line)
                     )
@@ -677,7 +683,9 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
                 context_stack.append("(")
             elif context is None and lookahead_matches(lookup):
                 return schema[begin:i].strip(" \t\r\n")
-            elif (context is None and c not in valid_chars_fn) or (context == "(" and c not in valid_chars_fn):
+            elif (context is None and c not in _VALID_CHARS_FN_SET) or (
+                context == "(" and c not in _VALID_CHARS_FN_SET
+            ):
                 raise ValueError(
                     format_parse_error(schema, i, pos, "wrong value, please check the schema syntax", line=line)
                 )
@@ -884,7 +892,7 @@ def engine_replicated_to_local(engine: str) -> str:
     if "Replicated" not in engine:
         return engine
 
-    return re.sub(r"Replicated(.*)MergeTree\(([^\)]*)\)(.*)", _replace, engine.strip())
+    return _RE_REPLICATED_MT.sub(_replace, engine.strip())
 
 
 def engine_patch_replicated_engine(engine: str, engine_full: Optional[str], new_table_name: str) -> Optional[str]:

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev290'
-__revision__ = '4d1cca4'
+__version__ = '0.0.1.dev292'
+__revision__ = '78dba1d'

tinybird/tb/modules/agent/agent.py

@@ -34,10 +34,12 @@ from tinybird.tb.modules.agent.mock_agent import MockAgent
 from tinybird.tb.modules.agent.models import create_model
 from tinybird.tb.modules.agent.prompts import (
     agent_system_prompt,
+    fixtures_prompt,
     load_custom_project_rules,
     resources_prompt,
     secrets_prompt,
     service_datasources_prompt,
+    vendor_files_prompt,
 )
 from tinybird.tb.modules.agent.testing_agent import TestingAgent
 from tinybird.tb.modules.agent.tools.analyze import analyze_file, analyze_url
@@ -322,6 +324,14 @@ class TinybirdAgent:
         def get_project_files(ctx: RunContext[TinybirdAgentContext]) -> str:
             return resources_prompt(self.project)
 
+        @self.agent.instructions
+        def get_vendor_files(ctx: RunContext[TinybirdAgentContext]) -> str:
+            return vendor_files_prompt(self.project)
+
+        @self.agent.instructions
+        def get_fixture_files(ctx: RunContext[TinybirdAgentContext]) -> str:
+            return fixtures_prompt(self.project)
+
         @self.agent.instructions
         def get_service_datasources(ctx: RunContext[TinybirdAgentContext]) -> str:
             return service_datasources_prompt()

tinybird/tb/modules/agent/explore_agent.py

@@ -12,6 +12,7 @@ from tinybird.tb.modules.agent.prompts import (
     resources_prompt,
     service_datasources_prompt,
     tone_and_style_instructions,
+    vendor_files_prompt,
 )
 from tinybird.tb.modules.agent.tools.diff_resource import diff_resource
 from tinybird.tb.modules.agent.tools.execute_query import execute_query
@@ -76,6 +77,10 @@ Once you finish the task, return a valid response for the task to complete.
         def get_project_files(ctx: RunContext[TinybirdAgentContext]) -> str:
             return resources_prompt(self.project)
 
+        @self.agent.instructions
+        def get_vendor_files(ctx: RunContext[TinybirdAgentContext]) -> str:
+            return vendor_files_prompt(self.project)
+
         @self.agent.instructions
         def get_service_datasources(ctx: RunContext[TinybirdAgentContext]) -> str:
             return service_datasources_prompt()

tinybird/tb/modules/agent/mock_agent.py

@@ -6,7 +6,7 @@ from pydantic_ai.usage import Usage
 
 from tinybird.tb.modules.agent.animations import ThinkingAnimation
 from tinybird.tb.modules.agent.models import create_model
-from tinybird.tb.modules.agent.prompts import resources_prompt
+from tinybird.tb.modules.agent.prompts import fixtures_prompt, resources_prompt
 from tinybird.tb.modules.agent.tools.mock import generate_mock_fixture
 from tinybird.tb.modules.agent.utils import TinybirdAgentContext
 from tinybird.tb.modules.project import Project
@@ -194,6 +194,10 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
         def get_project_files(ctx: RunContext[TinybirdAgentContext]) -> str:
             return resources_prompt(self.project)
 
+        @self.agent.instructions
+        def get_fixture_files(ctx: RunContext[TinybirdAgentContext]) -> str:
+            return fixtures_prompt(self.project)
+
     def run(self, task: str, deps: TinybirdAgentContext, usage: Usage):
         result = self.agent.run_sync(
             task,

tinybird/tb/modules/agent/prompts.py

@@ -35,8 +35,10 @@ available_commands = [
     "`tb sink ls`: List all sinks",
     "`tb workspace current`: Show the current workspace",
     "`tb workspace clear --yes`: Delete all resources in the workspace (Only available in Tinybird Local)",
+    "`tb workspace ls`: List all workspaces",
     "`tb local start --skip-new-version`: Start Tinybird Local container in non-interactive mode",
     "`tb local restart --skip-new-version --yes`: Restart Tinybird Local container in non-interactive mode",
+    "`tb pull --only-vendored`: Pull only the vendored datasources from other workspaces",
 ]
 
 plan_instructions = """
@@ -166,7 +168,6 @@ SQL >
 
 def resources_prompt(project: Project) -> str:
     files = project.get_project_files()
-    fixture_files = project.get_fixture_files()
 
     resources_content = "# Existing resources in the project:\n"
     if files:
@@ -184,7 +185,36 @@ def resources_prompt(project: Project) -> str:
     else:
         resources_content += "No resources found"
 
+    return resources_content
+
+
+def vendor_files_prompt(project: Project) -> str:
+    files = project.get_vendored_files()
+    content = "# Datasources shared from other workspaces:\n"
+    if files:
+        resources: list[dict[str, Any]] = []
+        for filename in files:
+            file_path = Path(filename)
+            workspace_name = file_path.parent.parent.name
+            resource = {
+                "path": str(file_path.relative_to(project.folder)),
+                "type": get_resource_type(file_path),
+                "name": f"{workspace_name}.{file_path.stem}",
+                "content": file_path.read_text(),
+                "origin_workspace": workspace_name,
+            }
+            resources.append(resource)
+        content += format_as_xml(resources, root_tag="resources", item_tag="resource")
+    else:
+        content += "No datasources shared from other workspaces"
+
+    return content
+
+
+def fixtures_prompt(project: Project) -> str:
+    fixture_files = project.get_fixture_files()
     fixture_content = "# Fixture files in the project:\n"
+
     if fixture_files:
         fixtures: list[dict[str, Any]] = []
         for filename in fixture_files:
@@ -199,7 +229,7 @@ def resources_prompt(project: Project) -> str:
     else:
         fixture_content += "No fixture files found"
 
-    return resources_content + "\n" + fixture_content
+    return fixture_content
 
 
 def service_datasources_prompt() -> str:
@@ -954,6 +984,23 @@ After changes have been deployed and promoted, if you want to deploy other chang
 If after running a deployment, the error contains a recommended forward query, use it to update the .datasource file.
 </dev_notes>
 
+## Sharing datasources with other workspaces
+To share a Data Source, in the .datasource file you want to share, add the destination workspace(s). For example:
+
+```
+SHARED_WITH >
+    destination_workspace,
+    other_destination_workspace
+```
+
+## Working with shared datasources:
+
+The following limitations apply to shared datasources:
+- Shared datasources are read-only.
+- You can't share a shared datasource, only the original.
+- You can't check the quarantine of a shared datasource.
+- You can't create a Materialized View from a shared datasource.
+
 # Working with any type of pipe file:
 {pipe_instructions}
 {pipe_example}

tinybird/tb/modules/agent/tools/run_command.py

@@ -3,6 +3,7 @@ import subprocess
 import click
 from pydantic_ai import RunContext
 
+from tinybird.tb.modules.agent.prompts import available_commands
 from tinybird.tb.modules.agent.utils import (
     AgentRunCancelled,
     SubAgentRunCancelled,
@@ -20,6 +21,14 @@ def run_command(ctx: RunContext[TinybirdAgentContext], command: str):
         command (str): The command to run. Required. Examples: `tb --local sql "select 1"`, `tb --cloud datasource ls`, `tb --help`
     """
     try:
+        clean_commands = [cmd.split(":")[0].replace("`", "").split("[")[0] for cmd in available_commands]
+        available_commands_to_cloud = [f"tb --cloud {command.replace('tb ', '')}" for command in clean_commands]
+        available_commands_to_local = [f"tb --local {command.replace('tb ', '')}" for command in clean_commands]
+        all_commands = clean_commands + available_commands_to_cloud + available_commands_to_local
+
+        if not any(cmd.startswith(command) for cmd in all_commands):
+            raise SubAgentRunCancelled(f"Command {command} not found in the list of available commands")
+
         ctx.deps.thinking_animation.stop()
         force_confirmation = " deploy" in command.lower() or " truncate" in command.lower()
         confirmation = show_confirmation(

tinybird/tb/modules/cli.py

@@ -212,16 +212,39 @@ def cli(
 
 
 @cli.command(hidden=True)
+@click.option("--only-vendored", is_flag=True, default=False, help="Only update vendored files")
 @click.option("-f", "--force", is_flag=True, default=False, help="Override existing files")
 @click.option("--fmt", is_flag=True, default=False, help="Format files before saving")
 @click.pass_context
-def pull(ctx: Context, force: bool, fmt: bool) -> None:
+def pull(ctx: Context, only_vendored: bool, force: bool, fmt: bool) -> None:
     """Retrieve latest versions for project files from Tinybird."""
 
     client = ctx.ensure_object(dict)["client"]
     project = ctx.ensure_object(dict)["project"]
 
-    return folder_pull(client, project.path, force, fmt=fmt)
+    if only_vendored:
+        force = True
+
+    written_files = folder_pull(client, project.path, force, only_vendored=only_vendored, fmt=fmt)
+
+    if only_vendored:
+        for_user_to_delete = set(project.get_vendored_files()) - set(written_files)
+        if for_user_to_delete:
+            # TODO(eclbg): this prints the full path of the files. Let's print the relative path from the project root
+            display_paths = []
+            for full_path in for_user_to_delete:
+                try:
+                    display_paths.append(str(Path(full_path).relative_to(project.path)))
+                except:
+                    display_paths.append(full_path)
+            click.echo(
+                FeedbackManager.warning(
+                    message=(
+                        f"This workspace no longer has access to the following files: {display_paths}. "
+                        "Please remove them manually to be able to deploy."
+                    )
+                )
+            )
 
 
 @cli.command()
@@ -392,7 +415,7 @@ def create_ctx_client(
     if not command or command in commands_without_ctx_client:
         return None
 
-    commands_always_cloud = ["pull", "infra"]
+    commands_always_cloud = ["infra"]
     commands_always_local = ["build", "dev"]
     command_always_test = ["test"]
 

tinybird/tb/modules/datafile/diff.py

@@ -121,12 +121,12 @@ def diff_command(
     if filenames:
         if len(filenames) == 1:
             filenames = [filenames[0], *get_project_filenames(filenames[0])]
-        folder_pull(client, target_dir, True, verbose=False)
+        folder_pull(client, target_dir, True, only_vendored=False, verbose=False)
     else:
         filenames = get_project_filenames(".")
         if verbose:
             click.echo("Saving remote resources in .diff_tmp folder.\n")
-        folder_pull(client, target_dir, True, verbose=verbose, progress_bar=progress_bar)
+        folder_pull(client, target_dir, True, verbose=verbose, only_vendored=False, progress_bar=progress_bar)
 
     remote_datasources: List[Dict[str, Any]] = client.datasources()
     remote_pipes: List[Dict[str, Any]] = client.pipes()

tinybird/tb/modules/datafile/pull.py

@@ -13,10 +13,11 @@ def folder_pull(
     client: TinyB,
     folder: str,
     force: bool,
+    only_vendored: bool,
     verbose: bool = True,
     progress_bar: bool = False,
     fmt: bool = False,
-):
+) -> list[str]:
     def get_file_folder(extension: str, resource_type: Optional[str]):
         if extension == "datasource":
             return "datasources"
@@ -40,8 +41,8 @@ def folder_pull(
         get_resource_function: str,
         progress_bar: bool = False,
         fmt: bool = False,
-    ):
-        def write_resource(k: dict[str, Any]):
+    ) -> list[Path]:
+        def write_resource(k: dict[str, Any]) -> Optional[Path]:
             name = f"{k['name']}.{extension}"
             try:
                 resource = getattr(client, get_resource_function)(k["name"])
@@ -72,59 +73,71 @@ def folder_pull(
                     with open(f, "w") as fd:
                         if resource_to_write:
                             fd.write(resource_to_write)
+                            return f
                 else:
                     if verbose:
                         click.echo(FeedbackManager.info_skip_already_exists())
+                return None
             except Exception as e:
                 raise click.ClickException(FeedbackManager.error_exception(error=e))
 
         if progress_bar:
+            written_files = []
             with click.progressbar(resources, label=f"Pulling {extension}s") as resources:  # type: ignore
                 for k in resources:
-                    write_resource(k)
+                    written_file = write_resource(k)
+                    if written_file:
+                        written_files.append(written_file)
         else:
-            tasks = [write_resource(k) for k in resources]
-            _gather_with_concurrency(5, *tasks)
+            written_files = list(filter(None, (write_resource(k) for k in resources)))
+
+        return written_files
 
     try:
         datasources = client.datasources()
         pipes = client.pipes()
         connections = client.connections()
 
-        write_files(
-            resources=datasources,
-            extension="datasource",
-            get_resource_function="datasource_file",
-            progress_bar=progress_bar,
-            fmt=fmt,
-        )
-        write_files(
-            resources=pipes,
-            extension="pipe",
-            get_resource_function="pipe_file",
-            progress_bar=progress_bar,
-            fmt=fmt,
-        )
-        write_files(
-            resources=connections,
-            extension="connection",
-            get_resource_function="connection_file",
-            progress_bar=progress_bar,
-            fmt=fmt,
-        )
-        return
+        if only_vendored:
+            written_files = write_files(
+                resources=[ds for ds in datasources if "shared_from" in ds],
+                extension="datasource",
+                get_resource_function="datasource_file",
+                progress_bar=progress_bar,
+                fmt=fmt,
+            )
+        else:
+            written_files = []
+            written_files.extend(
+                write_files(
+                    resources=datasources,
+                    extension="datasource",
+                    get_resource_function="datasource_file",
+                    progress_bar=progress_bar,
+                    fmt=fmt,
+                )
+            )
+            written_files.extend(
+                write_files(
+                    resources=pipes,
+                    extension="pipe",
+                    get_resource_function="pipe_file",
+                    progress_bar=progress_bar,
+                    fmt=fmt,
+                )
+            )
+            written_files.extend(
+                write_files(
+                    resources=connections,
+                    extension="connection",
+                    get_resource_function="connection_file",
+                    progress_bar=progress_bar,
+                    fmt=fmt,
+                )
+            )
+        return [str(wf) for wf in written_files]
 
     except AuthNoTokenException:
         raise
     except Exception as e:
         raise click.ClickException(FeedbackManager.error_pull(error=str(e)))
-
-
-def _gather_with_concurrency(n, *tasks):
-    results = []
-    for task in tasks:
-        if callable(task):
-            results.append(task())
-        else:
-            results.append(task)
-    return results

tinybird/tb/modules/deployment_common.py

@@ -222,6 +222,7 @@ def create_deployment(
     # TODO: This code is duplicated in build_server.py
     # Should be refactored to be shared
     MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
+    MULTIPART_BOUNDARY_DATA_PROJECT_VENDORED = "data_project_vendored://"
     DATAFILE_TYPE_TO_CONTENT_TYPE = {
         ".datasource": "text/plain",
         ".pipe": "text/plain",
@@ -248,6 +249,13 @@ def create_deployment(
         with open(file_path, "rb") as fd:
             content_type = DATAFILE_TYPE_TO_CONTENT_TYPE.get(Path(file_path).suffix, "application/unknown")
             files.append((MULTIPART_BOUNDARY_DATA_PROJECT, (relative_path, fd.read().decode("utf-8"), content_type)))
+    for file_path in project.get_vendored_files():
+        relative_path = Path(file_path).relative_to(project.path).as_posix()
+        with open(file_path, "rb") as fd:
+            content_type = DATAFILE_TYPE_TO_CONTENT_TYPE.get(Path(file_path).suffix, "application/unknown")
+            files.append(
+                (MULTIPART_BOUNDARY_DATA_PROJECT_VENDORED, (relative_path, fd.read().decode("utf-8"), content_type))
+            )
 
     deployment = None
     try:

tinybird/tb/modules/project.py

@@ -37,11 +37,13 @@ class Project:
     def has_deeper_level(self) -> bool:
         """Check if there are folders with depth greater than max_depth in project path.
 
+        Does not consider the vendor directory.
+
         Returns:
             bool: True if there are folders deeper than max_depth, False otherwise
         """
         for obj in glob.glob(f"{self.path}{'/*' * (self.max_depth - 1)}/*", recursive=False):
-            if Path(obj).is_dir():
+            if Path(obj).is_dir() and self.vendor_path not in obj:
                 return True
         return False
 
@@ -54,6 +56,13 @@ class Project:
                 project_files.append(project_file)
         return project_files
 
+    def get_vendored_files(self) -> List[str]:
+        vendored_files: List[str] = []
+        for extension in self.extensions:
+            for level in range(3):
+                vendored_files.extend(glob.glob(f"{self.vendor_path}{'/*' * level}/*.{extension}", recursive=True))
+        return vendored_files
+
     def get_fixture_files(self) -> List[str]:
         fixture_files: List[str] = []
         for extension in [

{tinybird-0.0.1.dev290.dist-info → tinybird-0.0.1.dev292.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev290
+Version: 0.0.1.dev292
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev290.dist-info → tinybird-0.0.1.dev292.dist-info}/RECORD

@@ -1,11 +1,11 @@
 tinybird/connectors.py,sha256=7Gjms7b5MAaBFGi3xytsJurCylprONpFcYrzp4Fw2Rc,15241
-tinybird/context.py,sha256=FfqYfrGX_I7PKGTQo93utaKPDNVYWelg4Hsp3evX5wM,1291
+tinybird/context.py,sha256=M2jhaNBDOFLlSU2ONDPwO7Coknh2z1PNY-TlNJAkEK0,1343
 tinybird/datatypes.py,sha256=r4WCvspmrXTJHiPjjyOTiZyZl31FO3Ynkwq4LQsYm6E,11059
 tinybird/feedback_manager.py,sha256=XY8d83pRlq-LH7xHMApkaEebfXEWLjDzrGe1prpcTHE,69778
 tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
 tinybird/prompts.py,sha256=HoDv9TxPiP8v2XoGTWYxP133dK9CEbXVv4XE5IT339c,45483
 tinybird/service_datasources.py,sha256=o6Az3T6OvcChR_7GXRu7sJH173KzGg1Gv-dNd0bI_vY,46085
-tinybird/sql.py,sha256=UZJLop6zA9tTPEaS-Fq7M-QyzmC5uV_tIeXZzkjnhso,48299
+tinybird/sql.py,sha256=7pkwQMUVy0CH5zFgLMZyCx1BeaJSQYC05EmOb9vO3Ls,48694
 tinybird/sql_template.py,sha256=kaF5pi-f2JiWSYXEF8JsU1OIxvdu2homHnw4MYjq0n8,101953
 tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
 tinybird/sql_toolset.py,sha256=Y2_fQrbk5GSNoRncAmRS_snpV61XQbiOy4uYkeF6pr4,19767
@@ -13,12 +13,12 @@ tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
 tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
 tinybird/ch_utils/constants.py,sha256=v5-nkXHUhysu4i9Z4WVv0-sBbh6xSYUH5q5xHSY2xTI,4194
 tinybird/ch_utils/engine.py,sha256=4X1B-iuhdW_mxKnX_m3iCsxgP9RPVgR75g7yH1vsJ6A,40851
-tinybird/datafile/common.py,sha256=Cj-CyO6TKa9_31Mao5qQffIDDtoLnKfe93cubGF5S-Y,105670
+tinybird/datafile/common.py,sha256=hr9Qv4Xot21l2dW6xKPa5_JUu8pnGuDNnVpDf8pgthc,105659
 tinybird/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
 tinybird/datafile/parse_connection.py,sha256=tRyn2Rpr1TeWet5BXmMoQgaotbGdYep1qiTak_OqC5E,1825
 tinybird/datafile/parse_datasource.py,sha256=ssW8QeFSgglVFi3sDZj_HgkJiTJ2069v2JgqnH3CkDE,1825
 tinybird/datafile/parse_pipe.py,sha256=xf4m0Tw44QWJzHzAm7Z7FwUoUUtr7noMYjU1NiWnX0k,3880
-tinybird/tb/__cli__.py,sha256=OrSAyOixAYmHAPopJr6x4hd9XD_ZWyUyoEWSDwnX-ZI,247
+tinybird/tb/__cli__.py,sha256=-6WKEO5Hmmfh0zrBTi4mPaxkqDYeWDApz2022V9XsK8,247
 tinybird/tb/check_pypi.py,sha256=Gp0HkHHDFMSDL6nxKlOY51z7z1Uv-2LRexNTZSHHGmM,552
 tinybird/tb/cli.py,sha256=FdDFEIayjmsZEVsVSSvRiVYn_FHOVg_zWQzchnzfWho,1008
 tinybird/tb/client.py,sha256=IQRaInDjOwr9Fzaz3_xXc3aUGqh94tM2lew7IZbB9eM,53733
@@ -26,7 +26,7 @@ tinybird/tb/config.py,sha256=mhMTGnMB5KcxGoh3dewIr2Jjsa6pHE183gCPAQWyp6o,3973
 tinybird/tb/modules/build.py,sha256=lSCbmrqDxqa8bG009qgp2nNRpQW7JWgSen7UQNCfLVE,7657
 tinybird/tb/modules/build_common.py,sha256=KGCtyY1JGexXXhcMVFI3e-gRavmRfCoZnfGPSTYHGxk,12962
 tinybird/tb/modules/cicd.py,sha256=0KLKccha9IP749QvlXBmzdWv1On3mFwMY4DUcJlBxiE,7326
-tinybird/tb/modules/cli.py,sha256=QpEGFMtmwqnJRaK1KAtKkLsKHeJ39VHU5wFkDJJiCnk,16748
+tinybird/tb/modules/cli.py,sha256=-hkwg2ZIWPtldkLkgdY7RYWacOJtkMx58eqqYzMoMfw,17793
 tinybird/tb/modules/common.py,sha256=zY0PFDQVWjhu2MivBtUmcXtAlv7imgO-RG_PUK37XvI,83659
 tinybird/tb/modules/config.py,sha256=gK7rgaWTDd4ZKCrNEg_Uemr26EQjqWt6TjyQKujxOws,11462
 tinybird/tb/modules/connection.py,sha256=axp8Fny1_4PSLJGN4UF6WygyRbQtM3Lbt6thxHKTxzw,17790
@@ -34,7 +34,7 @@ tinybird/tb/modules/copy.py,sha256=dPZkcIDvxjJrlQUIvToO0vsEEEs4EYumbNV77-BzNoU,4
 tinybird/tb/modules/create.py,sha256=pJxHXG69c9Z_21s-7VuJ3RZOF_nJU51LEwiAkvI3dZY,23251
 tinybird/tb/modules/datasource.py,sha256=kDFHdxckTnRosk2829icfltQvlJd8EY5c9oWB5eS5Xo,41797
 tinybird/tb/modules/deployment.py,sha256=v0layOmG0IMnuXc3RT39mpGfa5M8yPlrL9F089fJFCo,15964
-tinybird/tb/modules/deployment_common.py,sha256=8Cc0VyKthmTnULiTKgciPyOGtf1kaRghC3Q00bZJbD4,19897
+tinybird/tb/modules/deployment_common.py,sha256=ZX27oUw7u1ld20qiXH69j7m0DImOc5nckABZ37BPAn4,20406
 tinybird/tb/modules/deprecations.py,sha256=rrszC1f_JJeJ8mUxGoCxckQTJFBCR8wREf4XXXN-PRc,4507
 tinybird/tb/modules/dev_server.py,sha256=57FCKuWpErwYUYgHspYDkLWEm9F4pbvVOtMrFXX1fVU,10129
 tinybird/tb/modules/endpoint.py,sha256=ksRj6mfDb9Xv63PhTkV_uKSosgysHElqagg3RTt21Do,11958
@@ -55,7 +55,7 @@ tinybird/tb/modules/mock.py,sha256=ET8sRpmXnQsd2sSJXH_KCdREU1_XQgkORru6T357Akc,3
 tinybird/tb/modules/mock_common.py,sha256=72yKp--Zo40hrycUtiajSRW2BojOsgOZFqUorQ_KQ3w,2279
 tinybird/tb/modules/open.py,sha256=LYiuO8Z1I9O_v6pv58qpUCWFD6BT00BdeO21fRa4I4Y,1315
 tinybird/tb/modules/pipe.py,sha256=xPKtezhnWZ6k_g82r4XpgKslofhuIxb_PvynH4gdUzI,2393
-tinybird/tb/modules/project.py,sha256=_P6wUOFTY87ixE06SjfYIROMoVr6oipi6qHu-RAHLD8,7356
+tinybird/tb/modules/project.py,sha256=tP6XuG8dR8QCA4pojlXZ7Am6EfS2h3Xnzw5lA3sdtlY,7747
 tinybird/tb/modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb/modules/secret.py,sha256=9BIdh2PZDAbY2wRbf4ZDvkEltygztz1RMxgDmY1D0LI,3521
 tinybird/tb/modules/secret_common.py,sha256=8mEXOdNFrTuVWu7rgZ5-C8v-rB1EAdZQUHIyBIgwISI,1578
@@ -70,17 +70,17 @@ tinybird/tb/modules/watch.py,sha256=No0bK1M1_3CYuMaIgylxf7vYFJ72lTJe3brz6xQ-mJo,
 tinybird/tb/modules/workspace.py,sha256=tCP1zZMwBhLRGm22TGfpSd4cHvQLAS1o_azIXv_r6uw,11172
 tinybird/tb/modules/workspace_members.py,sha256=5JdkJgfuEwbq-t6vxkBhYwgsiTDxF790wsa6Xfif9nk,8608
 tinybird/tb/modules/agent/__init__.py,sha256=i3oe3vDIWWPaicdCM0zs7D7BJ1W0k7th93ooskHAV00,54
-tinybird/tb/modules/agent/agent.py,sha256=uVZ0yGOk4uRtF74mPHIYjawIDPSCweb5HCC7eLVuf2k,37339
+tinybird/tb/modules/agent/agent.py,sha256=BodnCrX5cU8x6RYIwnBSLJ2nsHFTdqJCyB0capBKB0g,37708
 tinybird/tb/modules/agent/animations.py,sha256=4WOC5_2BracttmMCrV0H91tXfWcUzQHBUaIJc5FA7tE,3490
 tinybird/tb/modules/agent/banner.py,sha256=l6cO5Fi7lbVKp-GsBP8jf3IkjOWxg2jpAt9NBCy0WR8,4085
 tinybird/tb/modules/agent/command_agent.py,sha256=0Z08rQsir59zQAr-kkOvsKIFpIBsBSTGJJ1VgqqF5WA,3654
 tinybird/tb/modules/agent/compactor.py,sha256=BK5AxZFhrp3xWnsRnYaleiYoIWtVNc-_m650Hsopt8g,13841
-tinybird/tb/modules/agent/explore_agent.py,sha256=gyD5uV5TJwV24eeQiSwhkgfNPb4mtbeH7t2qSdoc18U,4100
+tinybird/tb/modules/agent/explore_agent.py,sha256=QVMPmX6GDW8bh65_X4yXepvAgJmNRQh1S-YwmvGHkjw,4288
 tinybird/tb/modules/agent/file_agent.py,sha256=vE8VChVbG_-uyeBtp3I6QrXeNaPaVlVoNn4c9cGTAJA,2466
 tinybird/tb/modules/agent/memory.py,sha256=vBewB_64L_wHoT4tLT6UX2uxcHwSY880QZ26F9rPqXs,3793
-tinybird/tb/modules/agent/mock_agent.py,sha256=zbAZfAqdSLUtMr2VqO0erWpzjT2F1tTcuYjvHb-gvbA,8023
+tinybird/tb/modules/agent/mock_agent.py,sha256=9_w5-MJDC3vp0U5SkM-ucqhxftfpeUM4FPLl5BAr1_0,8200
 tinybird/tb/modules/agent/models.py,sha256=eokO8XlY-kVJOsbqiVporGUAOCyKAXCO5xgTEK9SM6Y,2208
-tinybird/tb/modules/agent/prompts.py,sha256=a8ZnOZCTGRnxa_dlrTSmeaM9snOI3sMx0nwUdv7aVGI,44165
+tinybird/tb/modules/agent/prompts.py,sha256=eua-QawthfG5X6Kz1dRzOZSjXU8tj0nr4k7onmvBZnk,45792
 tinybird/tb/modules/agent/testing_agent.py,sha256=AtwtJViH7805i7djyBgDb7SSUtDyJnw0TWJu6lBFsrg,2953
 tinybird/tb/modules/agent/utils.py,sha256=alvRNtpPPRZP0sNdOXbzX3uQZHqqeh1daIAg3aquq24,32015
 tinybird/tb/modules/agent/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -98,21 +98,21 @@ tinybird/tb/modules/agent/tools/get_openapi_definition.py,sha256=4TIMO2XzHBMhpt9
 tinybird/tb/modules/agent/tools/mock.py,sha256=Seo4WcYNLL1-SmPXutoaX94_pfOdIb47JXo8dHtUVhg,7106
 tinybird/tb/modules/agent/tools/plan.py,sha256=uAJEHZ-xXIq-EpURJYV7GUyY7IbIgactw9NWeCsIT9Y,2516
 tinybird/tb/modules/agent/tools/request_endpoint.py,sha256=bsLWrMn-ofJM3nn9vm8j_U8fdopVd3H5L0ii6ji-Kuw,4359
-tinybird/tb/modules/agent/tools/run_command.py,sha256=EOHkyK9nktlt5jVz9sQcSt4CN40LO490FjmAy3CuNmE,1793
+tinybird/tb/modules/agent/tools/run_command.py,sha256=B0jyXQW7d0vSBi7abCTlhmEsf6nvwWcVEbRpGatLFBo,2464
 tinybird/tb/modules/agent/tools/secret.py,sha256=8AGTZgHLPg1bxCA2cPMnb-zNutWEwn4emHo7kLjJC5w,4323
 tinybird/tb/modules/agent/tools/test.py,sha256=4XuEWVHLOTSO51Z9xJ08dTjk0j3IWY_JlPtSBO5aaUs,10373
 tinybird/tb/modules/datafile/build.py,sha256=NFKBrusFLU0WJNCXePAFWiEDuTaXpwc0lHlOQWEJ43s,51117
 tinybird/tb/modules/datafile/build_common.py,sha256=2yNdxe49IMA9wNvl25NemY2Iaz8L66snjOdT64dm1is,4511
 tinybird/tb/modules/datafile/build_datasource.py,sha256=Ra8pVQBDafbFRUKlhpgohhTsRyp_ADKZJVG8Gd69idY,17227
 tinybird/tb/modules/datafile/build_pipe.py,sha256=-mpzYcTEdH313R-FEDYuDOtAbZ5zehynzJwmtsfy16w,11262
-tinybird/tb/modules/datafile/diff.py,sha256=vCwBxUtQskmZSu-UxscqdgFkEJ9i3t1U1F6RrX9wRrA,6674
+tinybird/tb/modules/datafile/diff.py,sha256=RxkBkuSAPUZF7slWe0yaaYLea4ZDZ-qfEscVAT9xvWU,6716
 tinybird/tb/modules/datafile/fixture.py,sha256=gftYWeweeQDFK3cHgUmSOfltNjZVOuMt8v7WTMLhGBw,1579
 tinybird/tb/modules/datafile/format_common.py,sha256=5w6GX_8RDFueW9AtDfWpc1OicEu8wWdrjUBdCEidCQk,1951
 tinybird/tb/modules/datafile/format_datasource.py,sha256=X5N65vDpprFKaWaxkSwFnHaNzYIxVhG1Sya4BARTd2Q,6138
 tinybird/tb/modules/datafile/format_pipe.py,sha256=_64RGD8XI-gpQBMzhymUAVGMhpQK6zar_irDrJFQrHY,7378
 tinybird/tb/modules/datafile/pipe_checker.py,sha256=dxsCQoA6ruxg1fvF6sMLFowpjaqws8lUQcM7XyhgZPE,24609
 tinybird/tb/modules/datafile/playground.py,sha256=47q4XcvDhclHWYv7oFNEqmF1T5NXd9GPWkrT6Av2VhY,56343
-tinybird/tb/modules/datafile/pull.py,sha256=BB-JSAPDRL8WotaSo73PHBDdAKBx4ZM-YeZCVcXdRTQ,4311
+tinybird/tb/modules/datafile/pull.py,sha256=6mn3xqLxb6KQDcnqq63AaPzku9XsSOpscfcdsHyvvsY,5101
 tinybird/tb/modules/tinyunit/tinyunit.py,sha256=sJeBnKDHcUNn8t0sdsKB_M2w8IdzeeQ1PzJuleeQ2-E,11695
 tinybird/tb/modules/tinyunit/tinyunit_lib.py,sha256=hGh1ZaXC1af7rKnX7222urkj0QJMhMWclqMy59dOqwE,1922
 tinybird/tb_cli_modules/cicd.py,sha256=0lMkb6CVOFZl5HOwgY8mK4T4mgI7O8335UngLXtCc-c,13851
@@ -121,8 +121,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev290.dist-info/METADATA,sha256=rgJDi-VYXwLLOeFrK9n1nONEYic69YtTXbVcxDFW0yU,1845
-tinybird-0.0.1.dev290.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev290.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev290.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev290.dist-info/RECORD,,
+tinybird-0.0.1.dev292.dist-info/METADATA,sha256=d2RNeqvNyqjfXrpEIlGB9lV-Kh2KUu3bD8c99JGeGJQ,1845
+tinybird-0.0.1.dev292.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev292.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev292.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev292.dist-info/RECORD,,