tinybird 4.5.2.dev0__py3-none-any.whl → 4.5.3.dev0__py3-none-any.whl

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '4.5.2.dev0'
-__revision__ = '4cf88c3'
+__version__ = '4.5.3.dev0'
+__revision__ = 'afc5f42'

tinybird/tb/modules/build.py

@@ -75,6 +75,7 @@ def build(ctx: click.Context, watch: bool, with_connections: bool) -> None:
     tb_client: TinyB = ctx.ensure_object(dict)["client"]
     config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
     is_branch = bool(ctx.ensure_object(dict)["branch"])
+    use_deployment_api = obj["env"] == "cloud" and is_branch
 
     # TODO: Explain that you can use custom branches too once they are open for everyone
     if obj["env"] == "cloud" and not is_branch:
@@ -100,6 +101,7 @@ def build(ctx: click.Context, watch: bool, with_connections: bool) -> None:
         config=config,
         is_branch=is_branch,
         with_connections=with_connections,
+        use_deployment_api=use_deployment_api,
     )
     if watch:
         run_watch(
@@ -113,6 +115,7 @@ def build(ctx: click.Context, watch: bool, with_connections: bool) -> None:
                 config=config,
                 is_branch=is_branch,
                 with_connections=with_connections,
+                use_deployment_api=use_deployment_api,
             ),
         )
 
@@ -128,6 +131,7 @@ def dev(ctx: click.Context, with_connections: Optional[bool]) -> None:
     obj: Dict[str, Any] = ctx.ensure_object(dict)
     branch: Optional[str] = ctx.ensure_object(dict)["branch"]
     is_branch = bool(branch)
+    use_deployment_api = obj["env"] == "cloud" and is_branch
 
     # Default with_connections to True for branches, False otherwise
     if with_connections is None:
@@ -149,6 +153,7 @@ def dev(ctx: click.Context, with_connections: Optional[bool]) -> None:
         config=config,
         is_branch=is_branch,
         with_connections=with_connections,
+        use_deployment_api=use_deployment_api,
     )
     run_watch(
         project=project,
@@ -160,6 +165,7 @@ def dev(ctx: click.Context, with_connections: Optional[bool]) -> None:
             config=config,
             is_branch=is_branch,
             with_connections=with_connections,
+            use_deployment_api=use_deployment_api,
         ),
     )
 

tinybird/tb/modules/build_common.py

@@ -14,6 +14,7 @@ from tinybird.datafile.parse_datasource import parse_datasource
 from tinybird.tb.client import TinyB
 from tinybird.tb.modules.common import push_data, sys_exit
 from tinybird.tb.modules.datafile.fixture import FixtureExtension, get_fixture_dir, persist_fixture
+from tinybird.tb.modules.deployment_common import api_fetch
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.local_common import get_local_tokens
 from tinybird.tb.modules.project import Project
@@ -33,6 +34,7 @@ def process(
     project_with_vendors: Optional[Project] = None,
     is_branch: bool = False,
     with_connections: bool = False,
+    use_deployment_api: bool = False,
 ) -> Optional[str]:
     time_start = time.time()
 
@@ -65,6 +67,7 @@ def process(
                 load_fixtures,
                 project_with_vendors=project_with_vendors,
                 with_connections=with_connections,
+                use_deployment_api=use_deployment_api,
             )
 
         except click.ClickException as e:
@@ -196,13 +199,8 @@ def build_project(
     load_fixtures: bool = True,
     project_with_vendors: Optional[Project] = None,
     with_connections: bool = False,
+    use_deployment_api: bool = False,
 ) -> Optional[bool]:
-    MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
-    DATAFILE_TYPE_TO_CONTENT_TYPE = {
-        ".datasource": "text/plain",
-        ".pipe": "text/plain",
-        ".connection": "text/plain",
-    }
     build_url = "/v1/build"
     if with_connections:
         build_url = f"{build_url}?with_connections=true"
@@ -213,28 +211,20 @@ def build_project(
     error: Optional[str] = None
 
     try:
-        files = [
-            ("context://", ("cli-version", "1.0.0", "text/plain")),
-        ]
-        project_path = project.path
-        project_files = project.get_project_files()
+        if use_deployment_api:
+            return build_project_with_deploy_api(
+                project=project,
+                tb_client=tb_client,
+                silent=silent,
+                load_fixtures=load_fixtures,
+                project_with_vendors=project_with_vendors,
+            )
+
+        files, project_files = get_build_request_files(project, project_with_vendors)
 
         if not project_files:
             return False
 
-        for file_path in project_files:
-            relative_path = Path(file_path).relative_to(project_path).as_posix()
-            with open(file_path, "rb") as fd:
-                content_type = DATAFILE_TYPE_TO_CONTENT_TYPE.get(Path(file_path).suffix, "application/unknown")
-                content = fd.read().decode("utf-8")
-                if project_with_vendors:
-                    # Replace 'SHARED_WITH' and everything that comes after, including new lines, with 'SHARED_WITH Tinybird_Local_Test_'
-                    content = replace_shared_with(
-                        content,
-                        [project_with_vendors.workspace_name],
-                    )
-
-                files.append((MULTIPART_BOUNDARY_DATA_PROJECT, (relative_path, content, content_type)))
         HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
         params = {"from": request_from} if request_from else None
         r = requests.post(TINYBIRD_API_URL, files=files, headers=HEADERS, params=params)
@@ -249,90 +239,236 @@ def build_project(
 
         build_result = result.get("result")
         if build_result == "success":
-            build = result.get("build")
-            new_datasources = build.get("new_datasource_names", [])
-            new_pipes = build.get("new_pipe_names", [])
-            new_connections = build.get("new_data_connector_names", [])
-            changed_datasources = build.get("changed_datasource_names", [])
-            changed_pipes = build.get("changed_pipe_names", [])
-            changed_connections = build.get("changed_data_connector_names", [])
-            deleted_datasources = build.get("deleted_datasource_names", [])
-            deleted_pipes = build.get("deleted_pipe_names", [])
-            deleted_connections = build.get("deleted_data_connector_names", [])
-
-            no_changes = (
-                not new_datasources
-                and not changed_datasources
-                and not new_pipes
-                and not changed_pipes
-                and not new_connections
-                and not changed_connections
-                and not deleted_datasources
-                and not deleted_pipes
-                and not deleted_connections
-            )
-            if no_changes:
+            build = result.get("build") or {}
+            changes = get_build_changes(build)
+            if not has_build_changes(changes):
                 return False
-            if not silent:
-                echo_changes(project, new_datasources, ".datasource", "created")
-                echo_changes(project, changed_datasources, ".datasource", "changed")
-                echo_changes(project, deleted_datasources, ".datasource", "deleted")
-                echo_changes(project, new_pipes, ".pipe", "created")
-                echo_changes(project, changed_pipes, ".pipe", "changed")
-                echo_changes(project, deleted_pipes, ".pipe", "deleted")
-                echo_changes(project, new_connections, ".connection", "created")
-                echo_changes(project, changed_connections, ".connection", "changed")
-                echo_changes(project, deleted_connections, ".connection", "deleted")
+            echo_build_changes(project, changes, silent)
             if load_fixtures:
-                try:
-                    for filename in project_files:
-                        if filename.endswith(".datasource"):
-                            ds_path = Path(filename)
-                            ds_name = ds_path.stem
-                            fixture_folder = get_fixture_dir(project.folder)
-                            fixture_extensions = [FixtureExtension.NDJSON, FixtureExtension.CSV]
-                            fixture_path = next(
-                                (
-                                    fixture_folder / f"{ds_name}{ext}"
-                                    for ext in fixture_extensions
-                                    if (fixture_folder / f"{ds_name}{ext}").exists()
-                                ),
-                                None,
-                            )
-                            if not fixture_path:
-                                sql_path = fixture_folder / f"{ds_name}.sql"
-                                if sql_path.exists():
-                                    fixture_path = rebuild_fixture_sql(project, tb_client, str(sql_path))
-
-                            if fixture_path:
-                                append_fixture(tb_client, ds_name, str(fixture_path))
-
-                except Exception as e:
-                    click.echo(FeedbackManager.error_exception(error=f"Error appending fixtures for '{ds_name}': {e}"))
-
-            feedback = build.get("feedback", [])
-            for f in feedback:
-                click.echo(
-                    FeedbackManager.warning(message=f"△ {f.get('level')}: {f.get('resource')}: {f.get('message')}")
-                )
+                append_project_fixtures(project, tb_client, project_files)
+            echo_build_feedback(build.get("feedback", []))
+            return True
         elif build_result == "failed":
-            build_errors = result.get("errors")
-            full_error_msg = ""
-            for build_error in build_errors:
-                filename_bit = build_error.get("filename", build_error.get("resource", ""))
-                error_bit = build_error.get("error") or build_error.get("message") or ""
-                error_msg = ((filename_bit + "\n") if filename_bit else "") + error_bit
-                full_error_msg += error_msg + "\n\n"
-            error = full_error_msg.strip("\n") or "Unknown build error"
+            error = format_build_errors(result.get("errors", []))
         else:
             error = f"Unknown build result. Error: {result.get('error')}"
+    except click.ClickException:
+        raise
     except Exception as e:
         error = str(e)
 
     if error:
         raise click.ClickException(error)
 
-    return build_result
+    return False
+
+
+def build_project_with_deploy_api(
+    project: Project,
+    tb_client: TinyB,
+    silent: bool = False,
+    load_fixtures: bool = True,
+    project_with_vendors: Optional[Project] = None,
+) -> Optional[bool]:
+    deploy_url = urljoin(tb_client.host, "/v1/deploy")
+    logging.debug(deploy_url)
+    request_from = getattr(tb_client, "request_from", None)
+    files, project_files = get_build_request_files(project, project_with_vendors)
+
+    if not project_files:
+        return False
+
+    headers = {"Authorization": f"Bearer {tb_client.token}"}
+    params: dict[str, str] = {"auto_promote": "true"}
+    if request_from:
+        params["from"] = request_from
+
+    response = requests.post(deploy_url, files=files, headers=headers, params=params)
+    try:
+        result = response.json()
+    except Exception as e:
+        logging.debug(e, exc_info=True)
+        click.echo(FeedbackManager.error(message="Couldn't parse response from server"))
+        sys_exit("build_error", str(e))
+
+    logging.debug(json.dumps(result, indent=2))
+
+    build_result = result.get("result")
+    deployment = result.get("deployment") or {}
+    if build_result == "no_changes":
+        return False
+    if build_result != "success":
+        deployment_errors = deployment.get("errors", []) if deployment else result.get("errors", [])
+        raise click.ClickException(result.get("error") or format_build_errors(deployment_errors))
+    if not deployment:
+        raise click.ClickException("Couldn't parse deployment response from server")
+
+    deployment = wait_for_build_deployment_to_be_live(
+        tb_client=tb_client,
+        headers=headers,
+        deployment=deployment,
+        request_from=request_from,
+        silent=silent,
+    )
+    changes = get_build_changes(deployment)
+    if not has_build_changes(changes):
+        return False
+
+    echo_build_changes(project, changes, silent)
+    if load_fixtures:
+        append_project_fixtures(project, tb_client, project_files)
+    echo_build_feedback(deployment.get("feedback", []))
+    return True
+
+
+def get_build_request_files(
+    project: Project,
+    project_with_vendors: Optional[Project] = None,
+) -> tuple[list[tuple[str, tuple[str, str, str]]], list[str]]:
+    multipart_boundary_data_project = "data_project://"
+    datafile_type_to_content_type = {
+        ".datasource": "text/plain",
+        ".pipe": "text/plain",
+        ".connection": "text/plain",
+    }
+    files: list[tuple[str, tuple[str, str, str]]] = [
+        ("context://", ("cli-version", "1.0.0", "text/plain")),
+    ]
+    project_path = project.path
+    project_files = project.get_project_files()
+
+    for file_path in project_files:
+        relative_path = Path(file_path).relative_to(project_path).as_posix()
+        with open(file_path, "rb") as fd:
+            content_type = datafile_type_to_content_type.get(Path(file_path).suffix, "application/unknown")
+            content = fd.read().decode("utf-8")
+            if project_with_vendors:
+                # Replace SHARED_WITH targets when building vendored workspaces against the main project.
+                content = replace_shared_with(content, [project_with_vendors.workspace_name])
+
+            files.append((multipart_boundary_data_project, (relative_path, content, content_type)))
+
+    return files, project_files
+
+
+def wait_for_build_deployment_to_be_live(
+    tb_client: TinyB,
+    headers: dict[str, str],
+    deployment: dict[str, Any],
+    request_from: Optional[str],
+    silent: bool,
+) -> dict[str, Any]:
+    if not silent:
+        click.echo(FeedbackManager.highlight(message="» Waiting for deployment to be ready..."))
+
+    poll_interval = 5
+    times_seen_failed = 0
+    while True:
+        url = f"{tb_client.host}/v1/deployments/{deployment.get('id')}"
+        result = api_fetch(url, headers, request_from=request_from)
+        deployment = result.get("deployment", {})
+        if not deployment:
+            raise click.ClickException("Error parsing deployment from response")
+
+        status = deployment.get("status")
+        if status == "failed":
+            times_seen_failed += 1
+            if times_seen_failed > 60:
+                raise click.ClickException("Deployment failed and wasn't deleted automatically")
+            time.sleep(poll_interval)
+            continue
+
+        if status in ("deleting", "deleted"):
+            errors = deployment.get("errors", [])
+            raise click.ClickException(f"Deployment deleted after failure. Errors: {errors}")
+
+        if deployment.get("live"):
+            return deployment
+
+        time.sleep(poll_interval)
+
+
+def get_build_changes(result: dict[str, Any]) -> dict[str, list[str]]:
+    return {
+        "new_datasources": result.get("new_datasource_names", []),
+        "changed_datasources": result.get("changed_datasource_names", []),
+        "deleted_datasources": result.get("deleted_datasource_names", []),
+        "new_pipes": result.get("new_pipe_names", []),
+        "changed_pipes": result.get("changed_pipe_names", []),
+        "deleted_pipes": result.get("deleted_pipe_names", []),
+        "new_connections": result.get("new_data_connector_names", []),
+        "changed_connections": result.get("changed_data_connector_names", []),
+        "deleted_connections": result.get("deleted_data_connector_names", []),
+    }
+
+
+def has_build_changes(changes: dict[str, list[str]]) -> bool:
+    return any(changes.values())
+
+
+def echo_build_changes(project: Project, changes: dict[str, list[str]], silent: bool) -> None:
+    if silent:
+        return
+
+    echo_changes(project, changes["new_datasources"], ".datasource", "created")
+    echo_changes(project, changes["changed_datasources"], ".datasource", "changed")
+    echo_changes(project, changes["deleted_datasources"], ".datasource", "deleted")
+    echo_changes(project, changes["new_pipes"], ".pipe", "created")
+    echo_changes(project, changes["changed_pipes"], ".pipe", "changed")
+    echo_changes(project, changes["deleted_pipes"], ".pipe", "deleted")
+    echo_changes(project, changes["new_connections"], ".connection", "created")
+    echo_changes(project, changes["changed_connections"], ".connection", "changed")
+    echo_changes(project, changes["deleted_connections"], ".connection", "deleted")
+
+
+def append_project_fixtures(
+    project: Project,
+    tb_client: TinyB,
+    project_files: list[str],
+) -> None:
+    ds_name = ""
+    try:
+        for filename in project_files:
+            if not filename.endswith(".datasource"):
+                continue
+
+            ds_name = Path(filename).stem
+            fixture_folder = get_fixture_dir(project.folder)
+            fixture_extensions = [FixtureExtension.NDJSON, FixtureExtension.CSV]
+            fixture_path = next(
+                (
+                    fixture_folder / f"{ds_name}{ext}"
+                    for ext in fixture_extensions
+                    if (fixture_folder / f"{ds_name}{ext}").exists()
+                ),
+                None,
+            )
+            if not fixture_path:
+                sql_path = fixture_folder / f"{ds_name}.sql"
+                if sql_path.exists():
+                    fixture_path = rebuild_fixture_sql(project, tb_client, str(sql_path))
+
+            if fixture_path:
+                append_fixture(tb_client, ds_name, str(fixture_path))
+    except Exception as e:
+        click.echo(FeedbackManager.error_exception(error=f"Error appending fixtures for '{ds_name}': {e}"))
+
+
+def echo_build_feedback(feedback: list[dict[str, Any]]) -> None:
+    for item in feedback:
+        click.echo(
+            FeedbackManager.warning(message=f"△ {item.get('level')}: {item.get('resource')}: {item.get('message')}")
+        )
+
+
+def format_build_errors(build_errors: list[dict[str, Any]]) -> str:
+    full_error_msg = ""
+    for build_error in build_errors:
+        filename_bit = build_error.get("filename", build_error.get("resource", ""))
+        error_bit = build_error.get("error") or build_error.get("message") or ""
+        error_msg = ((filename_bit + "\n") if filename_bit else "") + error_bit
+        full_error_msg += error_msg + "\n\n"
+    return full_error_msg.strip("\n") or "Unknown build error"
 
 
 def echo_changes(project: Project, changes: list[str], extension: str, status: str):

{tinybird-4.5.2.dev0.dist-info → tinybird-4.5.3.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 4.5.2.dev0
+Version: 4.5.3.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -52,6 +52,11 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+4.5.2
+***********
+
+- `Fixed` `tb build` on branches now runs as an asynchronous job, avoiding timeouts on long-running executions.
+
 4.5.1
 *******
 

{tinybird-4.5.2.dev0.dist-info → tinybird-4.5.3.dev0.dist-info}/RECORD

@@ -17,14 +17,14 @@ tinybird/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1w
 tinybird/datafile/parse_connection.py,sha256=GxmGp_XnWbDZPDbh_PBxitlIMqZRYfDwxMBw-JQBp1g,1890
 tinybird/datafile/parse_datasource.py,sha256=yd58HrUF4yNJXLn6OsvKGpZJpvrcjLGAeJG1lgBe_zk,1891
 tinybird/datafile/parse_pipe.py,sha256=-9bbgVuiWRyDYydrLVflDBt8GstZotMy6dklsrc6MUY,3859
-tinybird/tb/__cli__.py,sha256=7WU6bivcBQ_tYlueRvNPz-IFZshSMj5WJx3OmMpSBLU,245
+tinybird/tb/__cli__.py,sha256=fEPZtz5yQ0OXcYpjwKowucqLmPf8le00OegJ776lpAQ,245
 tinybird/tb/check_pypi.py,sha256=Gp0HkHHDFMSDL6nxKlOY51z7z1Uv-2LRexNTZSHHGmM,552
 tinybird/tb/cli.py,sha256=IjiGfNIpxSxi1odK1kMj9s8lEhx3sAUgGA263XdmyR0,1119
 tinybird/tb/client.py,sha256=RM8VQwPZFonfvejuTf3oT09XbJmq7pD5VLzFt4zoBGM,53910
 tinybird/tb/config.py,sha256=CGCfBbCMlmVcBHQ0IMGc2IE4O2-1tZEPyD564JZoTbw,5659
 tinybird/tb/modules/branch.py,sha256=Ik6KPkWcraqnBCxfZpQlMLadH_OSqGVSGdNE0eMUHp0,9318
-tinybird/tb/modules/build.py,sha256=Sj2UobSvEj29byeN6LwFDY9fyFQotd6SE8vrW3XPAqs,10098
-tinybird/tb/modules/build_common.py,sha256=Ok1HIlDpVTCKMu2fYOdhgmrAEb0GdAW307-A1_UzzVg,19717
+tinybird/tb/modules/build.py,sha256=bGFoFppR_UbGUMDWFGDzfJ4nT3CGFwzCzl2o4OpwR2o,10420
+tinybird/tb/modules/build_common.py,sha256=KxAE_HCq4QupIghW4n6IVokznwgA3Lv-KZ_mE9YAUPk,23709
 tinybird/tb/modules/cicd.py,sha256=xGa1zop2Io0Y3YcvOea7IAboBSM_qm2zZdeBvl93KIs,8235
 tinybird/tb/modules/cli.py,sha256=WVk7zTQS5a2ROGXALQSM57Gpy8QCKy3fJ5tDtFfT1Ic,40701
 tinybird/tb/modules/common.py,sha256=j7fg_8rIQFPIzzDfs9Xuse5jU-QaomuzY-6aqXb9izo,91937
@@ -97,8 +97,8 @@ tinybird/tb_cli_modules/config.py,sha256=0kFDmsDcjKon32rgFGMHHKSbv4j5dOrXtVOlyuA
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=W098H6jmS4kpE7hN3tadaREBTf7oMocel-lkKWN0pU8,10466
-tinybird-4.5.2.dev0.dist-info/METADATA,sha256=ObQb41Q6V0SnnAeZYrCoRvJSITyJ1yhT_yLF_mSwqlI,9880
-tinybird-4.5.2.dev0.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-tinybird-4.5.2.dev0.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-4.5.2.dev0.dist-info/top_level.txt,sha256=ZIQJTPCzMqnfDzM_hEGZrJqDSEcKnIK_49T86DGWpyQ,78
-tinybird-4.5.2.dev0.dist-info/RECORD,,
+tinybird-4.5.3.dev0.dist-info/METADATA,sha256=3fpTA7h3b5AZYOU74U4xT0CGMET3aG6Mx3_ajbBabEk,10012
+tinybird-4.5.3.dev0.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
+tinybird-4.5.3.dev0.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-4.5.3.dev0.dist-info/top_level.txt,sha256=ZIQJTPCzMqnfDzM_hEGZrJqDSEcKnIK_49T86DGWpyQ,78
+tinybird-4.5.3.dev0.dist-info/RECORD,,