tinybird 0.0.1.dev42__py3-none-any.whl → 0.0.1.dev44__py3-none-any.whl

tinybird/tb/modules/datafile/build.py

@@ -15,14 +15,12 @@ from tinybird.client import TinyB
 from tinybird.sql import parse_table_structure, schema_to_sql_columns
 from tinybird.sql_template import get_used_tables_in_template, render_sql_template
 from tinybird.tb.modules.common import get_ca_pem_content
-from tinybird.tb.modules.datafile.build_common import update_tags_in_resource
-from tinybird.tb.modules.datafile.build_datasource import is_datasource, new_ds
+from tinybird.tb.modules.datafile.build_datasource import is_datasource
 from tinybird.tb.modules.datafile.build_pipe import (
     get_target_materialized_data_source_name,
     is_endpoint,
     is_endpoint_with_no_dependencies,
     is_materialized,
-    new_pipe,
 )
 from tinybird.tb.modules.datafile.common import (
     DEFAULT_CRON_PERIOD,
@@ -44,95 +42,32 @@ from tinybird.tb.modules.datafile.exceptions import AlreadyExistsException, Incl
 from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
 from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 from tinybird.tb.modules.feedback_manager import FeedbackManager
-from tinybird.tb.modules.local_common import get_tinybird_local_config
+from tinybird.tb.modules.local_common import get_tinybird_local_client
+from tinybird.tb.modules.project import Project
 
 
 async def folder_build(
-    tb_client: TinyB,
+    project: Project,
     filenames: Optional[List[str]] = None,
-    folder: str = ".",
-    ignore_sql_errors: bool = False,
     is_internal: bool = False,
     is_vendor: bool = False,
     current_ws: Optional[Dict[str, Any]] = None,
     local_ws: Optional[Dict[str, Any]] = None,
     watch: bool = False,
 ):
-    config = await get_tinybird_local_config(folder)
     build = True
     dry_run = False
     force = True
     only_changes = True
     debug = False
-    check = True
-    populate = False
-    populate_subset = None
-    populate_condition = None
-    tests_to_run = 0
-    override_datasource = False
-    skip_confirmation = True
-    wait = False
-    unlink_on_populate_error = False
-    only_response_times = False
     run_tests = False
     verbose = False
-    as_standard = False
     raise_on_exists = False
     fork_downstream = True
     fork = False
     release_created = False
-    tests_relative_change = 0.01
-    tests_sample_by_params = 0
-    tests_filter_by = None
-    tests_failfast = False
-    tests_ignore_order = False
-    tests_validate_processed_bytes = False
-    tests_check_requests_from_branch = False
-    vendor_paths = []
-
-    vendor_path = Path("vendor")
-    user_token = config.get_user_token()
-    user_client = deepcopy(tb_client)
-
-    if user_token:
-        user_client.token = user_token
-
-    vendor_workspaces = []
-
-    if vendor_path.exists() and not is_vendor and not watch:
-        user_workspaces = await user_client.user_workspaces()
-        for x in vendor_path.iterdir():
-            if x.is_dir() and x.name:
-                if user_token:
-                    try:
-                        ws_to_delete = next((ws for ws in user_workspaces["workspaces"] if ws["name"] == x.name), None)
-                        if ws_to_delete:
-                            await user_client.delete_workspace(ws_to_delete["id"], hard_delete_confirmation=x.name)
-                    except Exception:
-                        pass
-                    vendor_ws = await user_client.create_workspace(x.name, template=None)
-                    vendor_workspaces.append(vendor_ws)
-                vendor_paths.append((x.name, str(x)))
-
-    workspaces: List[Dict[str, Any]] = (await user_client.user_workspaces()).get("workspaces", [])
-
-    if not is_vendor:
-        local_workspace = await tb_client.workspace_info()
-        local_ws_id = local_workspace.get("id")
-        local_ws = next((ws for ws in workspaces if ws["id"] == local_ws_id), {})
-
-    current_ws = current_ws or local_ws
-
-    for vendor_ws in [ws for ws in workspaces if ws["name"] in [ws["name"] for ws in vendor_workspaces]]:
-        ws_client = deepcopy(tb_client)
-        ws_client.token = vendor_ws["token"]
-        shared_ws_path = Path(folder) / "vendor" / vendor_ws["name"]
-
-        if shared_ws_path.exists() and not is_vendor and not watch:
-            await folder_build(
-                ws_client, folder=shared_ws_path.as_posix(), is_vendor=True, current_ws=vendor_ws, local_ws=local_ws
-            )
-
+    folder = str(project.path)
+    tb_client = await get_tinybird_local_client(folder)
     datasources: List[Dict[str, Any]] = await tb_client.datasources()
     pipes: List[Dict[str, Any]] = await tb_client.pipes(dependencies=True)
 
@@ -150,7 +85,7 @@ async def folder_build(
         dir_path=folder,
         process_dependencies=True,
         skip_connectors=True,
-        vendor_paths=vendor_paths,
+        vendor_paths=[],
         current_ws=current_ws,
         only_changes=only_changes,
         fork_downstream=fork_downstream,
@@ -191,63 +126,14 @@ async def folder_build(
             resource = to_run[name]["resource"]
             if not dry_run:
                 if should_push_file(name, remote_resource_names, force, run_tests):
-                    click.echo(FeedbackManager.info_processing_new_resource(name=name, version=""))
-                    try:
-                        await exec_file(
-                            to_run[name],
-                            tb_client,
-                            force,
-                            check,
-                            debug and verbose,
-                            populate,
-                            populate_subset,
-                            populate_condition,
-                            unlink_on_populate_error,
-                            wait,
-                            user_token,
-                            override_datasource,
-                            ignore_sql_errors,
-                            skip_confirmation,
-                            only_response_times,
-                            run_tests,
-                            as_standard,
-                            tests_to_run,
-                            tests_relative_change,
-                            tests_sample_by_params,
-                            tests_filter_by,
-                            tests_failfast,
-                            tests_ignore_order,
-                            tests_validate_processed_bytes,
-                            tests_check_requests_from_branch,
-                            current_ws,
-                            local_ws,
-                            fork_downstream,
-                            fork,
-                            build,
-                            is_vendor,
-                        )
-                        if not run_tests:
-                            click.echo(
-                                FeedbackManager.success_create(
-                                    name=(
-                                        name
-                                        if to_run[name]["version"] is None
-                                        else f'{name}__v{to_run[name]["version"]}'
-                                    )
-                                )
-                            )
-                    except Exception as e:
-                        filename = to_run[name]["filename"]
-                        exception = FeedbackManager.error_push_file_exception(
-                            filename=filename,
-                            error=e,
-                        )
-                        raise click.ClickException(exception)
+                    filename = to_run[name]["filename"]
+                    filename = filename.replace(f"{folder}/", "")
+                    click.echo(FeedbackManager.info(message=f"✓ {filename}"))
                 else:
                     if raise_on_exists:
                         raise AlreadyExistsException(
                             FeedbackManager.warning_name_already_exists(
-                                name=name if to_run[name]["version"] is None else f'{name}__v{to_run[name]["version"]}'
+                                name=name if to_run[name]["version"] is None else f"{name}__v{to_run[name]['version']}"
                             )
                         )
                     else:
@@ -262,7 +148,7 @@ async def folder_build(
                                     name=(
                                         name
                                         if to_run[name]["version"] is None
-                                        else f'{name}__v{to_run[name]["version"]}'
+                                        else f"{name}__v{to_run[name]['version']}"
                                     )
                                 )
                             )
@@ -304,19 +190,6 @@ async def folder_build(
         # We need to deploy the datasources from left to right as some datasources might have MV that depend on the column types of previous datasources. Ex: `test_change_column_type_landing_datasource` test
         groups = [group for group in toposort(dependencies_graph_fork_downstream)]
 
-        for group in groups:
-            for name in group:
-                is_vendor = resources_to_run_fork_downstream.get(name, {}).get("filename", "").startswith("vendor/")
-                if not is_vendor:
-                    try:
-                        await tb_client.datasource_delete(name, force=True)
-                    except Exception:
-                        pass
-                    try:
-                        await tb_client.pipe_delete(name)
-                    except Exception:
-                        pass
-
         groups.reverse()
         for group in groups:
             for name in group:
@@ -433,88 +306,6 @@ async def name_matches_existing_resource(resource: str, name: str, tb_client: Ti
     return False
 
 
-async def exec_file(
-    r: Dict[str, Any],
-    tb_client: TinyB,
-    force: bool,
-    check: bool,
-    debug: bool,
-    populate: bool,
-    populate_subset,
-    populate_condition,
-    unlink_on_populate_error,
-    wait_populate,
-    user_token: Optional[str],
-    override_datasource: bool = False,
-    ignore_sql_errors: bool = False,
-    skip_confirmation: bool = False,
-    only_response_times: bool = False,
-    run_tests=False,
-    as_standard=False,
-    tests_to_run: int = 0,
-    tests_relative_change: float = 0.01,
-    tests_to_sample_by_params: int = 0,
-    tests_filter_by: Optional[List[str]] = None,
-    tests_failfast: bool = False,
-    tests_ignore_order: bool = False,
-    tests_validate_processed_bytes: bool = False,
-    tests_check_requests_from_branch: bool = False,
-    current_ws: Optional[Dict[str, Any]] = None,
-    local_ws: Optional[Dict[str, Any]] = None,
-    fork_downstream: Optional[bool] = False,
-    fork: Optional[bool] = False,
-    build: Optional[bool] = False,
-    is_vendor: Optional[bool] = False,
-):
-    if debug:
-        click.echo(FeedbackManager.debug_running_file(file=pp.pformat(r)))
-    if r["resource"] == "pipes":
-        await new_pipe(
-            r,
-            tb_client,
-            force,
-            check,
-            populate,
-            populate_subset,
-            populate_condition,
-            unlink_on_populate_error,
-            wait_populate,
-            ignore_sql_errors=ignore_sql_errors,
-            only_response_times=only_response_times,
-            run_tests=run_tests,
-            as_standard=as_standard,
-            tests_to_run=tests_to_run,
-            tests_relative_change=tests_relative_change,
-            tests_to_sample_by_params=tests_to_sample_by_params,
-            tests_filter_by=tests_filter_by,
-            tests_failfast=tests_failfast,
-            tests_ignore_order=tests_ignore_order,
-            tests_validate_processed_bytes=tests_validate_processed_bytes,
-            override_datasource=override_datasource,
-            tests_check_requests_from_branch=tests_check_requests_from_branch,
-            fork_downstream=fork_downstream,
-            fork=fork,
-        )
-        await update_tags_in_resource(r, "pipe", tb_client)
-    elif r["resource"] == "datasources":
-        await new_ds(
-            r,
-            tb_client,
-            user_token,
-            force,
-            skip_confirmation=skip_confirmation,
-            current_ws=current_ws,
-            local_ws=local_ws,
-            fork_downstream=fork_downstream,
-            fork=fork,
-            build=build,
-            is_vendor=is_vendor,
-        )
-        await update_tags_in_resource(r, "datasource", tb_client)
-    else:
-        raise click.ClickException(FeedbackManager.error_unknown_resource(resource=r["resource"]))
-
-
 def get_remote_resource_name_without_version(remote_resource_name: str) -> str:
     """
     >>> get_remote_resource_name_without_version("r__datasource")
@@ -707,6 +498,7 @@ async def process(
 ):
     name, kind = filename.rsplit(".", 1)
     warnings = []
+    embedded_datasources = {} if embedded_datasources is None else embedded_datasources
 
     try:
         res = await process_file(
@@ -806,31 +598,45 @@ async def get_processed(
     to_run: Optional[Dict[str, Any]] = None,
     vendor_paths: Optional[List[Tuple[str, str]]] = None,
     processed: Optional[Set[str]] = None,
-    tb_client: TinyB = None,
+    tb_client: Optional[TinyB] = None,
     skip_connectors: bool = False,
     current_ws: Optional[Dict[str, Any]] = None,
     fork_downstream: Optional[bool] = False,
     is_internal: Optional[bool] = False,
     dir_path: Optional[str] = None,
-    embedded_datasources: Optional[Dict[str, Any]] = None,
+    embedded_datasources: Optional[Dict[str, Dict[str, Any]]] = None,
 ):
-    if deps is None:
-        deps = []
-    if dep_map is None:
-        dep_map = {}
-    if to_run is None:
-        to_run = {}
-    if processed is None:
-        processed = set()
+    # Initialize with proper type annotations
+    deps_list: List[str] = [] if deps is None else deps
+    dep_map_dict: Dict[str, Any] = {} if dep_map is None else dep_map
+    to_run_dict: Dict[str, Any] = {} if to_run is None else to_run
+    processed_set: Set[str] = set() if processed is None else processed
+    embedded_ds: Dict[str, Dict[str, Any]] = {} if embedded_datasources is None else embedded_datasources
 
     for filename in filenames:
         # just process changed filenames (tb deploy and --only-changes)
-        if changed:
+        if changed is not None:
             resource = Path(filename).resolve().stem
             if resource in changed and (not changed[resource] or changed[resource] in ["shared", "remote"]):
                 continue
         if os.path.isdir(filename):
-            await get_processed(filenames=get_project_filenames(filename))
+            await get_processed(
+                filenames=get_project_filenames(filename),
+                changed=changed,
+                verbose=verbose,
+                deps=deps_list,
+                dep_map=dep_map_dict,
+                to_run=to_run_dict,
+                vendor_paths=vendor_paths,
+                processed=processed_set,
+                tb_client=tb_client,
+                skip_connectors=skip_connectors,
+                current_ws=current_ws,
+                fork_downstream=fork_downstream,
+                is_internal=is_internal,
+                dir_path=dir_path,
+                embedded_datasources=embedded_ds,
+            )
         else:
             if verbose:
                 click.echo(FeedbackManager.info_processing_file(filename=filename))
@@ -838,12 +644,15 @@ async def get_processed(
             if ".incl" in filename:
                 click.echo(FeedbackManager.warning_skipping_include_file(file=filename))
 
+            if tb_client is None:
+                raise ValueError("tb_client cannot be None")
+
             name, warnings = await process(
                 filename=filename,
                 tb_client=tb_client,
-                deps=deps,
-                dep_map=dep_map,
-                to_run=to_run,
+                deps=deps_list,
+                dep_map=dep_map_dict,
+                to_run=to_run_dict,
                 vendor_paths=vendor_paths,
                 skip_connectors=skip_connectors,
                 current_ws=current_ws,
@@ -852,9 +661,9 @@ async def get_processed(
                 is_internal=is_internal,
                 dir_path=dir_path,
                 verbose=verbose,
-                embedded_datasources=embedded_datasources,
+                embedded_datasources=embedded_ds,
             )
-            processed.add(name)
+            processed_set.add(name)
 
             if verbose:
                 if len(warnings) == 1:
@@ -890,7 +699,7 @@ async def build_graph(
     to_run: Dict[str, Any] = {}
     deps: List[str] = []
     dep_map: Dict[str, Any] = {}
-    embedded_datasources = {}
+    embedded_datasources: Dict[str, Dict[str, Any]] = {}
 
     # These dictionaries are used to store all the resources and there dependencies for the whole project
     # This is used for the downstream dependency graph
@@ -919,17 +728,18 @@ async def build_graph(
         all_dep_map = all_dependencies_graph.dep_map
         all_resources = all_dependencies_graph.to_run
 
-    processed = set()
+    processed: Set[str] = set()
 
     await get_processed(
         filenames=filenames,
-        tb_client=tb_client,
         changed=changed,
+        verbose=verbose,
         deps=deps,
         dep_map=dep_map,
         to_run=to_run,
         vendor_paths=vendor_paths,
         processed=processed,
+        tb_client=tb_client,
         skip_connectors=skip_connectors,
         current_ws=current_ws,
         fork_downstream=fork_downstream,
@@ -1198,18 +1008,22 @@ async def process_file(
                     raise Exception(f"Invalid import schedule: '{cron}'. Valid values are: {valid_values}")
 
                 if cron == ON_DEMAND_CRON:
+                    if import_params is None:
+                        import_params = {}
                     import_params["import_schedule"] = ON_DEMAND_CRON_EXPECTED_BY_THE_API
+
                 if cron == AUTO_CRON:
                     period: int = DEFAULT_CRON_PERIOD
 
-                    if current_ws:
+                    if current_ws is not None:
                         workspaces = (await tb_client.user_workspaces()).get("workspaces", [])
                         workspace_rate_limits: Dict[str, Dict[str, int]] = next(
                             (w.get("rate_limits", {}) for w in workspaces if w["id"] == current_ws["id"]), {}
                         )
-                        period = workspace_rate_limits.get("api_datasources_create_append_replace", {}).get(
-                            "period", DEFAULT_CRON_PERIOD
-                        )
+                        if workspace_rate_limits:
+                            rate_limit_config = workspace_rate_limits.get("api_datasources_create_append_replace", {})
+                            if rate_limit_config:
+                                period = rate_limit_config.get("period", DEFAULT_CRON_PERIOD)
 
                     def seconds_to_cron_expression(seconds: int) -> str:
                         minutes = seconds // 60
@@ -1223,10 +1037,13 @@ async def process_file(
                             return f"*/{minutes} * * * *"
                         return f"*/{seconds} * * * *"
 
+                    if import_params is None:
+                        import_params = {}
                     import_params["import_schedule"] = seconds_to_cron_expression(period)
 
-            # Include all import_ parameters in the datasource params
-            params.update(import_params)
+                # Include all import_ parameters in the datasource params
+                if import_params is not None:
+                    params.update(import_params)
 
             # Substitute the import parameters with the ones used by the
             # import API:

tinybird/tb/modules/datasource.py

@@ -30,7 +30,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
 @cli.group()
 @click.pass_context
 def datasource(ctx):
-    """Data Sources commands"""
+    """Data Source commands"""
 
 
 @datasource.command(name="ls")

tinybird/tb/modules/deployment.py

@@ -124,10 +124,95 @@ def deployment_group() -> None:
     help="Auto-promote the deployment. Only works if --wait is enabled. Disabled by default.",
 )
 @click.pass_context
-def create(ctx: click.Context, wait: bool, auto: bool) -> None:
+def deployment_create(ctx: click.Context, wait: bool, auto: bool) -> None:
     """
     Validate and deploy the project server side.
     """
+    create_deployment(ctx, wait, auto)
+
+
+@deployment_group.command(name="ls")
+@click.pass_context
+def deployment_ls(ctx: click.Context) -> None:
+    """
+    List all the deployments you have in the project.
+    """
+    client = ctx.ensure_object(dict)["client"]
+
+    TINYBIRD_API_KEY = client.token
+    HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
+    TINYBIRD_API_URL = f"{client.host}/v1/deployments"
+
+    r = requests.get(TINYBIRD_API_URL, headers=HEADERS)
+    result = r.json()
+    logging.debug(json.dumps(result, indent=2))
+
+    status_map = {"data_ready": "Ready", "failed": "Failed"}
+    columns = ["ID", "Status", "Created at", "Live"]
+    table = []
+    for deployment in result.get("deployments"):
+        table.append(
+            [
+                deployment.get("id"),
+                status_map.get(deployment.get("status"), "In progress"),
+                datetime.fromisoformat(deployment.get("created_at")).strftime("%Y-%m-%d %H:%M:%S"),
+                deployment.get("live"),
+            ]
+        )
+
+    echo_safe_humanfriendly_tables_format_smart_table(table, column_names=columns)
+
+
+@deployment_group.command(name="promote")
+@click.pass_context
+def deployment_promote(ctx: click.Context) -> None:
+    """
+    Promote last deploy to ready and remove old one.
+    """
+    client = ctx.ensure_object(dict)["client"]
+
+    TINYBIRD_API_KEY = client.token
+    HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
+
+    promote_deployment(client.host, HEADERS)
+
+
+@deployment_group.command(name="rollback")
+@click.pass_context
+def deployment_rollback(ctx: click.Context) -> None:
+    """
+    Rollback to the previous deployment.
+    """
+    client = ctx.ensure_object(dict)["client"]
+
+    TINYBIRD_API_KEY = client.token
+    HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
+
+    rollback_deployment(client.host, HEADERS)
+
+
+@cli.command(name="deploy", hidden=True)
+@click.option(
+    "--wait/--no-wait",
+    is_flag=True,
+    default=False,
+    help="Wait for deploy to finish. Disabled by default.",
+)
+@click.option(
+    "--auto/--no-auto",
+    is_flag=True,
+    default=False,
+    help="Auto-promote the deployment. Only works if --wait is enabled. Disabled by default.",
+)
+@click.pass_context
+def deploy(ctx: click.Context, wait: bool, auto: bool) -> None:
+    """
+    Deploy the project.
+    """
+    create_deployment(ctx, wait, auto)
+
+
+def create_deployment(ctx: click.Context, wait: bool, auto: bool) -> None:
     # TODO: This code is duplicated in build_server.py
     # Should be refactored to be shared
     MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
@@ -203,63 +288,3 @@ def create(ctx: click.Context, wait: bool, auto: bool) -> None:
 
         if auto:
             promote_deployment(client.host, HEADERS)
-
-
-@deployment_group.command(name="ls")
-@click.pass_context
-def deployment_ls(ctx: click.Context) -> None:
-    """
-    List all the deployments you have in the project.
-    """
-    client = ctx.ensure_object(dict)["client"]
-
-    TINYBIRD_API_KEY = client.token
-    HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
-    TINYBIRD_API_URL = f"{client.host}/v1/deployments"
-
-    r = requests.get(TINYBIRD_API_URL, headers=HEADERS)
-    result = r.json()
-    logging.debug(json.dumps(result, indent=2))
-
-    status_map = {"data_ready": "Ready", "failed": "Failed"}
-    columns = ["ID", "Status", "Created at", "Live"]
-    table = []
-    for deployment in result.get("deployments"):
-        table.append(
-            [
-                deployment.get("id"),
-                status_map.get(deployment.get("status"), "In progress"),
-                datetime.fromisoformat(deployment.get("created_at")).strftime("%Y-%m-%d %H:%M:%S"),
-                deployment.get("live"),
-            ]
-        )
-
-    echo_safe_humanfriendly_tables_format_smart_table(table, column_names=columns)
-
-
-@deployment_group.command(name="promote")
-@click.pass_context
-def deployment_promote(ctx: click.Context) -> None:
-    """
-    Promote last deploy to ready and remove old one.
-    """
-    client = ctx.ensure_object(dict)["client"]
-
-    TINYBIRD_API_KEY = client.token
-    HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
-
-    promote_deployment(client.host, HEADERS)
-
-
-@deployment_group.command(name="rollback")
-@click.pass_context
-def deployment_rollback(ctx: click.Context) -> None:
-    """
-    Rollback to the previous deployment.
-    """
-    client = ctx.ensure_object(dict)["client"]
-
-    TINYBIRD_API_KEY = client.token
-    HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
-
-    rollback_deployment(client.host, HEADERS)