tinybird 0.0.1.dev5__tar.gz → 0.0.1.dev7__tar.gz

{tinybird-0.0.1.dev5 → tinybird-0.0.1.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird
-Version: 0.0.1.dev5
+Version: 0.0.1.dev7
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

tinybird-0.0.1.dev7/tinybird/__cli__.py

@@ -0,0 +1,7 @@
+__name__ = "tinybird-cli"
+__description__ = "Tinybird Command Line Tool"
+__url__ = "https://www.tinybird.co/docs/cli/introduction.html"
+__author__ = "Tinybird"
+__author_email__ = "support@tinybird.co"
+__version__ = "x.y.z"
+__revision__ = "xxxxxxxxx"

{tinybird-0.0.1.dev5 → tinybird-0.0.1.dev7}/tinybird/datafile.py

@@ -3066,7 +3066,7 @@ async def new_pipe(
                     )
                 )
 
-            click.echo(FeedbackManager.success_test_endpoint(host=host, pipe=p["name"], token="$TB_TOKEN"))
+            click.echo(FeedbackManager.success_test_endpoint_no_token(host=host, pipe=p["name"]))
 
     if copy_node:
         pipe_id = data["id"]
@@ -3125,8 +3125,8 @@ async def new_pipe(
                 except Exception as e:
                     raise click.ClickException(FeedbackManager.error_creating_pipe(error=e))
 
-        if data.get("type") == "endpoint" and t:
-            click.echo(FeedbackManager.success_test_endpoint(host=host, pipe=p["name"], token=t["token"]))
+        if data.get("type") == "endpoint":
+            click.echo(FeedbackManager.success_test_endpoint_no_token(host=host, pipe=p["name"]))
 
 
 async def share_and_unshare_datasource(
@@ -3217,7 +3217,6 @@ async def new_ds(
     fork_downstream: Optional[bool] = False,
     fork: Optional[bool] = False,
     git_release: Optional[bool] = False,
-    build: Optional[bool] = False,
 ):
     ds_name = ds["params"]["name"]
 
@@ -3240,14 +3239,9 @@ async def new_ds(
                     scopes.append(sc)
                 await client.alter_tokens(token_name, scopes)
 
-    datasource_exists = False
     try:
         existing_ds = await client.get_datasource(ds_name)
-        if build:
-            await client.datasource_delete(ds_name, force=True)
-        else:
-            datasource_exists = True
-
+        datasource_exists = True
     except DoesNotExistException:
         datasource_exists = False
 
@@ -3580,7 +3574,6 @@ async def exec_file(
     fork_downstream: Optional[bool] = False,
     fork: Optional[bool] = False,
     git_release: Optional[bool] = False,
-    build: Optional[bool] = False,
 ):
     if debug:
         click.echo(FeedbackManager.debug_running_file(file=pp.pformat(r)))
@@ -3623,7 +3616,6 @@ async def exec_file(
             fork_downstream=fork_downstream,
             fork=fork,
             git_release=git_release,
-            build=build,
         )
         await update_tags_in_resource(r, "datasource", tb_client)
     elif r["resource"] == "tokens":
@@ -3908,7 +3900,6 @@ async def build_graph(
     only_changes: bool = False,
     fork_downstream: Optional[bool] = False,
     is_internal: Optional[bool] = False,
-    build: Optional[bool] = False,
 ) -> GraphDependencies:
     """
     This method will generate a dependency graph for the given files. It will also return a map of all the resources that are going to be deployed.
@@ -3945,7 +3936,6 @@ async def build_graph(
             changed=None,
             only_changes=False,
             is_internal=is_internal,
-            build=build,
         )
         all_dep_map = all_dependencies_graph.dep_map
         all_resources = all_dependencies_graph.to_run
@@ -4132,10 +4122,8 @@ def get_project_filenames(folder: str, with_vendor=False) -> List[str]:
         f"{folder}/*.pipe",
         f"{folder}/pipes/*.pipe",
         f"{folder}/endpoints/*.pipe",
-        f"{folder}/materializations/*.pipe",
-        f"{folder}/sinks/*.pipe",
-        f"{folder}/copies/*.pipe",
-        f"{folder}/playgrounds/*.pipe",
+        f"{folder}/*.token",
+        f"{folder}/tokens/*.token",
     ]
     if with_vendor:
         folders.append(f"{folder}/vendor/**/**/*.datasource")
@@ -4360,7 +4348,6 @@ async def folder_push(
         dry_run: bool,
         fork_downstream: Optional[bool] = False,
         fork: Optional[bool] = False,
-        build: Optional[bool] = False,
     ):
         if name in to_run:
             resource = to_run[name]["resource"]
@@ -4410,7 +4397,6 @@ async def folder_push(
                             fork_downstream,
                             fork,
                             git_release,
-                            build,
                         )
                         if not run_tests:
                             click.echo(
@@ -4460,13 +4446,7 @@ async def folder_push(
                         version = ""
                         if name in latest_datasource_versions:
                             version = f"(v{latest_datasource_versions[name]})"
-                        if build:
-                            extension = "pipe" if resource == "pipes" else "datasource"
-                            click.echo(
-                                FeedbackManager.info_building_resource(name=f"{name}.{extension}", version=version)
-                            )
-                        else:
-                            click.echo(FeedbackManager.info_dry_processing_new_resource(name=name, version=version))
+                        click.echo(FeedbackManager.info_dry_processing_new_resource(name=name, version=version))
                     else:
                         click.echo(
                             FeedbackManager.info_dry_processing_resource(
@@ -5607,517 +5587,3 @@ def is_file_a_datasource(filename: str) -> bool:
                 return True
 
     return False
-
-
-async def folder_build(
-    tb_client: TinyB,
-    workspaces: List[Dict[str, Any]],
-    datasources: List[Dict[str, Any]],
-    pipes: List[Dict[str, Any]],
-    filenames: Optional[List[str]] = None,
-    folder: str = ".",
-    ignore_sql_errors: bool = False,
-    is_internal: bool = False,
-    only_pipes: bool = False,
-):
-    if only_pipes:
-        filenames = [f for f in filenames if f.endswith(".pipe")]
-
-    config = CLIConfig.get_project_config()
-    build = True
-    dry_run = False
-    force = True
-    push_deps = False
-    only_changes = False
-    debug = False
-    check = True
-    populate = False
-    populate_subset = None
-    populate_condition = None
-    tests_to_run = 0
-    user_token = None
-    tests_failfast = True
-    override_datasource = True
-    tests_check_requests_from_branch = False
-    skip_confirmation = True
-    wait = False
-    unlink_on_populate_error = False
-    upload_fixtures = False
-    only_response_times = False
-    workspace_map: Dict[str, Any] = {}
-    tests_sample_by_params = 1
-    tests_ignore_order = False
-    tests_validate_processed_bytes = False
-    run_tests = False
-    verbose = False
-    as_standard = False
-    raise_on_exists = False
-    fork_downstream = False
-    fork = False
-    release_created = False
-    auto_promote = False
-    hide_folders = False
-    tests_relative_change = 0.01
-    tests_sample_by_params = 0
-    tests_filter_by = None
-    tests_failfast = False
-    tests_ignore_order = False
-    tests_validate_processed_bytes = False
-    tests_check_requests_from_branch = False
-    git_release = False
-    workspace_lib_paths = []
-    current_ws: Dict[str, Any] = next(
-        (workspace for workspace in workspaces if config and workspace.get("id", ".") == config.get("id", "..")), {}
-    )
-
-    workspace_lib_paths = list(workspace_lib_paths)
-    # include vendor libs without overriding user ones
-    existing_workspaces = set(x[1] for x in workspace_lib_paths)
-    vendor_path = Path("vendor")
-    if vendor_path.exists():
-        for x in vendor_path.iterdir():
-            if x.is_dir() and x.name not in existing_workspaces:
-                workspace_lib_paths.append((x.name, x))
-
-    datasources: List[Dict[str, Any]] = await tb_client.datasources()
-    pipes: List[Dict[str, Any]] = await tb_client.pipes(dependencies=True)
-
-    existing_resources: List[str] = [x["name"] for x in datasources] + [x["name"] for x in pipes]
-    # replace workspace mapping names
-    for old_ws, new_ws in workspace_map.items():
-        existing_resources = [re.sub(f"^{old_ws}\.", f"{new_ws}.", x) for x in existing_resources]
-
-    remote_resource_names = [get_remote_resource_name_without_version(x) for x in existing_resources]
-
-    # replace workspace mapping names
-    for old_ws, new_ws in workspace_map.items():
-        remote_resource_names = [re.sub(f"^{old_ws}\.", f"{new_ws}.", x) for x in remote_resource_names]
-
-    if not filenames:
-        filenames = get_project_filenames(folder)
-
-    changed = None
-
-    # build graph to get new versions for all the files involved in the query
-    # dependencies need to be processed always to get the versions
-    dependencies_graph = await build_graph(
-        filenames,
-        tb_client,
-        dir_path=folder,
-        process_dependencies=True,
-        workspace_map=workspace_map,
-        skip_connectors=True,
-        workspace_lib_paths=workspace_lib_paths,
-        current_ws=current_ws,
-        changed=changed,
-        only_changes=only_changes,
-        fork_downstream=fork_downstream,
-        is_internal=is_internal,
-        build=build,
-    )
-
-    resource_versions = {}
-    latest_datasource_versions = {}
-
-    # If we have datasources using VERSION, let's try to get the latest version
-    dependencies_graph = await build_graph(
-        filenames,
-        tb_client,
-        dir_path=folder,
-        resource_versions=latest_datasource_versions,
-        workspace_map=workspace_map,
-        process_dependencies=push_deps,
-        verbose=verbose,
-        workspace_lib_paths=workspace_lib_paths,
-        current_ws=current_ws,
-        changed=changed,
-        only_changes=only_changes,
-        fork_downstream=fork_downstream,
-        is_internal=is_internal,
-        build=build,
-    )
-
-    if debug:
-        pp.pprint(dependencies_graph.to_run)
-
-    def should_push_file(
-        name: str,
-        remote_resource_names: List[str],
-        latest_datasource_versions: Dict[str, Any],
-        force: bool,
-        run_tests: bool,
-    ) -> bool:
-        """
-        Function to know if we need to run a file or not
-        """
-        if name not in remote_resource_names:
-            return True
-        # When we need to try to push a file when it doesn't exist and the version is different that the existing one
-        resource_full_name = (
-            f"{name}__v{latest_datasource_versions.get(name)}" if name in latest_datasource_versions else name
-        )
-        if resource_full_name not in existing_resources:
-            return True
-        if force or run_tests:
-            return True
-        return False
-
-    async def push(
-        name: str,
-        to_run: Dict[str, Dict[str, Any]],
-        resource_versions: Dict[str, Any],
-        latest_datasource_versions: Dict[str, Any],
-        dry_run: bool,
-        fork_downstream: Optional[bool] = False,
-        fork: Optional[bool] = False,
-    ):
-        if name in to_run:
-            resource = to_run[name]["resource"]
-            if not dry_run:
-                if should_push_file(name, remote_resource_names, latest_datasource_versions, force, run_tests):
-                    if name not in resource_versions:
-                        version = ""
-                        if name in latest_datasource_versions:
-                            version = f"(v{latest_datasource_versions[name]})"
-                        click.echo(FeedbackManager.info_processing_new_resource(name=name, version=version))
-                    else:
-                        click.echo(
-                            FeedbackManager.info_processing_resource(
-                                name=name,
-                                version=latest_datasource_versions[name],
-                                latest_version=resource_versions.get(name),
-                            )
-                        )
-                    try:
-                        await exec_file(
-                            to_run[name],
-                            tb_client,
-                            force,
-                            check,
-                            debug and verbose,
-                            populate,
-                            populate_subset,
-                            populate_condition,
-                            unlink_on_populate_error,
-                            wait,
-                            user_token,
-                            override_datasource,
-                            ignore_sql_errors,
-                            skip_confirmation,
-                            only_response_times,
-                            run_tests,
-                            as_standard,
-                            tests_to_run,
-                            tests_relative_change,
-                            tests_sample_by_params,
-                            tests_filter_by,
-                            tests_failfast,
-                            tests_ignore_order,
-                            tests_validate_processed_bytes,
-                            tests_check_requests_from_branch,
-                            current_ws,
-                            fork_downstream,
-                            fork,
-                            git_release,
-                            build,
-                        )
-                        if not run_tests:
-                            click.echo(
-                                FeedbackManager.success_create(
-                                    name=(
-                                        name
-                                        if to_run[name]["version"] is None
-                                        else f'{name}__v{to_run[name]["version"]}'
-                                    )
-                                )
-                            )
-                    except Exception as e:
-                        filename = (
-                            os.path.basename(to_run[name]["filename"]) if hide_folders else to_run[name]["filename"]
-                        )
-                        exception = FeedbackManager.error_push_file_exception(
-                            filename=filename,
-                            error=e,
-                        )
-                        raise click.ClickException(exception)
-                else:
-                    if raise_on_exists:
-                        raise AlreadyExistsException(
-                            FeedbackManager.warning_name_already_exists(
-                                name=name if to_run[name]["version"] is None else f'{name}__v{to_run[name]["version"]}'
-                            )
-                        )
-                    else:
-                        if await name_matches_existing_resource(resource, name, tb_client):
-                            if resource == "pipes":
-                                click.echo(FeedbackManager.error_pipe_cannot_be_pushed(name=name))
-                            else:
-                                click.echo(FeedbackManager.error_datasource_cannot_be_pushed(name=name))
-                        else:
-                            click.echo(
-                                FeedbackManager.warning_name_already_exists(
-                                    name=(
-                                        name
-                                        if to_run[name]["version"] is None
-                                        else f'{name}__v{to_run[name]["version"]}'
-                                    )
-                                )
-                            )
-            else:
-                if should_push_file(name, remote_resource_names, latest_datasource_versions, force, run_tests):
-                    if name not in resource_versions:
-                        version = ""
-                        if name in latest_datasource_versions:
-                            version = f"(v{latest_datasource_versions[name]})"
-                        if build:
-                            extension = "pipe" if resource == "pipes" else "datasource"
-                            click.echo(
-                                FeedbackManager.info_building_resource(name=f"{name}.{extension}", version=version)
-                            )
-                        else:
-                            click.echo(FeedbackManager.info_dry_processing_new_resource(name=name, version=version))
-                    else:
-                        click.echo(
-                            FeedbackManager.info_dry_processing_resource(
-                                name=name,
-                                version=latest_datasource_versions[name],
-                                latest_version=resource_versions.get(name),
-                            )
-                        )
-                else:
-                    if await name_matches_existing_resource(resource, name, tb_client):
-                        if resource == "pipes":
-                            click.echo(FeedbackManager.warning_pipe_cannot_be_pushed(name=name))
-                        else:
-                            click.echo(FeedbackManager.warning_datasource_cannot_be_pushed(name=name))
-                    else:
-                        click.echo(FeedbackManager.warning_dry_name_already_exists(name=name))
-
-    async def push_files(
-        dependency_graph: GraphDependencies,
-        dry_run: bool = False,
-        check_backfill_required: bool = False,
-    ):
-        endpoints_dep_map = dict()
-        processed = set()
-
-        dependencies_graph = dependency_graph.dep_map
-        resources_to_run = dependency_graph.to_run
-
-        if not fork_downstream:
-            # First, we will deploy the all the resources following the dependency graph except for the endpoints
-            groups = [group for group in toposort(dependencies_graph)]
-            for group in groups:
-                for name in group:
-                    if name in processed:
-                        continue
-
-                    if is_endpoint_with_no_dependencies(
-                        resources_to_run.get(name, {}),
-                        dependencies_graph,
-                        resources_to_run,
-                    ):
-                        endpoints_dep_map[name] = dependencies_graph[name]
-                        continue
-
-                    await push(
-                        name,
-                        resources_to_run,
-                        resource_versions,
-                        latest_datasource_versions,
-                        dry_run,
-                        fork_downstream,
-                        fork,
-                    )
-                    processed.add(name)
-
-            # Then, we will deploy the endpoints that are on the dependency graph
-            groups = [group for group in toposort(endpoints_dep_map)]
-            for group in groups:
-                for name in group:
-                    if name not in processed:
-                        await push(
-                            name,
-                            resources_to_run,
-                            resource_versions,
-                            latest_datasource_versions,
-                            dry_run,
-                            fork_downstream,
-                            fork,
-                        )
-                        processed.add(name)
-        else:
-            # This will generate the graph from right to left and will fill the gaps of the dependencies
-            # If we have a graph like this:
-            # A -> B -> C
-            # If we only modify A, the normal dependencies graph will only contain a node like _{A => B}
-            # But we need a graph that contains A, B and C and the dependencies between them to deploy them in the right order
-            dependencies_graph_fork_downstream, resources_to_run_fork_downstream = generate_forkdownstream_graph(
-                dependency_graph.all_dep_map,
-                dependency_graph.all_resources,
-                resources_to_run,
-                list(dependency_graph.dep_map.keys()),
-            )
-
-            # First, we will deploy the datasources that need to be deployed.
-            # We need to deploy the datasources from left to right as some datasources might have MV that depend on the column types of previous datasources. Ex: `test_change_column_type_landing_datasource` test
-            groups = [group for group in toposort(dependencies_graph_fork_downstream)]
-            groups.reverse()
-            for group in groups:
-                for name in group:
-                    if name in processed or not is_datasource(resources_to_run_fork_downstream[name]):
-                        continue
-
-                    # If the resource is new, we will use the normal resource information to deploy it
-                    # This is mostly used for datasources with connections.
-                    # At the moment, `resources_to_run_fork_downstream` is generated by `all_resources` and this is generated using the parameter `skip_connectors=True`
-                    # TODO: Should the `resources_to_run_fork_downstream` be generated using the `skip_connectors` parameter?
-                    if is_new(name, changed, dependencies_graph_fork_downstream, dependencies_graph_fork_downstream):
-                        await push(
-                            name,
-                            resources_to_run,
-                            resource_versions,
-                            latest_datasource_versions,
-                            dry_run,
-                            fork_downstream,
-                            fork,
-                        )
-                    else:
-                        # If we are trying to modify a Kafka or CDK datasource, we need to inform the user that the resource needs to be post-released
-                        kafka_connection_name = (
-                            resources_to_run_fork_downstream[name].get("params", {}).get("kafka_connection_name")
-                        )
-                        service = resources_to_run_fork_downstream[name].get("params", {}).get("import_service")
-                        if release_created and (kafka_connection_name or service):
-                            connector = "Kafka" if kafka_connection_name else service
-                            error_msg = FeedbackManager.error_connector_require_post_release(connector=connector)
-                            raise click.ClickException(error_msg)
-
-                        # If we are pushing a modified datasource, inform about the backfill``
-                        if check_backfill_required and auto_promote and release_created:
-                            error_msg = FeedbackManager.error_check_backfill_required(resource_name=name)
-                            raise click.ClickException(error_msg)
-
-                        await push(
-                            name,
-                            resources_to_run_fork_downstream,
-                            resource_versions,
-                            latest_datasource_versions,
-                            dry_run,
-                            fork_downstream,
-                            fork,
-                        )
-                    processed.add(name)
-
-            # Now, we will create a map of all the endpoints and there dependencies
-            # We are using the forkdownstream graph to get the dependencies of the endpoints as the normal dependencies graph only contains the resources that are going to be deployed
-            # But does not include the missing gaps
-            # If we have ENDPOINT_A ----> MV_PIPE_B -----> DATASOURCE_B ------> ENDPOINT_C
-            # Where endpoint A is being used in the MV_PIPE_B, if we only modify the endpoint A
-            # The dependencies graph will only contain the endpoint A and the MV_PIPE_B, but not the DATASOURCE_B and the ENDPOINT_C
-            groups = [group for group in toposort(dependencies_graph_fork_downstream)]
-            for group in groups:
-                for name in group:
-                    if name in processed or not is_endpoint(resources_to_run_fork_downstream[name]):
-                        continue
-
-                    endpoints_dep_map[name] = dependencies_graph_fork_downstream[name]
-
-            # Now that we have the dependencies of the endpoints, we need to check that the resources has not been deployed yet and only care about the endpoints that depend on endpoints
-            groups = [group for group in toposort(endpoints_dep_map)]
-
-            # As we have used the forkdownstream graph to get the dependencies of the endpoints, we have all the dependencies of the endpoints
-            # But we need to deploy the endpoints and the dependencies of the endpoints from left to right
-            # So we need to reverse the groups
-            groups.reverse()
-            for group in groups:
-                for name in group:
-                    if name in processed or not is_endpoint(resources_to_run_fork_downstream[name]):
-                        continue
-
-                    await push(
-                        name,
-                        resources_to_run_fork_downstream,
-                        resource_versions,
-                        latest_datasource_versions,
-                        dry_run,
-                        fork_downstream,
-                        fork,
-                    )
-                    processed.add(name)
-
-            # Now we should have the endpoints and datasources deployed, we can deploy the rest of the pipes (copy & sinks)
-            # We need to rely on the forkdownstream graph as it contains all the modified pipes as well as the dependencies of the pipes
-            # In this case, we don't need to generate a new graph as we did for the endpoints as the pipes are not going to be used as dependencies and the datasources are already deployed
-            groups = [group for group in toposort(dependencies_graph_fork_downstream)]
-            for group in groups:
-                for name in group:
-                    if name in processed or is_materialized(resources_to_run_fork_downstream.get(name)):
-                        continue
-
-                    await push(
-                        name,
-                        resources_to_run_fork_downstream,
-                        resource_versions,
-                        latest_datasource_versions,
-                        dry_run,
-                        fork_downstream,
-                        fork,
-                    )
-                    processed.add(name)
-
-            # Finally, we need to deploy the materialized views from right to left.
-            # We need to rely on the forkdownstream graph as it contains all the modified materialized views as well as the dependencies of the materialized views
-            # In this case, we don't need to generate a new graph as we did for the endpoints as the pipes are not going to be used as dependencies and the datasources are already deployed
-            groups = [group for group in toposort(dependencies_graph_fork_downstream)]
-            for group in groups:
-                for name in group:
-                    if name in processed or not is_materialized(resources_to_run_fork_downstream.get(name)):
-                        continue
-
-                    await push(
-                        name,
-                        resources_to_run_fork_downstream,
-                        resource_versions,
-                        latest_datasource_versions,
-                        dry_run,
-                        fork_downstream,
-                        fork,
-                    )
-                    processed.add(name)
-
-    await push_files(dependencies_graph, dry_run)
-
-    if not dry_run and not run_tests:
-        if upload_fixtures:
-            click.echo(FeedbackManager.info_pushing_fixtures())
-
-            processed = set()
-            for group in toposort(dependencies_graph.dep_map):
-                for f in group:
-                    name = os.path.basename(f)
-                    if name not in processed and name in dependencies_graph.to_run:
-                        await check_fixtures_data(
-                            tb_client,
-                            dependencies_graph.to_run[name],
-                            debug,
-                            folder,
-                            force,
-                            mode="replace",
-                        )
-                        processed.add(name)
-            for f in dependencies_graph.to_run:
-                if f not in processed:
-                    await check_fixtures_data(
-                        tb_client,
-                        dependencies_graph.to_run[f],
-                        debug,
-                        folder,
-                        force,
-                        mode="replace",
-                    )
-        else:
-            if verbose:
-                click.echo(FeedbackManager.info_not_pushing_fixtures())
-
-    return dependencies_graph.to_run

tinybird-0.0.1.dev7/tinybird/tb/cli.py

@@ -0,0 +1,28 @@
+import asyncio
+import sys
+
+if sys.platform == "win32":
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+
+import tinybird.tb.modules.auth
+import tinybird.tb.modules.branch
+import tinybird.tb.modules.build
+import tinybird.tb.modules.cli
+import tinybird.tb.modules.common
+import tinybird.tb.modules.connection
+import tinybird.tb.modules.create
+import tinybird.tb.modules.datasource
+import tinybird.tb.modules.fmt
+import tinybird.tb.modules.job
+import tinybird.tb.modules.mock
+import tinybird.tb.modules.pipe
+import tinybird.tb.modules.tag
+import tinybird.tb.modules.test
+import tinybird.tb.modules.token
+import tinybird.tb.modules.workspace
+import tinybird.tb.modules.workspace_members
+
+cli = tinybird.tb.modules.cli.cli
+
+if __name__ == "__main__":
+    cli()