tinybird 0.0.1.dev234__py3-none-any.whl → 0.0.1.dev235__py3-none-any.whl

tinybird/tb/modules/datafile/diff.py

@@ -31,7 +31,7 @@ from tinybird.tb.modules.datafile.pull import folder_pull
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 
-async def diff_files(
+def diff_files(
     from_file: str,
     to_file: str,
     from_file_suffix: str = "[remote]",
@@ -45,12 +45,12 @@ async def diff_files(
         with open(filename) as file:
             return file.readlines()
 
-    async def parse(filename, with_format=True, unroll_includes=False):
+    def parse(filename, with_format=True, unroll_includes=False):
         extensions = Path(filename).suffixes
         lines = None
         if is_file_a_datasource(filename):
             lines = (
-                await format_datasource(
+                format_datasource(
                     filename,
                     unroll_includes=unroll_includes,
                     for_diff=True,
@@ -63,7 +63,7 @@ async def diff_files(
             )
         elif (".pipe" in extensions) or (".incl" in extensions):
             lines = (
-                await format_pipe(
+                format_pipe(
                     filename,
                     DEFAULT_FMT_LINE_LENGTH,
                     unroll_includes=unroll_includes,
@@ -79,8 +79,8 @@ async def diff_files(
             return [f"{l}\n" for l in lines.split("\n")] if with_format else lines  # noqa: E741
 
     try:
-        lines1 = await parse(from_file, with_format)
-        lines2 = await parse(to_file, with_format, unroll_includes=True)
+        lines1 = parse(from_file, with_format)
+        lines2 = parse(to_file, with_format, unroll_includes=True)
     except FileNotFoundError as e:
         filename = os.path.basename(str(e)).strip("'")
         raise click.ClickException(FeedbackManager.error_diff_file(filename=filename))
@@ -98,7 +98,7 @@ async def diff_files(
     return diff
 
 
-async def diff_command(
+def diff_command(
     filenames: Optional[List[str]],
     fmt: bool,
     client: TinyB,
@@ -121,15 +121,15 @@ async def diff_command(
     if filenames:
         if len(filenames) == 1:
             filenames = [filenames[0], *get_project_filenames(filenames[0])]
-        await folder_pull(client, target_dir, True, verbose=False)
+        folder_pull(client, target_dir, True, verbose=False)
     else:
         filenames = get_project_filenames(".")
         if verbose:
             click.echo("Saving remote resources in .diff_tmp folder.\n")
-        await folder_pull(client, target_dir, True, verbose=verbose, progress_bar=progress_bar)
+        folder_pull(client, target_dir, True, verbose=verbose, progress_bar=progress_bar)
 
-    remote_datasources: List[Dict[str, Any]] = await client.datasources()
-    remote_pipes: List[Dict[str, Any]] = await client.pipes()
+    remote_datasources: List[Dict[str, Any]] = client.datasources()
+    remote_pipes: List[Dict[str, Any]] = client.pipes()
 
     local_resources = {
         Path(file).resolve().stem: file
@@ -159,7 +159,7 @@ async def diff_command(
             suffix = ".datasource" if ".datasource" in file else ".pipe"
             target = target_dir + os.path.sep + rfilename + suffix
 
-            diff_lines = await diff_files(
+            diff_lines = diff_files(
                 target, file, with_format=fmt, with_color=(not no_color), client=client, for_deploy=for_deploy
             )
             not_empty, diff_lines = peek(diff_lines)

tinybird/tb/modules/datafile/format_datasource.py

@@ -15,7 +15,7 @@ from tinybird.tb.modules.datafile.format_common import (
 )
 
 
-async def format_datasource(
+def format_datasource(
     filename: str,
     unroll_includes: bool = False,
     for_diff: bool = False,
@@ -57,7 +57,7 @@ async def format_datasource(
         format_tags(file_parts, doc)
         format_schema(file_parts, doc.nodes[0])
         format_indices(file_parts, doc.nodes[0])
-        await format_engine(file_parts, doc.nodes[0], only_ttl=bool(not for_deploy_diff), client=client)
+        format_engine(file_parts, doc.nodes[0], only_ttl=bool(not for_deploy_diff), client=client)
         if for_deploy_diff:
             format_import_settings(file_parts, doc.nodes[0])
         format_shared_with(file_parts, doc)
@@ -68,7 +68,7 @@ async def format_datasource(
         format_tags(file_parts, doc)
         format_schema(file_parts, doc.nodes[0])
         format_indices(file_parts, doc.nodes[0])
-        await format_engine(file_parts, doc.nodes[0])
+        format_engine(file_parts, doc.nodes[0])
         format_include(file_parts, doc, unroll_includes=unroll_includes)
         format_data_connector(file_parts, doc.nodes[0])
         format_import_settings(file_parts, doc.nodes[0])
@@ -127,7 +127,7 @@ def format_shared_with(file_parts: List[str], doc: Datafile) -> List[str]:
     return file_parts
 
 
-async def format_engine(
+def format_engine(
     file_parts: List[str], node: Dict[str, Any], only_ttl: bool = False, client: Optional[TinyB] = None
 ) -> List[str]:
     if only_ttl:
@@ -137,7 +137,7 @@ async def format_engine(
                     elem = ", ".join([x.strip() for x in arg[1].split(",")])
                     try:
                         if client:
-                            ttl_sql = await client.sql_get_format(f"select {elem}", with_clickhouse_format=True)
+                            ttl_sql = client.sql_get_format(f"select {elem}", with_clickhouse_format=True)
                             formatted_ttl = ttl_sql[7:]
                         else:
                             formatted_ttl = elem

tinybird/tb/modules/datafile/format_pipe.py

@@ -35,7 +35,7 @@ def format_node_sql(
     return file_parts
 
 
-async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
+def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
     node_type = node.get("type", "").lower()
     node_type_upper = f"TYPE {node_type.upper()}"
     # Materialized pipe
@@ -44,7 +44,7 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
         file_parts.append(DATAFILE_NEW_LINE)
         file_parts.append(f"DATASOURCE {node['datasource']}")
         file_parts.append(DATAFILE_NEW_LINE)
-        await format_engine(file_parts, node)
+        format_engine(file_parts, node)
 
     # Copy pipe
     if node_type == PipeNodeTypes.COPY:
@@ -98,7 +98,7 @@ def format_sql(sql: str, DATAFILE_INDENT: str, line_length: Optional[int] = None
     return "\n".join([f"{DATAFILE_INDENT}{part}" for part in sql.split("\n") if len(part.strip())])
 
 
-async def format_node(
+def format_node(
     file_parts: List[str],
     node: Dict[str, Any],
     includes: Dict[str, Any],
@@ -120,10 +120,10 @@ async def format_node(
     Doc = namedtuple("Doc", ["description"])
     format_description(file_parts, Doc(node.get("description", "")))
     format_node_sql(file_parts, node, line_length=line_length, lower_keywords=lower_keywords)
-    await format_node_type(file_parts, node)
+    format_node_type(file_parts, node)
 
 
-async def format_pipe(
+def format_pipe(
     filename: str,
     line_length: Optional[int] = DEFAULT_FMT_LINE_LENGTH,
     unroll_includes: bool = False,
@@ -171,7 +171,7 @@ async def format_pipe(
                 if unrolled_included_node:
                     doc.nodes.remove(unrolled_included_node)
     for node in doc.nodes:
-        await format_node(
+        format_node(
             file_parts,
             node,
             doc.includes,

tinybird/tb/modules/datafile/playground.py

@@ -47,7 +47,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.project import Project
 
 
-async def folder_playground(
+def folder_playground(
     project: Project,
     config: CLIConfig,
     tb_client: TinyB,
@@ -68,8 +68,8 @@ async def folder_playground(
     fork = False
     release_created = False
     folder = str(project.path)
-    datasources: List[Dict[str, Any]] = await tb_client.datasources()
-    pipes: List[Dict[str, Any]] = await tb_client.pipes(dependencies=True)
+    datasources: List[Dict[str, Any]] = tb_client.datasources()
+    pipes: List[Dict[str, Any]] = tb_client.pipes(dependencies=True)
     build = True
     dry_run = False
     force = True
@@ -112,7 +112,7 @@ async def folder_playground(
 
     # build graph to get new versions for all the files involved in the query
     # dependencies need to be processed always to get the versions
-    dependencies_graph = await build_graph(
+    dependencies_graph = build_graph(
         filenames,
         tb_client,
         dir_path=folder,
@@ -148,7 +148,7 @@ async def folder_playground(
             return True
         return False
 
-    async def push(
+    def push(
         name: str,
         to_run: Dict[str, Dict[str, Any]],
         dry_run: bool,
@@ -163,7 +163,7 @@ async def folder_playground(
                 if should_push_file(name, remote_resource_names, force, run_tests):
                     click.echo(FeedbackManager.info_processing_new_resource(name=name, version=""))
                     try:
-                        await exec_file(
+                        exec_file(
                             to_run[name],
                             config,
                             tb_client,
@@ -222,7 +222,7 @@ async def folder_playground(
                             )
                         )
                     else:
-                        if await name_matches_existing_resource(resource, name, tb_client):
+                        if name_matches_existing_resource(resource, name, tb_client):
                             if resource == "pipes":
                                 click.echo(FeedbackManager.error_pipe_cannot_be_pushed(name=name))
                             else:
@@ -242,7 +242,7 @@ async def folder_playground(
                     extension = "pipe" if resource == "pipes" else "datasource"
                     click.echo(FeedbackManager.info_building_resource(name=f"{name}.{extension}", version=""))
                 else:
-                    if await name_matches_existing_resource(resource, name, tb_client):
+                    if name_matches_existing_resource(resource, name, tb_client):
                         if resource == "pipes":
                             click.echo(FeedbackManager.warning_pipe_cannot_be_pushed(name=name))
                         else:
@@ -250,7 +250,7 @@ async def folder_playground(
                     else:
                         click.echo(FeedbackManager.warning_dry_name_already_exists(name=name))
 
-    async def push_files(
+    def push_files(
         dependency_graph: GraphDependencies,
         dry_run: bool = False,
     ):
@@ -291,7 +291,7 @@ async def folder_playground(
                     error_msg = FeedbackManager.error_connector_require_post_release(connector=connector)
                     raise click.ClickException(error_msg)
 
-                await push(
+                push(
                     name,
                     resources_to_run_fork_downstream,
                     dry_run,
@@ -326,7 +326,7 @@ async def folder_playground(
                 if name in processed or not is_endpoint(resources_to_run_fork_downstream[name]):
                     continue
 
-                await push(
+                push(
                     name,
                     resources_to_run_fork_downstream,
                     dry_run,
@@ -344,7 +344,7 @@ async def folder_playground(
                 if name in processed or is_materialized(resources_to_run_fork_downstream.get(name)):
                     continue
 
-                await push(
+                push(
                     name,
                     resources_to_run_fork_downstream,
                     dry_run,
@@ -362,7 +362,7 @@ async def folder_playground(
                 if name in processed or not is_materialized(resources_to_run_fork_downstream.get(name)):
                     continue
 
-                await push(
+                push(
                     name,
                     resources_to_run_fork_downstream,
                     dry_run,
@@ -371,7 +371,7 @@ async def folder_playground(
                 )
                 processed.add(name)
 
-    await push_files(dependencies_graph, dry_run)
+    push_files(dependencies_graph, dry_run)
 
     if not dry_run and not run_tests and verbose:
         click.echo(FeedbackManager.info_not_pushing_fixtures())
@@ -379,19 +379,19 @@ async def folder_playground(
     return dependencies_graph.to_run
 
 
-async def name_matches_existing_resource(resource: str, name: str, tb_client: TinyB):
+def name_matches_existing_resource(resource: str, name: str, tb_client: TinyB):
     if resource == "datasources":
-        current_pipes: List[Dict[str, Any]] = await tb_client.pipes()
+        current_pipes: List[Dict[str, Any]] = tb_client.pipes()
         if name in [x["name"] for x in current_pipes]:
             return True
     else:
-        current_datasources: List[Dict[str, Any]] = await tb_client.datasources()
+        current_datasources: List[Dict[str, Any]] = tb_client.datasources()
         if name in [x["name"] for x in current_datasources]:
             return True
     return False
 
 
-async def exec_file(
+def exec_file(
     r: Dict[str, Any],
     config: CLIConfig,
     tb_client: TinyB,
@@ -428,7 +428,7 @@ async def exec_file(
     if debug:
         click.echo(FeedbackManager.debug_running_file(file=pp.pformat(r)))
     if r["resource"] == "pipes":
-        await new_pipe(
+        new_pipe(
             r,
             config,
             tb_client,
@@ -636,7 +636,7 @@ class GraphDependencies:
     all_resources: Dict[str, Dict[str, Any]]
 
 
-async def process(
+def process(
     filename: str,
     tb_client: TinyB,
     deps: List[str],
@@ -657,7 +657,7 @@ async def process(
     embedded_datasources = {} if embedded_datasources is None else embedded_datasources
 
     try:
-        res = await process_file(
+        res = process_file(
             filename,
             tb_client,
             skip_connectors=skip_connectors,
@@ -745,7 +745,7 @@ async def process(
     return os.path.basename(name), warnings
 
 
-async def get_processed(
+def get_processed(
     filenames: Iterable[str],
     changed: Optional[Dict[str, Any]] = None,
     verbose: bool = False,
@@ -776,7 +776,7 @@ async def get_processed(
             if resource in changed and (not changed[resource] or changed[resource] in ["shared", "remote"]):
                 continue
         if os.path.isdir(filename):
-            await get_processed(
+            get_processed(
                 filenames=get_project_filenames(filename),
                 changed=changed,
                 verbose=verbose,
@@ -803,7 +803,7 @@ async def get_processed(
             if tb_client is None:
                 raise ValueError("tb_client cannot be None")
 
-            name, warnings = await process(
+            name, warnings = process(
                 filename=filename,
                 tb_client=tb_client,
                 deps=deps_list,
@@ -833,7 +833,7 @@ async def get_processed(
                     )
 
 
-async def build_graph(
+def build_graph(
     filenames: Iterable[str],
     tb_client: TinyB,
     dir_path: Optional[str] = None,
@@ -868,7 +868,7 @@ async def build_graph(
     # When using fork-downstream or --only-changes, we need to generate all the graph of all the resources and their dependencies
     # This way we can add more resources into the to_run dictionary if needed.
     if process_dependencies and only_changes:
-        all_dependencies_graph = await build_graph(
+        all_dependencies_graph = build_graph(
             get_project_filenames(dir_path),
             tb_client,
             dir_path=dir_path,
@@ -886,7 +886,7 @@ async def build_graph(
 
     processed: Set[str] = set()
 
-    await get_processed(
+    get_processed(
         filenames=filenames,
         changed=changed,
         verbose=verbose,
@@ -938,7 +938,7 @@ async def build_graph(
                                 click.echo(FeedbackManager.info_skipping_resource(resource=processed_filename))
                                 continue
                             click.echo(FeedbackManager.info_processing_file(filename=processed_filename))
-                        await process(
+                        process(
                             filename=str(f),
                             tb_client=tb_client,
                             deps=deps,
@@ -957,7 +957,7 @@ async def build_graph(
     return GraphDependencies(dep_map, to_run, all_dep_map, all_resources)
 
 
-async def process_file(
+def process_file(
     filename: str,
     tb_client: TinyB,
     skip_connectors: bool = False,
@@ -978,7 +978,7 @@ async def process_file(
                 params[f"engine_{k}"] = v
         return params
 
-    async def get_kafka_params(node: Dict[str, Any]):
+    def get_kafka_params(node: Dict[str, Any]):
         params = {key: value for key, value in node.items() if key.startswith("kafka")}
 
         if not skip_connectors:
@@ -994,7 +994,7 @@ async def process_file(
                     "kafka_sasl_mechanism": params.get("kafka_sasl_mechanism", None),
                 }
 
-                connector = await tb_client.get_connection(**connector_params)
+                connector = tb_client.get_connection(**connector_params)
                 if not connector:
                     click.echo(
                         FeedbackManager.info_creating_kafka_connection(connection_name=params["kafka_connection_name"])
@@ -1008,7 +1008,7 @@ async def process_file(
                     if not all(required_params):
                         raise click.ClickException(FeedbackManager.error_unknown_kafka_connection(datasource=name))
 
-                    connector = await tb_client.connection_create_kafka(**connector_params)
+                    connector = tb_client.connection_create_kafka(**connector_params)
             except Exception as e:
                 raise click.ClickException(
                     FeedbackManager.error_connection_create(
@@ -1027,7 +1027,7 @@ async def process_file(
 
         return params
 
-    async def get_import_params(datasource: Dict[str, Any], node: Dict[str, Any]) -> Dict[str, Any]:
+    def get_import_params(datasource: Dict[str, Any], node: Dict[str, Any]) -> Dict[str, Any]:
         params: Dict[str, Any] = {key: value for key, value in node.items() if key.startswith("import_")}
 
         if len(params) == 0 or skip_connectors:
@@ -1036,7 +1036,7 @@ async def process_file(
         service: Optional[str] = node.get("import_service", None)
 
         if service and service.lower() == "bigquery":
-            if not await tb_client.check_gcp_read_permissions():
+            if not tb_client.check_gcp_read_permissions():
                 raise click.ClickException(FeedbackManager.error_unknown_bq_connection(datasource=datasource["name"]))
 
             # Bigquery doesn't have a datalink, so we can stop here
@@ -1052,7 +1052,7 @@ async def process_file(
         if not connector_id:
             assert isinstance(connector_name, str)
 
-            connector: Optional[Dict[str, Any]] = await tb_client.get_connector(connector_name, service)
+            connector: Optional[Dict[str, Any]] = tb_client.get_connector(connector_name, service)
 
             if not connector:
                 raise Exception(
@@ -1102,13 +1102,13 @@ async def process_file(
             if x["default_value"] and x["default_value"].lower().startswith("materialized"):
                 # turn expression to a select query to sql_get_used_tables can get the used tables
                 q = "select " + x["default_value"][len("materialized") :]
-                tables = await tb_client.sql_get_used_tables(q)
+                tables = tb_client.sql_get_used_tables(q)
                 # materialized columns expressions could have joins so we need to add them as a dep
                 deps += tables
                 # generate replacements and replace the query
                 replacements = {t: t for t in tables}
 
-                replaced_results = await tb_client.replace_tables(q, replacements)
+                replaced_results = tb_client.replace_tables(q, replacements)
                 x["default_value"] = replaced_results.replace("SELECT", "materialized", 1)
             if x.get("jsonpath", None):
                 _format = "ndjson"
@@ -1146,7 +1146,7 @@ async def process_file(
         if "import_service" in node or "import_connection_name" in node:
             VALID_SERVICES: Tuple[str, ...] = ("bigquery", "snowflake", "s3", "s3_iamrole", "gcs", "dynamodb")
 
-            import_params = await get_import_params(params, node)
+            import_params = get_import_params(params, node)
 
             service = import_params.get("import_service", None)
             if service and service not in VALID_SERVICES:
@@ -1172,7 +1172,7 @@ async def process_file(
                     period: int = DEFAULT_CRON_PERIOD
 
                     if current_ws is not None:
-                        workspaces = (await tb_client.user_workspaces(version="v1")).get("workspaces", [])
+                        workspaces = (tb_client.user_workspaces(version="v1")).get("workspaces", [])
                         workspace_rate_limits: Dict[str, Dict[str, int]] = next(
                             (w.get("rate_limits", {}) for w in workspaces if w["id"] == current_ws["id"]), {}
                         )
@@ -1224,7 +1224,7 @@ async def process_file(
                     params[replacement] = value
 
         if "kafka_connection_name" in node:
-            kafka_params = await get_kafka_params(node)
+            kafka_params = get_kafka_params(node)
             params.update(kafka_params)
             del params["format"]
 
@@ -1289,7 +1289,7 @@ async def process_file(
                 sql_rendered = sql
 
             try:
-                dependencies = await tb_client.sql_get_used_tables(sql_rendered, raising=True, is_copy=is_copy)
+                dependencies = tb_client.sql_get_used_tables(sql_rendered, raising=True, is_copy=is_copy)
                 deps += [t for t in dependencies if t not in [n["name"] for n in doc.nodes]]
 
             except Exception as e:
@@ -1316,7 +1316,7 @@ async def process_file(
 
             replacements = {x: x for x in deps if x not in [n["name"] for n in doc.nodes]}
 
-            # FIXME: Ideally we should use await tb_client.replace_tables(sql, replacements)
+            # FIXME: Ideally we should use tb_client.replace_tables(sql, replacements)
             for old, new in replacements.items():
                 sql = re.sub("([\t \\n']+|^)" + old + "([\t \\n'\\)]+|$)", "\\1" + new + "\\2", sql)
 

tinybird/tb/modules/datafile/pull.py

@@ -1,8 +1,6 @@
-from asyncio import Semaphore, gather
 from pathlib import Path
 from typing import Any, Optional
 
-import aiofiles
 import click
 
 from tinybird.tb.client import AuthNoTokenException, TinyB
@@ -11,7 +9,7 @@ from tinybird.tb.modules.datafile.format_pipe import format_pipe
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 
-async def folder_pull(
+def folder_pull(
     client: TinyB,
     folder: str,
     force: bool,
@@ -36,24 +34,24 @@ async def folder_pull(
             return "pipes"
         return None
 
-    async def write_files(
+    def write_files(
         resources: list[dict[str, Any]],
         extension: str,
         get_resource_function: str,
         progress_bar: bool = False,
         fmt: bool = False,
     ):
-        async def write_resource(k: dict[str, Any]):
+        def write_resource(k: dict[str, Any]):
             name = f"{k['name']}.{extension}"
             try:
-                resource = await getattr(client, get_resource_function)(k["name"])
+                resource = getattr(client, get_resource_function)(k["name"])
                 resource_to_write = resource
 
                 if fmt:
                     if extension == "datasource":
-                        resource_to_write = await format_datasource(name, content=resource)
+                        resource_to_write = format_datasource(name, content=resource)
                     elif extension == "pipe":
-                        resource_to_write = await format_pipe(name, content=resource)
+                        resource_to_write = format_pipe(name, content=resource)
 
                 dest_folder = folder
                 if "." in k["name"]:
@@ -71,9 +69,9 @@ async def folder_pull(
                 if verbose:
                     click.echo(FeedbackManager.info_writing_resource(resource=f))
                 if not f.exists() or force:
-                    async with aiofiles.open(f, "w") as fd:
+                    with open(f, "w") as fd:
                         if resource_to_write:
-                            await fd.write(resource_to_write)
+                            fd.write(resource_to_write)
                 else:
                     if verbose:
                         click.echo(FeedbackManager.info_skip_already_exists())
@@ -83,31 +81,31 @@ async def folder_pull(
         if progress_bar:
             with click.progressbar(resources, label=f"Pulling {extension}s") as resources:  # type: ignore
                 for k in resources:
-                    await write_resource(k)
+                    write_resource(k)
         else:
             tasks = [write_resource(k) for k in resources]
-            await _gather_with_concurrency(5, *tasks)
+            _gather_with_concurrency(5, *tasks)
 
     try:
-        datasources = await client.datasources()
-        pipes = await client.pipes()
-        connections = await client.connections(skip_bigquery=True)
+        datasources = client.datasources()
+        pipes = client.pipes()
+        connections = client.connections(skip_bigquery=True)
 
-        await write_files(
+        write_files(
             resources=datasources,
             extension="datasource",
             get_resource_function="datasource_file",
             progress_bar=progress_bar,
             fmt=fmt,
         )
-        await write_files(
+        write_files(
             resources=pipes,
             extension="pipe",
             get_resource_function="pipe_file",
             progress_bar=progress_bar,
             fmt=fmt,
         )
-        await write_files(
+        write_files(
             resources=connections,
             extension="connection",
             get_resource_function="connection_file",
@@ -122,11 +120,11 @@ async def folder_pull(
         raise click.ClickException(FeedbackManager.error_pull(error=str(e)))
 
 
-async def _gather_with_concurrency(n, *tasks):
-    semaphore = Semaphore(n)
-
-    async def sem_task(task):
-        async with semaphore:
-            return await task
-
-    return await gather(*(sem_task(task) for task in tasks))
+def _gather_with_concurrency(n, *tasks):
+    results = []
+    for task in tasks:
+        if callable(task):
+            results.append(task())
+        else:
+            results.append(task)
+    return results