tinybird-cli 2.1.0.dev10__py3-none-any.whl → 2.1.0.dev12__py3-none-any.whl

tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://docs.tinybird.co/cli.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '2.1.0.dev10'
-__revision__ = '1fec830'
+__version__ = '2.1.0.dev12'
+__revision__ = '8032292'

tinybird/ch_utils/engine.py

@@ -1,11 +1,10 @@
+import logging
 import re
 from collections import defaultdict
 from typing import Any, Callable, Dict, Iterable, List, Optional
-import logging
 
 from ..sql import col_name, engine_replicated_to_local, parse_table_structure
 
-
 DEFAULT_EMPTY_PARAMETERS = ["ttl", "partition_key", "sorting_key"]
 DEFAULT_JOIN_EMPTY_PARAMETERS = ["join_strictness", "join_type", "key_columns"]
 

tinybird/check_pypi.py

@@ -1,7 +1,9 @@
 from typing import Optional
+
 import requests
-from tinybird.syncasync import sync_to_async
+
 from tinybird.feedback_manager import FeedbackManager
+from tinybird.syncasync import sync_to_async
 from tinybird.tb_cli_modules.common import CLIException, getenv_bool
 
 PYPY_URL = "https://pypi.org/pypi/tinybird-cli/json"

tinybird/client.py

@@ -1,19 +1,19 @@
 import asyncio
 import json
+import logging
+import ssl
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, Set, Mapping, Union
+from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Union
+from urllib.parse import quote, urlencode
 
 import requests
 import requests.adapters
 from requests import Response
-import logging
-import ssl
-from urllib.parse import quote, urlencode
 from urllib3 import Retry
+
 from tinybird.syncasync import sync_to_async
-from tinybird.tb_cli_modules.telemetry import add_telemetry_event
 from tinybird.tb_cli_modules.regions import fill_with_public_regions
-
+from tinybird.tb_cli_modules.telemetry import add_telemetry_event
 
 HOST = "https://api.tinybird.co"
 LIMIT_RETRIES = 10

tinybird/config.py

@@ -1,7 +1,8 @@
+import json
 from os import environ, getcwd
 from pathlib import Path
-import json
 from typing import Any, Dict, Optional
+
 import click
 
 from tinybird import __cli__

tinybird/connectors.py

@@ -1,11 +1,12 @@
-import os
 import logging
-from typing import Any, Dict, List, Optional, Tuple
+import os
 import time
+from abc import ABC, abstractmethod
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Tuple
 from urllib.parse import unquote
+
 import requests
-from datetime import datetime
-from abc import ABC, abstractmethod
 
 from .client import ConnectorNothingToLoad
 
@@ -26,8 +27,8 @@ except ImportError:
 
 try:
     import googleapiclient.discovery
-    from google.api_core.exceptions import PreconditionFailed
     import snowflake.connector
+    from google.api_core.exceptions import PreconditionFailed
 except ImportError:
     UNINSTALLED_CONNECTORS += ["snowflake"]
 

tinybird/context.py

@@ -1,6 +1,5 @@
 from contextvars import ContextVar
 
-
 workspace_id: ContextVar[str] = ContextVar("workspace_id")
 workspace: ContextVar[str] = ContextVar("workspace")
 table_id: ContextVar[str] = ContextVar("table_id")

tinybird/datafile.py

@@ -1,10 +1,12 @@
 """
 Datafile is like a Dockerfile but to describe ETL processes
 """
+from asyncio import Semaphore, gather
+
 from requests import Response
 
 try:
-    from colorama import Fore, Back, Style, init
+    from colorama import Back, Fore, Style, init
 
     init()
 except ImportError:  # fallback so that the imported classes always exist
@@ -15,61 +17,60 @@ except ImportError:  # fallback so that the imported classes always exist
 
     Fore = Back = Style = ColorFallback()
 
-import os
+import difflib
+import glob
 import itertools
-from os import getcwd
-import logging
-import textwrap
 import json
-from operator import itemgetter
+import logging
+import math
+import os
+import os.path
+import pprint
+import re
 import shlex
+import shutil
 import sys
-import re
-from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Tuple, Union, cast
-from enum import Enum
-from mypy_extensions import VarArg, KwArg
-import difflib
-import click
-import glob
-import pprint
-import requests
+import textwrap
+import traceback
 import unittest
-from string import Template
-from toposort import toposort
-from pathlib import Path
 import urllib.parse
-from urllib.parse import urlencode, urlparse, parse_qs
 from collections import namedtuple
-from io import StringIO
-import os.path
 from copy import deepcopy
-import traceback
 from dataclasses import dataclass
+from enum import Enum
+from io import StringIO
+from operator import itemgetter
+from os import getcwd
+from pathlib import Path
+from statistics import mean, median
+from string import Template
+from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Tuple, Union, cast
+from urllib.parse import parse_qs, urlencode, urlparse
+
+import click
+import requests
 from croniter import croniter
-from contextlib import nullcontext
-import shutil
+from humanfriendly import format_size
+from humanfriendly.tables import format_pretty_table
+from mypy_extensions import KwArg, VarArg
+from toposort import toposort
 
-from .tornado_template import UnClosedIfError
-from .sql import parse_table_structure, schema_to_sql_columns
-from .client import TinyB, DoesNotExistException, AuthException, CanNotBeDeletedException
-from .sql_template import render_sql_template, get_used_tables_in_template
+from tinybird.config import PROJECT_PATHS
 from tinybird.sql_template_fmt import format_sql_template
-from .feedback_manager import FeedbackManager
-from .ch_utils.engine import ENABLED_ENGINES
 from tinybird.syncasync import sync_to_async
-from tinybird.config import PROJECT_PATHS
-from statistics import mean, median
-import math
-from humanfriendly.tables import format_pretty_table
-from humanfriendly import format_size
-
 from tinybird.tb_cli_modules.common import _get_tb_client, get_current_main_workspace, getenv_bool, wait_job
 from tinybird.tb_cli_modules.config import CLIConfig
-from tinybird.tb_cli_modules.exceptions import CLIPipeException, CLIGitReleaseException
+from tinybird.tb_cli_modules.exceptions import CLIGitReleaseException, CLIPipeException
 
-os.environ["GIT_PYTHON_REFRESH"] = "quiet"
-from git import Repo, HEAD, Diff, InvalidGitRepositoryError, GitCommandError  # noqa: E402
+from .ch_utils.engine import ENABLED_ENGINES
+from .client import AuthException, CanNotBeDeletedException, DoesNotExistException, TinyB
+from .feedback_manager import FeedbackManager
+from .sql import parse_table_structure, schema_to_sql_columns
+from .sql_template import get_used_tables_in_template, render_sql_template
+from .tornado_template import UnClosedIfError
 
+os.environ["GIT_PYTHON_REFRESH"] = "quiet"
+from git import HEAD, Diff, GitCommandError, InvalidGitRepositoryError, Repo  # noqa: E402
 
 INTERNAL_TABLES: Tuple[str, ...] = (
     "datasources_ops_log",
@@ -1876,7 +1877,7 @@ class PipeCheckerRunner:
                             method as http_method
                         FROM {pipe_stats_rt}
                         WHERE
-                            url like '%/{self.pipe_name}.%'
+                            pipe_name = '{self.pipe_name}'
                             AND url IS NOT NULL
                             AND extractURLParameter(assumeNotNull(url), 'from') <> 'ui'
                             AND extractURLParameter(assumeNotNull(url), 'pipe_checker') <> 'true'
@@ -1885,6 +1886,7 @@ class PipeCheckerRunner:
                             AND not mapContains(parameters, '__tb__semver')
                             {" AND " + " AND ".join([f"mapContains(post_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ''}
                             { extra_where_clause }
+                        Limit 5000000 -- Enough to bring data while not processing all requests from highly used pipes
                         )
                     group by request_param_names, http_method
                     FORMAT JSON
@@ -1902,8 +1904,8 @@ class PipeCheckerRunner:
                                         method as http_method
                                     FROM {pipe_stats_rt}
                                     WHERE
-                                        url like '%/{self.pipe_name}.%'
-                                        AND url is not null
+                                        pipe_name = '{self.pipe_name}'
+                                        AND url IS NOT NULL
                                         AND extractURLParameter(assumeNotNull(url), 'from') <> 'ui'
                                         AND extractURLParameter(assumeNotNull(url), 'pipe_checker') <> 'true'
                                         AND extractURLParameter(assumeNotNull(url), 'debug') <> 'query'
@@ -4250,6 +4252,16 @@ def format_sql(sql: str, DATAFILE_INDENT: str, line_length: Optional[int] = None
     return "\n".join([f"{DATAFILE_INDENT}{part}" for part in sql.split("\n") if len(part.strip())])
 
 
+async def _gather_with_concurrency(n, *tasks):
+    semaphore = Semaphore(n)
+
+    async def sem_task(task):
+        async with semaphore:
+            return await task
+
+    return await gather(*(sem_task(task) for task in tasks))
+
+
 async def folder_pull(
     client: TinyB,
     folder: str,
@@ -4293,54 +4305,60 @@ async def folder_pull(
         get_resource_function: str,
         progress_bar: bool = False,
     ):
-        values = versions.values()
-        with click.progressbar(values, label=f"Pulling {extension}s") if progress_bar else nullcontext(values) as values:  # type: ignore
-            for k in values:
-                name = f"{k['name']}.{extension}"
-
-                try:
-                    if pattern and not pattern.search(name):
-                        if verbose:
-                            click.echo(FeedbackManager.info_skipping_resource(resource=name))
-                        continue
+        async def write_resource(k: Dict[str, Any]):
+            name = f"{k['name']}.{extension}"
+            try:
+                if pattern and not pattern.search(name):
+                    if verbose:
+                        click.echo(FeedbackManager.info_skipping_resource(resource=name))
+                    return
 
-                    resource = await getattr(client, get_resource_function)(k["original_name"])
+                resource = await getattr(client, get_resource_function)(k["original_name"])
 
-                    dest_folder = folder
-                    if "." in k["name"] and extension != "token":
-                        dest_folder = Path(folder) / "vendor" / k["name"].split(".", 1)[0]
-                        name = f"{k['name'].split('.', 1)[1]}.{extension}"
+                dest_folder = folder
+                if "." in k["name"] and extension != "token":
+                    dest_folder = Path(folder) / "vendor" / k["name"].split(".", 1)[0]
+                    name = f"{k['name'].split('.', 1)[1]}.{extension}"
 
-                    file_folder = get_file_folder(extension)
-                    f = Path(dest_folder) / file_folder if file_folder is not None else Path(dest_folder)
+                file_folder = get_file_folder(extension)
+                f = Path(dest_folder) / file_folder if file_folder is not None else Path(dest_folder)
 
-                    if not f.exists():
-                        f.mkdir(parents=True)
+                if not f.exists():
+                    f.mkdir(parents=True)
 
-                    f = f / name
-                    resource_names = [x.split(".")[-1] for x in resources]
+                f = f / name
+                resource_names = [x.split(".")[-1] for x in resources]
 
+                if verbose:
+                    click.echo(FeedbackManager.info_writing_resource(resource=f))
+                if not f.exists() or force:
+                    with open(f, "w") as fd:
+                        # versions are a client only thing so
+                        # datafiles from the server do not contains information about versions
+                        if k["version"] >= 0:
+                            resource = f"VERSION {k['version']}\n" + resource
+                        if resource:
+                            matches = re.findall(r"([^\s\.]*__v\d+)", resource)
+                            for match in set(matches):
+                                m = match.split("__v")[0]
+                                if m in resources or m in resource_names:
+                                    resource = resource.replace(match, m)
+                            fd.write(resource)
+                else:
                     if verbose:
-                        click.echo(FeedbackManager.info_writing_resource(resource=f))
-                    if not f.exists() or force:
-                        with open(f, "w") as fd:
-                            # versions are a client only thing so
-                            # datafiles from the server do not contains information about versions
-                            if k["version"] >= 0:
-                                resource = f"VERSION {k['version']}\n" + resource
-                            if resource:
-                                matches = re.findall(r"([^\s\.]*__v\d+)", resource)
-                                for match in set(matches):
-                                    m = match.split("__v")[0]
-                                    if m in resources or m in resource_names:
-                                        resource = resource.replace(match, m)
-                                fd.write(resource)
-                    else:
-                        if verbose:
-                            click.echo(FeedbackManager.info_skip_already_exists())
-                except Exception as e:
-                    raise click.ClickException(FeedbackManager.error_exception(error=e))
-        return
+                        click.echo(FeedbackManager.info_skip_already_exists())
+            except Exception as e:
+                raise click.ClickException(FeedbackManager.error_exception(error=e))
+
+        values = versions.values()
+
+        if progress_bar:
+            with click.progressbar(values, label=f"Pulling {extension}s"):
+                for k in values:
+                    await write_resource(k)
+        else:
+            tasks = [write_resource(k) for k in values]
+            await _gather_with_concurrency(5, *tasks)
 
     try:
         datasources = await client.datasources()
@@ -4404,6 +4422,7 @@ async def diff_command(
     if filenames:
         if len(filenames) == 1:
             filenames = [filenames[0]] + get_project_filenames(filenames[0])
+        await folder_pull(client, target_dir, False, None, True, verbose=False)
     else:
         filenames = get_project_filenames(".")
         if verbose:
@@ -4439,9 +4458,6 @@ async def diff_command(
                 continue
 
             suffix = ".datasource" if ".datasource" in file else ".pipe"
-
-            if with_explicit_filenames:
-                await folder_pull(client, target_dir, False, None, True, verbose=False)
             target = target_dir + os.path.sep + rfilename + suffix
 
             diff_lines = await diff_files(target, file, with_format=fmt, with_color=(not no_color), client=client)

tinybird/datatypes.py

@@ -1,5 +1,5 @@
-import re
 import ast
+import re
 
 datetime64_patterns = [
     r"\d\d\d\d.\d\d.\d\d(T|\s)\d\d:\d\d:\d\d.\d\d\d",

tinybird/feedback_manager.py

@@ -1,5 +1,5 @@
 from collections import namedtuple
-from typing import Callable, Any
+from typing import Any, Callable
 
 FeedbackMessage = namedtuple("FeedbackMessage", "message")
 

tinybird/sql.py

@@ -1,6 +1,6 @@
-from collections import namedtuple
 import re
 import string
+from collections import namedtuple
 from typing import Any, Dict, Iterable, List, Optional
 
 

tinybird/sql_template.py

@@ -1,17 +1,18 @@
-import logging
+import ast
 import builtins
+import linecache
+import logging
+import re
+from datetime import datetime
 from functools import lru_cache
-from tornado import escape
-from typing import Optional, Tuple, Union
 from io import StringIO
-import ast
-import re
-import linecache
+from typing import Optional, Tuple, Union
+
+from tornado import escape
 from tornado.util import ObjectDict, exec_in, unicode_type
-from .datatypes import testers
-from datetime import datetime
 
-from .tornado_template import Template, SecurityException, VALID_CUSTOM_FUNCTION_NAMES
+from .datatypes import testers
+from .tornado_template import VALID_CUSTOM_FUNCTION_NAMES, SecurityException, Template
 
 
 class SQLTemplateCustomError(Exception):

tinybird/sql_template_fmt.py

@@ -1,20 +1,18 @@
 import logging
-from typing import Any, Dict, List, Optional
 from dataclasses import dataclass
 from types import MethodType
+from typing import Any, Dict, List, Optional
 
-from sqlfmt import api
-from sqlfmt.mode import Mode
-from sqlfmt.dialect import ClickHouse
-from sqlfmt import actions
+from sqlfmt import actions, api
 from sqlfmt.analyzer import Rule, group
-from sqlfmt.line import Line
 from sqlfmt.comment import Comment
+from sqlfmt.dialect import ClickHouse
+from sqlfmt.jinjafmt import JinjaFormatter, JinjaTag
+from sqlfmt.line import Line
+from sqlfmt.mode import Mode
 from sqlfmt.node import Node
 from sqlfmt.node_manager import NodeManager
 from sqlfmt.token import Token, TokenType
-from sqlfmt.jinjafmt import JinjaTag, JinjaFormatter
-
 
 # This class extends and monkey patches https://github.com/tconbeer/sqlfm
 INDENT = " " * 4

tinybird/sql_toolset.py

@@ -1,13 +1,14 @@
+import copy
 import logging
-from datetime import datetime
-from chtoolset import query as chquery
 from collections import defaultdict
+from datetime import datetime
 from functools import lru_cache
+from typing import Any, List, Optional, Set, Tuple
+
+from chtoolset import query as chquery
 from toposort import toposort
-from typing import List, Tuple, Set, Optional, Any
-from tinybird.ch_utils.constants import ENABLED_TABLE_FUNCTIONS
 
-import copy
+from tinybird.ch_utils.constants import ENABLED_TABLE_FUNCTIONS
 
 VALID_REMOTE = "VALID_REMOTE"
 
@@ -238,9 +239,9 @@ def replace_tables(
                     logging.info(
                         "Resource not found in replace_tables: %s",
                         {
-                            "sql": sql,
-                            "replacements": replacements,
-                            "deps_sorted": deps_sorted,
+                            "r": r,
+                            "default_database": default_database,
+                            "_replaced_with": _replaced_with,
                             "valid_tables": valid_tables,
                         },
                     )

tinybird/syncasync.py

@@ -37,7 +37,7 @@ import sys
 import threading
 import warnings
 import weakref
-from concurrent.futures import Future, ThreadPoolExecutor, Executor
+from concurrent.futures import Executor, Future, ThreadPoolExecutor
 from typing import Any, Callable, Dict, Optional
 
 if sys.version_info >= (3, 7):

tinybird/tb_cli.py

@@ -1,22 +1,21 @@
-import sys
 import asyncio
+import sys
 
 if sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 
-import tinybird.tb_cli_modules.common
-import tinybird.tb_cli_modules.cli
 import tinybird.tb_cli_modules.auth
-import tinybird.tb_cli_modules.workspace
-import tinybird.tb_cli_modules.workspace_members
 import tinybird.tb_cli_modules.branch
-import tinybird.tb_cli_modules.datasource
-import tinybird.tb_cli_modules.pipe
+import tinybird.tb_cli_modules.cli
+import tinybird.tb_cli_modules.common
 import tinybird.tb_cli_modules.connection
+import tinybird.tb_cli_modules.datasource
 import tinybird.tb_cli_modules.job
+import tinybird.tb_cli_modules.pipe
 import tinybird.tb_cli_modules.test
 import tinybird.tb_cli_modules.token
-
+import tinybird.tb_cli_modules.workspace
+import tinybird.tb_cli_modules.workspace_members
 
 cli = tinybird.tb_cli_modules.cli.cli
 

tinybird/tb_cli_modules/auth.py

@@ -3,23 +3,21 @@
 # - If it makes sense and only when strictly necessary, you can create utility functions in this file.
 # - But please, **do not** interleave utility functions and command definitions.
 
-import click
-
 from typing import Any, Dict, List, Optional
 
+import click
 import humanfriendly.tables
 
 from tinybird.feedback_manager import FeedbackManager
-
 from tinybird.tb_cli_modules.cli import cli
 from tinybird.tb_cli_modules.common import (
-    try_update_config_with_remote,
-    try_authenticate,
     configure_connector,
     coro,
+    echo_safe_humanfriendly_tables_format_smart_table,
     get_host_from_region_temp,
     get_regions_temp,
-    echo_safe_humanfriendly_tables_format_smart_table,
+    try_authenticate,
+    try_update_config_with_remote,
 )
 from tinybird.tb_cli_modules.config import CLIConfig, ConfigValueOrigin
 from tinybird.tb_cli_modules.exceptions import CLIAuthException

tinybird/tb_cli_modules/branch.py

@@ -3,30 +3,31 @@
 # - If it makes sense and only when strictly necessary, you can create utility functions in this file.
 # - But please, **do not** interleave utility functions and command definitions.
 
+import os
 from os import getcwd
-from typing import List, Tuple, Optional
+from typing import List, Optional, Tuple
+
 import click
 import yaml
-import os
 
+from tinybird.datafile import create_release, wait_job
+from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb_cli_modules.cli import cli
 from tinybird.tb_cli_modules.common import (
+    MAIN_BRANCH,
     coro,
-    get_current_main_workspace_temp,
     create_workspace_branch,
-    switch_to_workspace_by_user_workspace_data,
-    print_current_workspace,
-    print_data_branch_summary,
     echo_safe_humanfriendly_tables_format_smart_table,
+    get_current_main_workspace_temp,
     get_current_workspace_branches,
-    MAIN_BRANCH,
-    print_current_branch,
     print_branch_regression_tests_summary,
+    print_current_branch,
+    print_current_workspace,
+    print_data_branch_summary,
+    switch_to_workspace_by_user_workspace_data,
     switch_workspace,
     try_update_config_with_remote,
 )
-from tinybird.feedback_manager import FeedbackManager
-from tinybird.datafile import wait_job, create_release
 from tinybird.tb_cli_modules.config import CLIConfig
 from tinybird.tb_cli_modules.exceptions import CLIBranchException, CLIException, CLIReleaseException
 

tinybird/tb_cli_modules/cicd.py

@@ -1,11 +1,11 @@
+from dataclasses import dataclass
 from enum import Enum
-from pathlib import Path
-from typing import Optional, Any, Dict, Union, Type, List
 from os import getcwd
-from tornado.template import Template
-from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Type, Union
 
 import click
+from tornado.template import Template
 
 from tinybird.client import TinyB
 from tinybird.feedback_manager import FeedbackManager