tinybird 0.0.1.dev13__tar.gz → 0.0.1.dev15__tar.gz

{tinybird-0.0.1.dev13 → tinybird-0.0.1.dev15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird
-Version: 0.0.1.dev13
+Version: 0.0.1.dev15
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev13 → tinybird-0.0.1.dev15}/tinybird/datafile.py

@@ -1051,9 +1051,9 @@ def parse(
         def _f(sql: str, **kwargs: Any) -> None:
             if not parser_state.current_node:
                 raise ParseException("SQL must be called after a NODE command")
-            parser_state.current_node[var_name] = (
-                textwrap.dedent(sql).rstrip() if "%" not in sql.strip()[0] else sql.strip()
-            )
+            _sql = textwrap.dedent(sql).rstrip() if "%" not in sql.strip()[0] else sql.strip()
+            _sql = eval_var(_sql)
+            parser_state.current_node[var_name] = _sql
 
         # HACK this cast is needed because Mypy
         return cast(Callable[[str, KwArg(Any)], None], _f)

{tinybird-0.0.1.dev13 → tinybird-0.0.1.dev15}/tinybird/tb/cli.py

@@ -18,7 +18,6 @@ import tinybird.tb.modules.login
 import tinybird.tb.modules.mock
 import tinybird.tb.modules.pipe
 import tinybird.tb.modules.tag
-import tinybird.tb.modules.test
 import tinybird.tb.modules.token
 import tinybird.tb.modules.workspace
 import tinybird.tb.modules.workspace_members

{tinybird-0.0.1.dev13 → tinybird-0.0.1.dev15}/tinybird/tb/modules/build.py

@@ -11,11 +11,14 @@ from typing import Any, Awaitable, Callable, List, Union
 
 import click
 import humanfriendly
+from click import Context
 from watchdog.events import FileSystemEventHandler
 from watchdog.observers import Observer
 
 import tinybird.context as context
-from tinybird.client import TinyB
+from tinybird.syncasync import async_to_sync
+from tinybird.client import TinyB, AuthNoTokenException
+from tinybird.tb.modules.common import CLIException
 from tinybird.config import FeatureFlags
 from tinybird.feedback_manager import FeedbackManager, bcolors
 from tinybird.tb.modules.cli import cli
@@ -33,10 +36,10 @@ from tinybird.tb.modules.table import format_table
 class BuildShell(cmd.Cmd):
     prompt = "\n\001\033[1;32m\002TB > \001\033[0m\002"
 
-    def __init__(self, folder: str):
+    def __init__(self, folder: str, client: TinyB):
         super().__init__()
         self.folder = folder
-
+        self.client = client
     def do_exit(self, arg):
         sys.exit(0)
 
@@ -55,7 +58,11 @@ class BuildShell(cmd.Cmd):
                 extra_args = f" --folder {self.folder}" if arg_stripped.startswith("tb mock") else ""
                 subprocess.run(arg_stripped + extra_args, shell=True, text=True)
             elif arg_stripped.startswith("with") or arg_stripped.startswith("select"):
-                subprocess.run(f'tb sql "{arg_stripped}"', shell=True, text=True)
+                try:
+                    run_sql(self.client, argline)
+                except Exception as e:
+                    click.echo(FeedbackManager.error(message=str(e)))
+                
             elif arg_stripped.startswith("mock "):
                 subprocess.run(f"tb {arg_stripped} --folder {self.folder}", shell=True, text=True)
             else:
@@ -72,7 +79,12 @@ class FileChangeHandler(FileSystemEventHandler):
         self.process = process
 
     def on_modified(self, event: Any) -> None:
-        if not event.is_directory and any(event.src_path.endswith(ext) for ext in [".datasource", ".pipe", ".ndjson"]):
+        is_not_vendor = "vendor/" not in event.src_path
+        if (
+            is_not_vendor
+            and not event.is_directory
+            and any(event.src_path.endswith(ext) for ext in [".datasource", ".pipe", ".ndjson"])
+        ):
             filename = event.src_path.split("/")[-1]
             click.echo(FeedbackManager.highlight(message=f"\n\n⟲ Changes detected in {filename}\n"))
             try:
@@ -119,6 +131,44 @@ def watch_files(
     observer.join()
 
 
+
+def run_sql(client: TinyB, query, rows_limit=20):
+    try:
+        q = query.strip()
+        if q.startswith("insert"):
+            click.echo(FeedbackManager.info_append_data())
+            raise CLIException(FeedbackManager.error_invalid_query())
+        if q.startswith("delete"):
+            raise CLIException(FeedbackManager.error_invalid_query())
+
+        # fuck my life
+        def run_query_in_thread():
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            try:
+                return loop.run_until_complete(client.query(f"SELECT * FROM ({query}) LIMIT {rows_limit} FORMAT JSON"))
+            finally:
+                loop.close()
+
+        # Run the query in a separate thread
+        import concurrent.futures
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            res = executor.submit(run_query_in_thread).result()
+
+    except AuthNoTokenException:
+        raise
+    except Exception as e:
+        raise CLIException(FeedbackManager.error_exception(error=str(e)))
+
+    if isinstance(res, dict) and "error" in res:
+        raise CLIException(FeedbackManager.error_exception(error=res["error"]))
+
+    if isinstance(res, dict) and "data" in res and res["data"]:
+        print_table_formatted(res, 'QUERY')
+    else:
+        click.echo(FeedbackManager.info_no_rows())
+
+
 @cli.command()
 @click.option(
     "--folder",
@@ -132,16 +182,12 @@ def watch_files(
     is_flag=True,
     help="Watch for changes in the files and re-check them.",
 )
-@click.option(
-    "--skip-datasources",
-    is_flag=True,
-    help="Skip rebuilding datasources.",
-)
+@click.pass_context
 @coro
 async def build(
+    ctx: Context,
     folder: str,
     watch: bool,
-    skip_datasources: bool,
 ) -> None:
     """
     Watch for changes in the files and re-check them.
@@ -171,7 +217,7 @@ async def build(
 
             parser(filename)
 
-    async def process(filenames: List[str], watch: bool = False, only_pipes: bool = False):
+    async def process(filenames: List[str], watch: bool = False):
         datafiles = [f for f in filenames if f.endswith(".datasource") or f.endswith(".pipe")]
         if len(datafiles) > 0:
             check_filenames(filenames=datafiles)
@@ -180,7 +226,7 @@ async def build(
                 filenames=datafiles,
                 ignore_sql_errors=ignore_sql_errors,
                 is_internal=is_internal,
-                only_pipes=only_pipes,
+                watch=watch,
             )
 
         filename = filenames[0]
@@ -209,7 +255,7 @@ async def build(
         try:
             click.echo("⚡ Building project...\n")
             time_start = time.time()
-            await process(filenames=filenames, watch=False, only_pipes=skip_datasources)
+            await process(filenames=filenames, watch=False)
             time_end = time.time()
             elapsed_time = time_end - time_start
             for filename in filenames:
@@ -226,7 +272,7 @@ async def build(
     await build_once(filenames)
 
     if watch:
-        shell = BuildShell(folder=folder)
+        shell = BuildShell(folder=folder, client=tb_client)
         click.echo(FeedbackManager.highlight(message="◎ Watching for changes..."))
         watcher_thread = threading.Thread(target=watch_files, args=(filenames, process, shell, folder), daemon=True)
         watcher_thread.start()
@@ -234,13 +280,16 @@ async def build(
 
 
 async def build_and_print_resource(tb_client: TinyB, filename: str):
-    rebuild_colors = [bcolors.FAIL, bcolors.OKBLUE, bcolors.WARNING, bcolors.OKGREEN, bcolors.HEADER]
-    rebuild_index = random.randint(0, len(rebuild_colors) - 1)
-    rebuild_color = rebuild_colors[rebuild_index % len(rebuild_colors)]
     resource_path = Path(filename)
     name = resource_path.stem
     pipeline = name if filename.endswith(".pipe") else None
     res = await tb_client.query(f"SELECT * FROM {name} FORMAT JSON", pipeline=pipeline)
+    print_table_formatted(res, name)
+
+def print_table_formatted(res: dict, name: str):
+    rebuild_colors = [bcolors.FAIL, bcolors.OKBLUE, bcolors.WARNING, bcolors.OKGREEN, bcolors.HEADER]
+    rebuild_index = random.randint(0, len(rebuild_colors) - 1)
+    rebuild_color = rebuild_colors[rebuild_index % len(rebuild_colors)]
     data = []
     limit = 5
     for d in res["data"][:5]:

{tinybird-0.0.1.dev13 → tinybird-0.0.1.dev15}/tinybird/tb/modules/cli.py

@@ -10,7 +10,7 @@ import pprint
 import re
 import shutil
 import sys
-from os import environ, getcwd
+from os import getcwd
 from pathlib import Path
 from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
 
@@ -26,7 +26,7 @@ from tinybird.client import (
     DoesNotExistException,
     TinyB,
 )
-from tinybird.config import CURRENT_VERSION, SUPPORTED_CONNECTORS, VERSION, FeatureFlags, get_config
+from tinybird.config import SUPPORTED_CONNECTORS, VERSION, FeatureFlags, get_config
 from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb.modules.common import (
     OLDEST_ROLLBACK,
@@ -50,6 +50,7 @@ from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.datafile.build import build_graph, folder_push
 from tinybird.tb.modules.datafile.common import (
     Datafile,
+    DatafileSyntaxError,
     get_project_filenames,
     get_resource_versions,
     has_internal_datafiles,
@@ -106,12 +107,6 @@ DEFAULT_PATTERNS: List[Tuple[str, Union[str, Callable[[str], str]]]] = [
 @click.option(
     "--with-headers", help="Flag to enable connector to export with headers", is_flag=True, default=False, hidden=True
 )
-@click.option(
-    "--version-warning/--no-version-warning",
-    envvar="TB_VERSION_WARNING",
-    default=True,
-    help="Don't print version warning message if there's a new available version. You can use TB_VERSION_WARNING envar",
-)
 @click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
 @click.version_option(version=VERSION)
 @click.pass_context
@@ -135,7 +130,6 @@ async def cli(
     sf_storage_integration: str,
     sf_stage,
     with_headers: bool,
-    version_warning: bool,
     show_tokens: bool,
 ) -> None:
     """
@@ -151,17 +145,6 @@ async def cli(
     if getenv_bool("TB_DISABLE_SSL_CHECKS", False):
         click.echo(FeedbackManager.warning_disabled_ssl_checks())
 
-    # ensure that ctx.obj exists and is a dict (in case `cli()` is called)
-    # by means other than the `if` block below
-    if not environ.get("PYTEST", None) and version_warning and not token:
-        from tinybird.check_pypi import CheckPypi
-
-        latest_version = await CheckPypi().get_latest_version()
-
-        if "x.y.z" not in CURRENT_VERSION and latest_version != CURRENT_VERSION:
-            click.echo(FeedbackManager.warning_update_version(latest_version=latest_version))
-            click.echo(FeedbackManager.warning_current_version(current_version=CURRENT_VERSION))
-
     if debug:
         logging.basicConfig(level=logging.DEBUG)
 
@@ -361,6 +344,9 @@ def check(filenames: List[str], debug: bool) -> None:
                     for x in doc.nodes:
                         pp.pprint(x)
 
+        except DatafileSyntaxError as e:
+            # TODO(eclbg): add the filename to the error message
+            raise CLIException(e)
         except ParseException as e:
             raise CLIException(FeedbackManager.error_exception(error=e))
 
@@ -565,24 +551,17 @@ async def push(
 @click.option(
     "--folder", default=None, type=click.Path(exists=True, file_okay=False), help="Folder where files will be placed"
 )
-@click.option(
-    "--auto/--no-auto",
-    is_flag=True,
-    default=True,
-    help="Saves datafiles automatically into their default directories (/datasources or /pipes). Default is True",
-)
-@click.option("--match", default=None, help="Retrieve any resourcing matching the pattern. eg --match _test")
 @click.option("-f", "--force", is_flag=True, default=False, help="Override existing files")
 @click.option("--fmt", is_flag=True, default=False, help="Format files before saving")
 @click.pass_context
 @coro
-async def pull(ctx: Context, folder: str, auto: bool, match: Optional[str], force: bool, fmt: bool) -> None:
+async def pull(ctx: Context, folder: str, force: bool, fmt: bool) -> None:
     """Retrieve latest versions for project files from Tinybird."""
 
     client = ctx.ensure_object(dict)["client"]
     folder = folder if folder else getcwd()
 
-    return await folder_pull(client, folder, auto, match, force, fmt=fmt)
+    return await folder_pull(client, folder, force, fmt=fmt)
 
 
 @cli.command()

{tinybird-0.0.1.dev13 → tinybird-0.0.1.dev15}/tinybird/tb/modules/create.py

@@ -16,7 +16,6 @@ from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.datafile.fixture import build_fixture_name, persist_fixture
 from tinybird.tb.modules.exceptions import CLIException
 from tinybird.tb.modules.llm import LLM
-from tinybird.tb.modules.local import get_tinybird_local_client
 
 
 @cli.command()
@@ -51,7 +50,8 @@ async def create(
     """Initialize a new project."""
     folder = folder or getcwd()
     try:
-        tb_client = await get_tinybird_local_client(os.path.abspath(folder))
+        config = CLIConfig.get_project_config()
+        tb_client = config.get_client()
         click.echo(FeedbackManager.gray(message="Creating new project structure..."))
         await project_create(tb_client, data, prompt, folder)
         click.echo(FeedbackManager.success(message="✓ Scaffolding completed!\n"))
@@ -74,13 +74,12 @@ async def create(
             datasource_files = [f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")]
             for datasource_file in datasource_files:
                 datasource_path = Path(folder) / "datasources" / datasource_file
-                llm_config = CLIConfig.get_llm_config()
-                llm = LLM(key=llm_config["api_key"])
+                llm = LLM(client=tb_client)
                 datasource_name = datasource_path.stem
                 datasource_content = datasource_path.read_text()
                 has_json_path = "`json:" in datasource_content
                 if has_json_path:
-                    sql = await llm.generate_sql_sample_data(tb_client, datasource_name, datasource_content, rows)
+                    sql = await llm.generate_sql_sample_data(schema=datasource_content, rows=rows)
                     result = await tb_client.query(f"{sql} FORMAT JSON")
                     data = result.get("data", [])
                     fixture_name = build_fixture_name(datasource_path.absolute(), datasource_name, datasource_content)
@@ -88,8 +87,6 @@ async def create(
                     click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}"))
 
         click.echo(FeedbackManager.success(message="✓ Done!\n"))
-
-        click.echo(FeedbackManager.success(message="✓ Tinybird local is ready!"))
     except Exception as e:
         click.echo(FeedbackManager.error(message=f"Error: {str(e)}"))
 
@@ -130,8 +127,7 @@ TYPE ENDPOINT
         )
     elif prompt:
         try:
-            llm_config = CLIConfig.get_llm_config()
-            llm = LLM(key=llm_config["api_key"])
+            llm = LLM(client=client)
             result = await llm.create_project(prompt)
             for ds in result.datasources:
                 content = ds.content.replace("```", "")

{tinybird-0.0.1.dev13 → tinybird-0.0.1.dev15}/tinybird/tb/modules/datafile/build.py

@@ -54,15 +54,12 @@ async def folder_build(
     folder: str = ".",
     ignore_sql_errors: bool = False,
     is_internal: bool = False,
-    only_pipes: bool = False,
     is_vendor: bool = False,
     current_ws: Optional[Dict[str, Any]] = None,
     local_ws: Optional[Dict[str, Any]] = None,
     workspaces: Optional[List[Dict[str, Any]]] = None,
+    watch: bool = False,
 ):
-    if only_pipes:
-        filenames = [f for f in filenames if f.endswith(".pipe")]
-
     config = CLIConfig.get_project_config()
     build = True
     dry_run = False
@@ -118,7 +115,7 @@ async def folder_build(
 
     vendor_workspaces = []
 
-    if vendor_path.exists() and not is_vendor:
+    if vendor_path.exists() and not is_vendor and not watch:
         user_workspaces = await user_client.user_workspaces()
         for x in vendor_path.iterdir():
             if x.is_dir() and x.name not in existing_workspaces:
@@ -147,7 +144,7 @@ async def folder_build(
         ws_client.token = vendor_ws["token"]
         shared_ws_path = Path(folder) / "vendor" / vendor_ws["name"]
 
-        if shared_ws_path.exists() and not is_vendor:
+        if shared_ws_path.exists() and not is_vendor and not watch:
             await folder_build(
                 ws_client, folder=shared_ws_path.as_posix(), is_vendor=True, current_ws=vendor_ws, local_ws=local_ws
             )
@@ -440,14 +437,16 @@ async def folder_build(
 
             for group in groups:
                 for name in group:
-                    try:
-                        await tb_client.datasource_delete(name, force=True)
-                    except Exception:
-                        pass
-                    try:
-                        await tb_client.pipe_delete(name)
-                    except Exception:
-                        pass
+                    is_vendor = resources_to_run_fork_downstream.get(name, {}).get("filename", "").startswith("vendor/")
+                    if not is_vendor:
+                        try:
+                            await tb_client.datasource_delete(name, force=True)
+                        except Exception:
+                            pass
+                        try:
+                            await tb_client.pipe_delete(name)
+                        except Exception:
+                            pass
 
             groups.reverse()
             for group in groups:
@@ -1036,9 +1035,13 @@ async def build_graph(
             raise click.ClickException(str(e))
 
         for r in res:
-            fn = r["resource_name"]
+            resource_name = r["resource_name"]
             warnings = r.get("warnings", [])
-            if changed and fn in changed and (not changed[fn] or changed[fn] in ["shared", "remote"]):
+            if (
+                changed
+                and resource_name in changed
+                and (not changed[resource_name] or changed[resource_name] in ["shared", "remote"])
+            ):
                 continue
 
             if (
@@ -1046,9 +1049,9 @@ async def build_graph(
                 and r.get("resource", "") == "pipes"
                 and any(["engine" in x.get("params", {}) for x in r.get("nodes", [])])
             ):
-                raise click.ClickException(FeedbackManager.error_forkdownstream_pipes_with_engine(pipe=fn))
+                raise click.ClickException(FeedbackManager.error_forkdownstream_pipes_with_engine(pipe=resource_name))
 
-            to_run[fn] = r
+            to_run[resource_name] = r
             file_deps = r.get("deps", [])
             deps += file_deps
             # calculate and look for deps
@@ -1064,7 +1067,7 @@ async def build_graph(
                         to_run[ds_fn] = deepcopy(r)
                         try:
                             to_run[ds_fn]["deps"] = list(
-                                set(to_run[ds_fn].get("deps", []) + prev.get("deps", []) + [fn])
+                                set(to_run[ds_fn].get("deps", []) + prev.get("deps", []) + [resource_name])
                             )
                         except ValueError:
                             pass
@@ -1085,7 +1088,7 @@ async def build_graph(
                     )
                 r["shared_with"] = mapped_workspaces
 
-            dep_map[fn] = set(dep_list)
+            dep_map[resource_name] = set(dep_list)
         return os.path.basename(name), warnings
 
     processed = set()
@@ -1169,7 +1172,10 @@ async def process_file(
     workspace_map: Optional[Dict] = None,
     workspace_lib_paths: Optional[List[Tuple[str, str]]] = None,
     current_ws: Optional[Dict[str, Any]] = None,
-):
+) -> List[Dict[str, Any]]:
+    """Returns a list of resources
+
+    For both datasources and pipes, a list of just one item is returned"""
     if workspace_map is None:
         workspace_map = {}
 
@@ -1349,6 +1355,8 @@ async def process_file(
         indexes = None
         if indexes_list:
             indexes = "\n".join([index.to_sql() for index in indexes_list])
+        # Here is where we lose the columns
+        # I don't know why we don't return something more similar to the parsed doc
         params = {
             "name": append_version_to_name(name, version),
             "description": description,

{tinybird-0.0.1.dev13 → tinybird-0.0.1.dev15}/tinybird/tb/modules/datafile/build_pipe.py

@@ -264,6 +264,10 @@ async def new_pipe(
                 except Exception as e:
                     raise click.ClickException(FeedbackManager.error_creating_pipe(error=e))
 
+    if data.get("type") == "endpoint":
+        token = tb_client.token
+        print(f"""** => Test endpoint with:\n** $ curl {host}/v0/pipes/{p["name"]}.json?token={token}""")  # noqa: T201
+
 
 async def get_token_from_main_branch(branch_tb_client: TinyB) -> Optional[str]:
     token_from_main_branch = None