tinybird 0.0.1.dev29__py3-none-any.whl → 0.0.1.dev30__py3-none-any.whl

tinybird/tb/cli.py

@@ -6,7 +6,7 @@ if sys.platform == "win32":
 
 import tinybird.tb.modules.auth
 import tinybird.tb.modules.build
-import tinybird.tb.modules.build_server
+import tinybird.tb.modules.build_client
 import tinybird.tb.modules.cli
 import tinybird.tb.modules.common
 import tinybird.tb.modules.connection
@@ -22,6 +22,7 @@ import tinybird.tb.modules.pipe
 import tinybird.tb.modules.tag
 import tinybird.tb.modules.test
 import tinybird.tb.modules.token
+import tinybird.tb.modules.update
 import tinybird.tb.modules.workspace
 import tinybird.tb.modules.workspace_members
 

tinybird/tb/modules/build.py

@@ -1,214 +1,74 @@
 import asyncio
-import os
-import threading
-import time
+import glob
+import json
+import logging
 from pathlib import Path
 from typing import List
 
 import click
+import requests
 
-import tinybird.context as context
-from tinybird.client import TinyB
-from tinybird.config import FeatureFlags
 from tinybird.tb.modules.cli import cli
-from tinybird.tb.modules.common import push_data
-from tinybird.tb.modules.datafile.build import folder_build
-from tinybird.tb.modules.datafile.common import get_project_filenames, get_project_fixtures, has_internal_datafiles
-from tinybird.tb.modules.datafile.exceptions import ParseException
-from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
-from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
-from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.local_common import get_tinybird_local_client
-from tinybird.tb.modules.shell import Shell, print_table_formatted
-from tinybird.tb.modules.watch import watch_files
 
 
-@cli.command()
-@click.option(
-    "--folder",
-    default=".",
-    help="Folder from where to execute the command. By default the current folder.",
-    hidden=True,
-    type=click.types.STRING,
-)
-@click.option(
-    "--watch",
-    is_flag=True,
-    help="Watch for changes in the files and rebuild them.",
-)
-def build(
-    folder: str,
-    watch: bool,
-) -> None:
-    """Build the project in Tinybird Local."""
-    ignore_sql_errors = FeatureFlags.ignore_sql_errors()
-    context.disable_template_security_validation.set(True)
-    is_internal = has_internal_datafiles(folder)
-    folder_path = os.path.abspath(folder)
-    tb_client = asyncio.run(get_tinybird_local_client(folder_path))
-
-    def check_filenames(filenames: List[str]):
-        parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
-        incl_suffix = ".incl"
-
-        for filename in filenames:
-            if os.path.isdir(filename):
-                check_filenames(filenames=get_project_filenames(filename))
-
-            file_suffix = Path(filename).suffix
-            if file_suffix == incl_suffix:
-                continue
-
-            parser = parser_matrix.get(file_suffix)
-            if not parser:
-                raise ParseException(FeedbackManager.error_unsupported_datafile(extension=file_suffix))
-
-            parser(filename)
-
-    async def process(filenames: List[str], watch: bool = False):
-        datafiles = [f for f in filenames if f.endswith(".datasource") or f.endswith(".pipe")]
-        if len(datafiles) > 0:
-            check_filenames(filenames=datafiles)
-            await folder_build(
-                tb_client,
-                filenames=datafiles,
-                ignore_sql_errors=ignore_sql_errors,
-                is_internal=is_internal,
-                watch=watch,
-                folder=folder,
-            )
-        if len(filenames) > 0:
-            filename = filenames[0]
-            if filename.endswith(".ndjson"):
-                fixture_path = Path(filename)
-                datasources_path = Path(folder) / "datasources"
-                ds_name = fixture_path.stem
-                ds_path = datasources_path / f"{ds_name}.datasource"
-
-                if not ds_path.exists():
-                    try:
-                        ds_name = "_".join(fixture_path.stem.split("_")[:-1])
-                        ds_path = datasources_path / f"{ds_name}.datasource"
-                    except Exception:
-                        pass
-
-                if ds_path.exists():
-                    await append_datasource(tb_client, ds_name, str(fixture_path))
-
-            if watch:
-                if filename.endswith(".datasource"):
-                    ds_path = Path(filename)
-                    ds_name = ds_path.stem
-                    name = build_fixture_name(filename, ds_name, ds_path.read_text())
-                    fixture_folder = get_fixture_dir(folder)
-                    fixture_path = fixture_folder / f"{name}.ndjson"
-
-                    if not fixture_path.exists():
-                        fixture_path = fixture_folder / f"{ds_name}.ndjson"
-
-                    if fixture_path.exists():
-                        await append_datasource(tb_client, ds_name, str(fixture_path))
-
-                if not filename.endswith(".ndjson"):
-                    await build_and_print_resource(tb_client, filename)
-
-    datafiles = get_project_filenames(folder)
-    fixtures = get_project_fixtures(folder)
-    filenames = datafiles + fixtures
+def project_files(project_path: Path) -> List[str]:
+    project_file_extensions = ("datasource", "pipe")
+    project_files = []
+    for extension in project_file_extensions:
+        for project_file in glob.glob(f"{project_path}/**/*.{extension}", recursive=True):
+            logging.debug(f"Found project file: {project_file}")
+            project_files.append(project_file)
+    return project_files
 
-    async def build_once(filenames: List[str]):
-        ok = False
-        try:
-            click.echo(FeedbackManager.highlight(message="» Building project...\n"))
-            time_start = time.time()
-            await process(filenames=filenames, watch=False)
-            time_end = time.time()
-            elapsed_time = time_end - time_start
-            for filename in filenames:
-                if filename.endswith(".datasource"):
-                    ds_path = Path(filename)
-                    ds_name = ds_path.stem
-                    name = build_fixture_name(filename, ds_name, ds_path.read_text())
-                    fixture_folder = get_fixture_dir(folder)
-                    fixture_path = fixture_folder / f"{name}.ndjson"
 
-                    if not fixture_path.exists():
-                        fixture_path = fixture_folder / f"{ds_name}.ndjson"
-
-                    if fixture_path.exists():
-                        await append_datasource(tb_client, ds_name, str(fixture_path))
-            click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s"))
-            ok = True
-        except Exception as e:
-            error_path = Path(".tb_error.txt")
-            if error_path.exists():
-                content = error_path.read_text()
-                content += f"\n\n{str(e)}"
-                error_path.write_text(content)
-            else:
-                error_path.write_text(str(e))
-            click.echo(FeedbackManager.error_exception(error=e))
-            ok = False
-        return ok
-
-    build_ok = asyncio.run(build_once(filenames))
-
-    if watch:
-        paths = [Path(f) for f in get_project_filenames(folder, with_vendor=True)]
-
-        def is_vendor(f: Path) -> bool:
-            return f.parts[0] == "vendor"
-
-        def get_vendor_workspace(f: Path) -> str:
-            return f.parts[1]
-
-        def is_endpoint(f: Path) -> bool:
-            return f.suffix == ".pipe" and not is_vendor(f) and f.parts[0] == "endpoints"
-
-        def is_pipe(f: Path) -> bool:
-            return f.suffix == ".pipe" and not is_vendor(f)
-
-        datasource_paths = [f for f in paths if f.suffix == ".datasource"]
-        datasources = [f.stem for f in datasource_paths if not is_vendor(f)]
-        shared_datasources = [f"{get_vendor_workspace(f)}.{f.stem}" for f in datasource_paths if is_vendor(f)]
-        pipes = [f.stem for f in paths if is_pipe(f) and not is_endpoint(f)]
-        endpoints = [f.stem for f in paths if is_endpoint(f)]
-        shell = Shell(
-            folder=folder,
-            client=tb_client,
-            datasources=datasources,
-            shared_datasources=shared_datasources,
-            pipes=pipes,
-            endpoints=endpoints,
-        )
-        click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
-        watcher_thread = threading.Thread(
-            target=watch_files, args=(filenames, process, shell, folder, build_ok), daemon=True
-        )
-        watcher_thread.start()
-        shell.run()
-
-
-async def build_and_print_resource(tb_client: TinyB, filename: str):
-    resource_path = Path(filename)
-    name = resource_path.stem
-    pipeline = name if filename.endswith(".pipe") else None
-    res = await tb_client.query(f"SELECT * FROM {name} FORMAT JSON", pipeline=pipeline)
-    print_table_formatted(res, name)
-
-
-async def append_datasource(
-    tb_client: TinyB,
-    datasource_name: str,
-    url: str,
-):
-    await tb_client.datasource_truncate(datasource_name)
-    await push_data(
-        tb_client,
-        datasource_name,
-        url,
-        mode="append",
-        concurrency=1,
-        silent=True,
-    )
+@cli.command()
+@click.argument("project_path", type=click.Path(exists=True), default=Path.cwd())
+def build(project_path: Path) -> None:
+    """
+    Validate and build the project server side.
+    """
+    MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
+    DATAFILE_TYPE_TO_CONTENT_TYPE = {
+        ".datasource": "text/plain",
+        ".pipe": "text/plain",
+    }
+
+    tb_client = asyncio.run(get_tinybird_local_client(str(project_path)))
+    TINYBIRD_API_URL = tb_client.host + "/v1/build"
+    TINYBIRD_API_KEY = tb_client.token
+
+    files = [
+        ("context://", ("cli-version", "1.0.0", "text/plain")),
+    ]
+    fds = []
+    for file_path in project_files(project_path):
+        relative_path = str(Path(file_path).relative_to(project_path))
+        fd = open(file_path, "rb")
+        fds.append(fd)
+        content_type = DATAFILE_TYPE_TO_CONTENT_TYPE.get(Path(file_path).suffix, "application/unknown")
+        files.append((MULTIPART_BOUNDARY_DATA_PROJECT, (relative_path, fd.read().decode("utf-8"), content_type)))
+
+    try:
+        HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
+
+        r = requests.post(TINYBIRD_API_URL, files=files, headers=HEADERS)
+        result = r.json()
+        logging.debug(json.dumps(result, indent=2))
+
+        build_result = result.get("result")
+        if build_result == "success":
+            click.echo(FeedbackManager.success(message="Build completed successfully"))
+        elif build_result == "failed":
+            click.echo(FeedbackManager.error(message="Build failed"))
+            build_errors = result.get("errors")
+            for build_error in build_errors:
+                filename_bit = f"{build_error.get('filename', '')}"
+                error_msg = (filename_bit + "\n\n") if filename_bit else "" + build_error.get("error")
+                click.echo(FeedbackManager.error(message=error_msg))
+        else:
+            click.echo(FeedbackManager.error(message=f"Unknown build result. Error: {result.get('error')}"))
+    finally:
+        for fd in fds:
+            fd.close()

tinybird/tb/modules/build_client.py

@@ -0,0 +1,219 @@
+import asyncio
+import os
+import threading
+import time
+from pathlib import Path
+from typing import List
+
+import click
+
+import tinybird.context as context
+from tinybird.client import TinyB
+from tinybird.config import FeatureFlags
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.common import push_data
+from tinybird.tb.modules.datafile.build import folder_build
+from tinybird.tb.modules.datafile.common import get_project_filenames, get_project_fixtures, has_internal_datafiles
+from tinybird.tb.modules.datafile.exceptions import ParseException
+from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
+from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
+from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.local_common import get_tinybird_local_client
+from tinybird.tb.modules.shell import Shell, print_table_formatted
+from tinybird.tb.modules.watch import watch_files
+
+
+def is_vendor(f: Path) -> bool:
+    return f.parts[0] == "vendor"
+
+
+def get_vendor_workspace(f: Path) -> str:
+    return f.parts[1]
+
+
+def is_endpoint(f: Path) -> bool:
+    return f.suffix == ".pipe" and not is_vendor(f) and f.parts[0] == "endpoints"
+
+
+def is_pipe(f: Path) -> bool:
+    return f.suffix == ".pipe" and not is_vendor(f)
+
+
+def check_filenames(filenames: List[str]):
+    parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
+    incl_suffix = ".incl"
+
+    for filename in filenames:
+        if os.path.isdir(filename):
+            check_filenames(filenames=get_project_filenames(filename))
+
+        file_suffix = Path(filename).suffix
+        if file_suffix == incl_suffix:
+            continue
+
+        parser = parser_matrix.get(file_suffix)
+        if not parser:
+            raise ParseException(FeedbackManager.error_unsupported_datafile(extension=file_suffix))
+
+        parser(filename)
+
+
+@cli.command()
+@click.option(
+    "--folder",
+    default=".",
+    help="Folder from where to execute the command. By default the current folder.",
+    hidden=True,
+    type=click.types.STRING,
+)
+@click.option(
+    "--watch",
+    is_flag=True,
+    help="Watch for changes in the files and rebuild them.",
+)
+def build_client(
+    folder: str,
+    watch: bool,
+) -> None:
+    """Build the project in Tinybird Local."""
+    ignore_sql_errors = FeatureFlags.ignore_sql_errors()
+    context.disable_template_security_validation.set(True)
+    is_internal = has_internal_datafiles(folder)
+    folder_path = os.path.abspath(folder)
+    tb_client = asyncio.run(get_tinybird_local_client(folder_path))
+
+    async def process(filenames: List[str], watch: bool = False):
+        datafiles = [f for f in filenames if f.endswith(".datasource") or f.endswith(".pipe")]
+        if len(datafiles) > 0:
+            check_filenames(filenames=datafiles)
+            await folder_build(
+                tb_client,
+                filenames=datafiles,
+                ignore_sql_errors=ignore_sql_errors,
+                is_internal=is_internal,
+                watch=watch,
+                folder=folder,
+            )
+        if len(filenames) > 0:
+            filename = filenames[0]
+            if filename.endswith(".ndjson"):
+                fixture_path = Path(filename)
+                datasources_path = Path(folder) / "datasources"
+                ds_name = fixture_path.stem
+                ds_path = datasources_path / f"{ds_name}.datasource"
+
+                if not ds_path.exists():
+                    try:
+                        ds_name = "_".join(fixture_path.stem.split("_")[:-1])
+                        ds_path = datasources_path / f"{ds_name}.datasource"
+                    except Exception:
+                        pass
+
+                if ds_path.exists():
+                    await append_datasource(tb_client, ds_name, str(fixture_path))
+
+            if watch:
+                if filename.endswith(".datasource"):
+                    ds_path = Path(filename)
+                    ds_name = ds_path.stem
+                    name = build_fixture_name(filename, ds_name, ds_path.read_text())
+                    fixture_folder = get_fixture_dir(folder)
+                    fixture_path = fixture_folder / f"{name}.ndjson"
+
+                    if not fixture_path.exists():
+                        fixture_path = fixture_folder / f"{ds_name}.ndjson"
+
+                    if fixture_path.exists():
+                        await append_datasource(tb_client, ds_name, str(fixture_path))
+
+                if not filename.endswith(".ndjson"):
+                    await build_and_print_resource(tb_client, filename)
+
+    datafiles = get_project_filenames(folder)
+    fixtures = get_project_fixtures(folder)
+    filenames = datafiles + fixtures
+
+    async def build_once(filenames: List[str]):
+        ok = False
+        try:
+            click.echo(FeedbackManager.highlight(message="» Building project...\n"))
+            time_start = time.time()
+            await process(filenames=filenames, watch=False)
+            time_end = time.time()
+            elapsed_time = time_end - time_start
+            for filename in filenames:
+                if filename.endswith(".datasource"):
+                    ds_path = Path(filename)
+                    ds_name = ds_path.stem
+                    name = build_fixture_name(filename, ds_name, ds_path.read_text())
+                    fixture_folder = get_fixture_dir(folder)
+                    fixture_path = fixture_folder / f"{name}.ndjson"
+
+                    if not fixture_path.exists():
+                        fixture_path = fixture_folder / f"{ds_name}.ndjson"
+
+                    if fixture_path.exists():
+                        await append_datasource(tb_client, ds_name, str(fixture_path))
+            click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s"))
+            ok = True
+        except Exception as e:
+            error_path = Path(".tb_error.txt")
+            if error_path.exists():
+                content = error_path.read_text()
+                content += f"\n\n{str(e)}"
+                error_path.write_text(content)
+            else:
+                error_path.write_text(str(e))
+            click.echo(FeedbackManager.error_exception(error=e))
+            ok = False
+        return ok
+
+    build_ok = asyncio.run(build_once(filenames))
+
+    if watch:
+        paths = [Path(f) for f in get_project_filenames(folder, with_vendor=True)]
+
+        datasource_paths = [f for f in paths if f.suffix == ".datasource"]
+        datasources = [f.stem for f in datasource_paths if not is_vendor(f)]
+        shared_datasources = [f"{get_vendor_workspace(f)}.{f.stem}" for f in datasource_paths if is_vendor(f)]
+        pipes = [f.stem for f in paths if is_pipe(f) and not is_endpoint(f)]
+        endpoints = [f.stem for f in paths if is_endpoint(f)]
+        shell = Shell(
+            folder=folder,
+            client=tb_client,
+            datasources=datasources,
+            shared_datasources=shared_datasources,
+            pipes=pipes,
+            endpoints=endpoints,
+        )
+        click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
+        watcher_thread = threading.Thread(
+            target=watch_files, args=(filenames, process, shell, folder, build_ok), daemon=True
+        )
+        watcher_thread.start()
+        shell.run()
+
+
+async def build_and_print_resource(tb_client: TinyB, filename: str):
+    resource_path = Path(filename)
+    name = resource_path.stem
+    pipeline = name if filename.endswith(".pipe") else None
+    res = await tb_client.query(f"SELECT * FROM {name} FORMAT JSON", pipeline=pipeline)
+    print_table_formatted(res, name)
+
+
+async def append_datasource(
+    tb_client: TinyB,
+    datasource_name: str,
+    url: str,
+):
+    await tb_client.datasource_truncate(datasource_name)
+    await push_data(
+        tb_client,
+        datasource_name,
+        url,
+        mode="append",
+        concurrency=1,
+        silent=True,
+    )

tinybird/tb/modules/cli.py

@@ -9,7 +9,7 @@ import os
 import pprint
 from os import getcwd
 from pathlib import Path
-from typing import Any, Callable, Iterable, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
 
 import click
 import humanfriendly
@@ -20,7 +20,8 @@ from tinybird.client import (
     AuthNoTokenException,
     TinyB,
 )
-from tinybird.config import SUPPORTED_CONNECTORS, get_config
+from tinybird.config import get_config
+from tinybird.prompts import ask_prompt
 from tinybird.tb import __cli__
 from tinybird.tb.modules.common import (
     CatchAuthExceptions,
@@ -30,7 +31,6 @@ from tinybird.tb.modules.common import (
     echo_safe_format_table,
     get_current_main_workspace,
     getenv_bool,
-    load_connector_config,
     try_update_config_with_remote,
 )
 from tinybird.tb.modules.config import CLIConfig
@@ -94,7 +94,7 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
     "--with-headers", help="Flag to enable connector to export with headers", is_flag=True, default=False, hidden=True
 )
 @click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
-@click.option("--local", is_flag=True, default=False, help="Run in local mode")
+@click.option("--prod", is_flag=True, default=False, help="Run against production")
 @click.version_option(version=VERSION)
 @click.pass_context
 @coro
@@ -117,14 +117,13 @@ async def cli(
     sf_stage,
     with_headers: bool,
     show_tokens: bool,
-    local: bool,
+    prod: bool,
 ) -> None:
     """
     Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
     """
-
     # We need to unpatch for our tests not to break
-    if show_tokens or local or ctx.invoked_subcommand == "build":
+    if show_tokens or not prod or ctx.invoked_subcommand == "build":
         __unpatch_click_output()
     else:
         __patch_click_output()
@@ -211,12 +210,10 @@ async def cli(
 
     logging.debug("debug enabled")
 
-    ctx.ensure_object(dict)["client"] = (
-        await get_tinybird_local_client() if local else _get_tb_client(config.get("token", None), config["host"])
-    )
+    client = await create_ctx_client(config, prod)
 
-    for connector in SUPPORTED_CONNECTORS:
-        load_connector_config(ctx, connector, debug, check_uninstalled=True)
+    if client:
+        ctx.ensure_object(dict)["client"] = client
 
 
 @cli.command(hidden=True)
@@ -489,11 +486,10 @@ async def sql(
 
 @cli.command(hidden=True)
 @click.argument("prompt")
-@click.option("--model", default="gpt-4o", help="The model to use for the LLM")
 @click.option("--folder", default=".", help="The folder to use for the project")
 @coro
-async def llm(prompt: str, model: str, folder: str) -> None:
-    """Run a prompt against the LLM api."""
+async def ask(prompt: str, folder: str) -> None:
+    """Ask things about your data project."""
     try:
         config = CLIConfig.get_project_config(folder)
         user_token = config.get_user_token()
@@ -501,9 +497,39 @@ async def llm(prompt: str, model: str, folder: str) -> None:
         if not user_token:
             raise CLIException("This action requires authentication. Run 'tb login' first.")
 
+        datasource_paths = [
+            Path(folder) / "datasources" / f
+            for f in os.listdir(Path(folder) / "datasources")
+            if f.endswith(".datasource")
+        ]
+        pipe_folders = ["endpoints", "pipes", "copies", "materializations", "sinks"]
+        pipe_paths = [
+            Path(folder) / pipe_folder / f
+            for pipe_folder in pipe_folders
+            if (Path(folder) / pipe_folder).exists()
+            for f in os.listdir(Path(folder) / pipe_folder)
+            if f.endswith(".pipe")
+        ]
+        resources_xml = "\n".join(
+            [
+                f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
+                for resource_type, resource_name, resource_content in [
+                    ("datasource", ds.stem, ds.read_text()) for ds in datasource_paths
+                ]
+                + [
+                    (
+                        "pipe",
+                        pipe.stem,
+                        pipe.read_text(),
+                    )
+                    for pipe in pipe_paths
+                ]
+            ]
+        )
+
         client = config.get_client()
         llm = LLM(user_token=user_token, client=client)
-        click.echo(await llm.ask(prompt, model=model))
+        click.echo(await llm.ask(prompt, system_prompt=ask_prompt(resources_xml)))
     except Exception as e:
         raise CLIException(FeedbackManager.error_exception(error=e))
 
@@ -543,3 +569,13 @@ def __patch_click_output():
 def __unpatch_click_output():
     click.echo = __old_click_echo
     click.secho = __old_click_secho
+
+
+async def create_ctx_client(config: Dict[str, Any], prod: bool):
+    if prod:
+        return _get_tb_client(config.get("token", None), config["host"])
+
+    try:
+        return await get_tinybird_local_client()
+    except Exception:
+        return None