tinybird 0.0.1.dev29__py3-none-any.whl → 0.0.1.dev31__py3-none-any.whl

tinybird/sql_template.py

@@ -12,7 +12,11 @@ from typing import Any, Dict, List, Optional, Tuple, Union
 from tornado import escape
 from tornado.util import ObjectDict, exec_in, unicode_type
 
-from tinybird.context import ff_preprocess_parameters_circuit_breaker, ff_split_to_array_escape
+from tinybird.context import (
+    ff_column_json_backticks_circuit_breaker,
+    ff_preprocess_parameters_circuit_breaker,
+    ff_split_to_array_escape,
+)
 
 from .datatypes import testers
 from .tornado_template import VALID_CUSTOM_FUNCTION_NAMES, SecurityException, Template
@@ -271,6 +275,8 @@ def columns(x, default=None, fn=None):
 
 
 def column(x, default=None):
+    bypass_colunn_json_backticks = ff_column_json_backticks_circuit_breaker.get(False)
+
     if x is None or isinstance(x, Placeholder):
         if default is None:
             raise SQLTemplateException(
@@ -278,7 +284,18 @@ def column(x, default=None):
                 documentation="/cli/advanced-templates.html#column",
             )
         x = default
-    return Symbol("`" + sqlescape(x) + "`")
+
+    quote = "`"
+    if bypass_colunn_json_backticks:
+        return Symbol(quote + sqlescape(x) + quote)
+
+    try:
+        slices = x.split(".")
+        escaped_slices = [quote + sqlescape(s) + quote for s in slices]
+        escaped = ".".join(escaped_slices)
+        return Symbol(escaped)
+    except Exception:  # in case there's a problem with .split
+        return Symbol(quote + sqlescape(x) + quote)
 
 
 def symbol(x, quote="`"):
@@ -2109,6 +2126,8 @@ def render_sql_template(
     ('SELECT `foo`', {}, [])
     >>> render_sql_template("SELECT {{column(agg)}}", {'agg': '"foo"'})
     ('SELECT `foo`', {}, [])
+    >>> render_sql_template("SELECT {{column(agg)}}", {'agg': 'json.a'})
+    ('SELECT `json`.`a`', {}, [])
     >>> render_sql_template('{% if not defined(test) %}error("This is an error"){% end %}', {})
     ('error("This is an error")', {}, [])
     >>> render_sql_template('{% if not defined(test) %}custom_error({error: "This is an error"}){% end %}', {})

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev29'
-__revision__ = '1ff2823'
+__version__ = '0.0.1.dev31'
+__revision__ = '0e056e8'

tinybird/tb/cli.py

@@ -6,7 +6,7 @@ if sys.platform == "win32":
 
 import tinybird.tb.modules.auth
 import tinybird.tb.modules.build
-import tinybird.tb.modules.build_server
+import tinybird.tb.modules.build_client
 import tinybird.tb.modules.cli
 import tinybird.tb.modules.common
 import tinybird.tb.modules.connection
@@ -22,6 +22,7 @@ import tinybird.tb.modules.pipe
 import tinybird.tb.modules.tag
 import tinybird.tb.modules.test
 import tinybird.tb.modules.token
+import tinybird.tb.modules.update
 import tinybird.tb.modules.workspace
 import tinybird.tb.modules.workspace_members
 

tinybird/tb/modules/build.py

@@ -1,214 +1,101 @@
 import asyncio
-import os
+import glob
+import json
+import logging
 import threading
-import time
 from pathlib import Path
 from typing import List
 
 import click
+import requests
 
-import tinybird.context as context
 from tinybird.client import TinyB
-from tinybird.config import FeatureFlags
 from tinybird.tb.modules.cli import cli
-from tinybird.tb.modules.common import push_data
-from tinybird.tb.modules.datafile.build import folder_build
-from tinybird.tb.modules.datafile.common import get_project_filenames, get_project_fixtures, has_internal_datafiles
-from tinybird.tb.modules.datafile.exceptions import ParseException
-from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
-from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
-from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.local_common import get_tinybird_local_client
-from tinybird.tb.modules.shell import Shell, print_table_formatted
-from tinybird.tb.modules.watch import watch_files
+from tinybird.tb.modules.shell import Shell
+from tinybird.tb.modules.watch import watch_project
 
 
 @cli.command()
-@click.option(
-    "--folder",
-    default=".",
-    help="Folder from where to execute the command. By default the current folder.",
-    hidden=True,
-    type=click.types.STRING,
-)
-@click.option(
-    "--watch",
-    is_flag=True,
-    help="Watch for changes in the files and rebuild them.",
-)
-def build(
-    folder: str,
-    watch: bool,
-) -> None:
-    """Build the project in Tinybird Local."""
-    ignore_sql_errors = FeatureFlags.ignore_sql_errors()
-    context.disable_template_security_validation.set(True)
-    is_internal = has_internal_datafiles(folder)
-    folder_path = os.path.abspath(folder)
-    tb_client = asyncio.run(get_tinybird_local_client(folder_path))
+@click.option("--folder", type=str, default=".")
+@click.option("--watch", is_flag=True, default=False, help="Watch for changes and rebuild automatically")
+def build(folder: str, watch: bool) -> None:
+    """
+    Validate and build the project server side.
+    """
 
-    def check_filenames(filenames: List[str]):
-        parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
-        incl_suffix = ".incl"
+    tb_client = asyncio.run(get_tinybird_local_client(folder))
 
-        for filename in filenames:
-            if os.path.isdir(filename):
-                check_filenames(filenames=get_project_filenames(filename))
+    def process() -> None:
+        build_project(folder, tb_client)
 
-            file_suffix = Path(filename).suffix
-            if file_suffix == incl_suffix:
-                continue
-
-            parser = parser_matrix.get(file_suffix)
-            if not parser:
-                raise ParseException(FeedbackManager.error_unsupported_datafile(extension=file_suffix))
-
-            parser(filename)
-
-    async def process(filenames: List[str], watch: bool = False):
-        datafiles = [f for f in filenames if f.endswith(".datasource") or f.endswith(".pipe")]
-        if len(datafiles) > 0:
-            check_filenames(filenames=datafiles)
-            await folder_build(
-                tb_client,
-                filenames=datafiles,
-                ignore_sql_errors=ignore_sql_errors,
-                is_internal=is_internal,
-                watch=watch,
-                folder=folder,
-            )
-        if len(filenames) > 0:
-            filename = filenames[0]
-            if filename.endswith(".ndjson"):
-                fixture_path = Path(filename)
-                datasources_path = Path(folder) / "datasources"
-                ds_name = fixture_path.stem
-                ds_path = datasources_path / f"{ds_name}.datasource"
-
-                if not ds_path.exists():
-                    try:
-                        ds_name = "_".join(fixture_path.stem.split("_")[:-1])
-                        ds_path = datasources_path / f"{ds_name}.datasource"
-                    except Exception:
-                        pass
-
-                if ds_path.exists():
-                    await append_datasource(tb_client, ds_name, str(fixture_path))
-
-            if watch:
-                if filename.endswith(".datasource"):
-                    ds_path = Path(filename)
-                    ds_name = ds_path.stem
-                    name = build_fixture_name(filename, ds_name, ds_path.read_text())
-                    fixture_folder = get_fixture_dir(folder)
-                    fixture_path = fixture_folder / f"{name}.ndjson"
-
-                    if not fixture_path.exists():
-                        fixture_path = fixture_folder / f"{ds_name}.ndjson"
-
-                    if fixture_path.exists():
-                        await append_datasource(tb_client, ds_name, str(fixture_path))
-
-                if not filename.endswith(".ndjson"):
-                    await build_and_print_resource(tb_client, filename)
-
-    datafiles = get_project_filenames(folder)
-    fixtures = get_project_fixtures(folder)
-    filenames = datafiles + fixtures
-
-    async def build_once(filenames: List[str]):
-        ok = False
-        try:
-            click.echo(FeedbackManager.highlight(message="» Building project...\n"))
-            time_start = time.time()
-            await process(filenames=filenames, watch=False)
-            time_end = time.time()
-            elapsed_time = time_end - time_start
-            for filename in filenames:
-                if filename.endswith(".datasource"):
-                    ds_path = Path(filename)
-                    ds_name = ds_path.stem
-                    name = build_fixture_name(filename, ds_name, ds_path.read_text())
-                    fixture_folder = get_fixture_dir(folder)
-                    fixture_path = fixture_folder / f"{name}.ndjson"
-
-                    if not fixture_path.exists():
-                        fixture_path = fixture_folder / f"{ds_name}.ndjson"
-
-                    if fixture_path.exists():
-                        await append_datasource(tb_client, ds_name, str(fixture_path))
-            click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s"))
-            ok = True
-        except Exception as e:
-            error_path = Path(".tb_error.txt")
-            if error_path.exists():
-                content = error_path.read_text()
-                content += f"\n\n{str(e)}"
-                error_path.write_text(content)
-            else:
-                error_path.write_text(str(e))
-            click.echo(FeedbackManager.error_exception(error=e))
-            ok = False
-        return ok
-
-    build_ok = asyncio.run(build_once(filenames))
+    process()
 
     if watch:
-        paths = [Path(f) for f in get_project_filenames(folder, with_vendor=True)]
-
-        def is_vendor(f: Path) -> bool:
-            return f.parts[0] == "vendor"
-
-        def get_vendor_workspace(f: Path) -> str:
-            return f.parts[1]
-
-        def is_endpoint(f: Path) -> bool:
-            return f.suffix == ".pipe" and not is_vendor(f) and f.parts[0] == "endpoints"
-
-        def is_pipe(f: Path) -> bool:
-            return f.suffix == ".pipe" and not is_vendor(f)
-
-        datasource_paths = [f for f in paths if f.suffix == ".datasource"]
-        datasources = [f.stem for f in datasource_paths if not is_vendor(f)]
-        shared_datasources = [f"{get_vendor_workspace(f)}.{f.stem}" for f in datasource_paths if is_vendor(f)]
-        pipes = [f.stem for f in paths if is_pipe(f) and not is_endpoint(f)]
-        endpoints = [f.stem for f in paths if is_endpoint(f)]
-        shell = Shell(
-            folder=folder,
-            client=tb_client,
-            datasources=datasources,
-            shared_datasources=shared_datasources,
-            pipes=pipes,
-            endpoints=endpoints,
-        )
+        shell = Shell(folder=folder, client=tb_client)
         click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
         watcher_thread = threading.Thread(
-            target=watch_files, args=(filenames, process, shell, folder, build_ok), daemon=True
+            target=watch_project,
+            args=(shell, process, folder),
+            daemon=True,
         )
         watcher_thread.start()
         shell.run()
 
 
-async def build_and_print_resource(tb_client: TinyB, filename: str):
-    resource_path = Path(filename)
-    name = resource_path.stem
-    pipeline = name if filename.endswith(".pipe") else None
-    res = await tb_client.query(f"SELECT * FROM {name} FORMAT JSON", pipeline=pipeline)
-    print_table_formatted(res, name)
-
-
-async def append_datasource(
-    tb_client: TinyB,
-    datasource_name: str,
-    url: str,
-):
-    await tb_client.datasource_truncate(datasource_name)
-    await push_data(
-        tb_client,
-        datasource_name,
-        url,
-        mode="append",
-        concurrency=1,
-        silent=True,
-    )
+def get_project_files(project_path: Path) -> List[str]:
+    project_file_extensions = ("datasource", "pipe")
+    project_files = []
+    for extension in project_file_extensions:
+        for project_file in glob.glob(f"{project_path}/**/*.{extension}", recursive=True):
+            logging.debug(f"Found project file: {project_file}")
+            project_files.append(project_file)
+    return project_files
+
+
+def build_project(folder: str, tb_client: TinyB) -> None:
+    MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
+    DATAFILE_TYPE_TO_CONTENT_TYPE = {
+        ".datasource": "text/plain",
+        ".pipe": "text/plain",
+    }
+    TINYBIRD_API_URL = tb_client.host + "/v1/build"
+    TINYBIRD_API_KEY = tb_client.token
+    try:
+        files = [
+            ("context://", ("cli-version", "1.0.0", "text/plain")),
+        ]
+        fds = []
+        project_path = Path(folder)
+        project_files = get_project_files(project_path)
+        for file_path in project_files:
+            relative_path = str(Path(file_path).relative_to(project_path))
+            fd = open(file_path, "rb")
+            fds.append(fd)
+            content_type = DATAFILE_TYPE_TO_CONTENT_TYPE.get(Path(file_path).suffix, "application/unknown")
+            files.append((MULTIPART_BOUNDARY_DATA_PROJECT, (relative_path, fd.read().decode("utf-8"), content_type)))
+        HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
+
+        r = requests.post(TINYBIRD_API_URL, files=files, headers=HEADERS)
+        result = r.json()
+
+        logging.debug(json.dumps(result, indent=2))
+
+        build_result = result.get("result")
+        if build_result == "success":
+            click.echo(FeedbackManager.success(message="Build completed successfully"))
+        elif build_result == "failed":
+            click.echo(FeedbackManager.error(message="Build failed"))
+            build_errors = result.get("errors")
+            for build_error in build_errors:
+                filename_bit = f"{build_error.get('filename', '')}"
+                error_msg = (filename_bit + "\n\n") if filename_bit else "" + build_error.get("error")
+                click.echo(FeedbackManager.error(message=error_msg))
+        else:
+            click.echo(FeedbackManager.error(message=f"Unknown build result. Error: {result.get('error')}"))
+    except Exception as e:
+        click.echo(FeedbackManager.error_exception(error="Error building project: " + str(e)))
+    finally:
+        for fd in fds:
+            fd.close()

tinybird/tb/modules/build_client.py

@@ -0,0 +1,219 @@
+import asyncio
+import os
+import threading
+import time
+from pathlib import Path
+from typing import List
+
+import click
+
+import tinybird.context as context
+from tinybird.client import TinyB
+from tinybird.config import FeatureFlags
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.common import push_data
+from tinybird.tb.modules.datafile.build import folder_build
+from tinybird.tb.modules.datafile.common import get_project_filenames, get_project_fixtures, has_internal_datafiles
+from tinybird.tb.modules.datafile.exceptions import ParseException
+from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
+from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
+from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.local_common import get_tinybird_local_client
+from tinybird.tb.modules.shell import Shell, print_table_formatted
+from tinybird.tb.modules.watch import watch_files
+
+
+def is_vendor(f: Path) -> bool:
+    return f.parts[0] == "vendor"
+
+
+def get_vendor_workspace(f: Path) -> str:
+    return f.parts[1]
+
+
+def is_endpoint(f: Path) -> bool:
+    return f.suffix == ".pipe" and not is_vendor(f) and f.parts[0] == "endpoints"
+
+
+def is_pipe(f: Path) -> bool:
+    return f.suffix == ".pipe" and not is_vendor(f)
+
+
+def check_filenames(filenames: List[str]):
+    parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
+    incl_suffix = ".incl"
+
+    for filename in filenames:
+        if os.path.isdir(filename):
+            check_filenames(filenames=get_project_filenames(filename))
+
+        file_suffix = Path(filename).suffix
+        if file_suffix == incl_suffix:
+            continue
+
+        parser = parser_matrix.get(file_suffix)
+        if not parser:
+            raise ParseException(FeedbackManager.error_unsupported_datafile(extension=file_suffix))
+
+        parser(filename)
+
+
+@cli.command()
+@click.option(
+    "--folder",
+    default=".",
+    help="Folder from where to execute the command. By default the current folder.",
+    hidden=True,
+    type=click.types.STRING,
+)
+@click.option(
+    "--watch",
+    is_flag=True,
+    help="Watch for changes in the files and rebuild them.",
+)
+def build_client(
+    folder: str,
+    watch: bool,
+) -> None:
+    """Build the project in Tinybird Local."""
+    ignore_sql_errors = FeatureFlags.ignore_sql_errors()
+    context.disable_template_security_validation.set(True)
+    is_internal = has_internal_datafiles(folder)
+    folder_path = os.path.abspath(folder)
+    tb_client = asyncio.run(get_tinybird_local_client(folder_path))
+
+    async def process(filenames: List[str], watch: bool = False):
+        datafiles = [f for f in filenames if f.endswith(".datasource") or f.endswith(".pipe")]
+        if len(datafiles) > 0:
+            check_filenames(filenames=datafiles)
+            await folder_build(
+                tb_client,
+                filenames=datafiles,
+                ignore_sql_errors=ignore_sql_errors,
+                is_internal=is_internal,
+                watch=watch,
+                folder=folder,
+            )
+        if len(filenames) > 0:
+            filename = filenames[0]
+            if filename.endswith(".ndjson"):
+                fixture_path = Path(filename)
+                datasources_path = Path(folder) / "datasources"
+                ds_name = fixture_path.stem
+                ds_path = datasources_path / f"{ds_name}.datasource"
+
+                if not ds_path.exists():
+                    try:
+                        ds_name = "_".join(fixture_path.stem.split("_")[:-1])
+                        ds_path = datasources_path / f"{ds_name}.datasource"
+                    except Exception:
+                        pass
+
+                if ds_path.exists():
+                    await append_datasource(tb_client, ds_name, str(fixture_path))
+
+            if watch:
+                if filename.endswith(".datasource"):
+                    ds_path = Path(filename)
+                    ds_name = ds_path.stem
+                    name = build_fixture_name(filename, ds_name, ds_path.read_text())
+                    fixture_folder = get_fixture_dir(folder)
+                    fixture_path = fixture_folder / f"{name}.ndjson"
+
+                    if not fixture_path.exists():
+                        fixture_path = fixture_folder / f"{ds_name}.ndjson"
+
+                    if fixture_path.exists():
+                        await append_datasource(tb_client, ds_name, str(fixture_path))
+
+                if not filename.endswith(".ndjson"):
+                    await build_and_print_resource(tb_client, filename)
+
+    datafiles = get_project_filenames(folder)
+    fixtures = get_project_fixtures(folder)
+    filenames = datafiles + fixtures
+
+    async def build_once(filenames: List[str]):
+        ok = False
+        try:
+            click.echo(FeedbackManager.highlight(message="» Building project...\n"))
+            time_start = time.time()
+            await process(filenames=filenames, watch=False)
+            time_end = time.time()
+            elapsed_time = time_end - time_start
+            for filename in filenames:
+                if filename.endswith(".datasource"):
+                    ds_path = Path(filename)
+                    ds_name = ds_path.stem
+                    name = build_fixture_name(filename, ds_name, ds_path.read_text())
+                    fixture_folder = get_fixture_dir(folder)
+                    fixture_path = fixture_folder / f"{name}.ndjson"
+
+                    if not fixture_path.exists():
+                        fixture_path = fixture_folder / f"{ds_name}.ndjson"
+
+                    if fixture_path.exists():
+                        await append_datasource(tb_client, ds_name, str(fixture_path))
+            click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s"))
+            ok = True
+        except Exception as e:
+            error_path = Path(".tb_error.txt")
+            if error_path.exists():
+                content = error_path.read_text()
+                content += f"\n\n{str(e)}"
+                error_path.write_text(content)
+            else:
+                error_path.write_text(str(e))
+            click.echo(FeedbackManager.error_exception(error=e))
+            ok = False
+        return ok
+
+    build_ok = asyncio.run(build_once(filenames))
+
+    if watch:
+        paths = [Path(f) for f in get_project_filenames(folder, with_vendor=True)]
+
+        datasource_paths = [f for f in paths if f.suffix == ".datasource"]
+        datasources = [f.stem for f in datasource_paths if not is_vendor(f)]
+        shared_datasources = [f"{get_vendor_workspace(f)}.{f.stem}" for f in datasource_paths if is_vendor(f)]
+        pipes = [f.stem for f in paths if is_pipe(f) and not is_endpoint(f)]
+        endpoints = [f.stem for f in paths if is_endpoint(f)]
+        shell = Shell(
+            folder=folder,
+            client=tb_client,
+            datasources=datasources,
+            shared_datasources=shared_datasources,
+            pipes=pipes,
+            endpoints=endpoints,
+        )
+        click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
+        watcher_thread = threading.Thread(
+            target=watch_files, args=(filenames, process, shell, folder, build_ok), daemon=True
+        )
+        watcher_thread.start()
+        shell.run()
+
+
+async def build_and_print_resource(tb_client: TinyB, filename: str):
+    resource_path = Path(filename)
+    name = resource_path.stem
+    pipeline = name if filename.endswith(".pipe") else None
+    res = await tb_client.query(f"SELECT * FROM {name} FORMAT JSON", pipeline=pipeline)
+    print_table_formatted(res, name)
+
+
+async def append_datasource(
+    tb_client: TinyB,
+    datasource_name: str,
+    url: str,
+):
+    await tb_client.datasource_truncate(datasource_name)
+    await push_data(
+        tb_client,
+        datasource_name,
+        url,
+        mode="append",
+        concurrency=1,
+        silent=True,
+    )