tinybird 0.0.1.dev26__py3-none-any.whl → 0.0.1.dev27__py3-none-any.whl

tinybird/sql_toolset.py

@@ -69,6 +69,7 @@ def sql_get_used_tables_cached(
     default_database: str = "",
     table_functions: bool = True,
     function_allow_list: Optional[FrozenSet[str]] = None,
+    function_deny_list: Optional[FrozenSet[str]] = None,
 ) -> List[Tuple[str, str, str]]:
     """More like: get used sql names
 
@@ -92,9 +93,13 @@ def sql_get_used_tables_cached(
     """
     try:
         _function_allow_list = list() if function_allow_list is None else list(function_allow_list)
+        _function_deny_list = list() if function_deny_list is None else list(function_deny_list)
 
         tables: List[Tuple[str, str, str]] = chquery.tables(
-            sql, default_database=default_database, function_allow_list=_function_allow_list
+            sql,
+            default_database=default_database,
+            function_allow_list=_function_allow_list,
+            function_deny_list=_function_deny_list,
         )
         if not table_functions:
             return [(t[0], t[1], "") for t in tables if t[0] or t[1]]
@@ -118,15 +123,24 @@ def sql_get_used_tables(
     default_database: str = "",
     table_functions: bool = True,
     function_allow_list: Optional[FrozenSet[str]] = None,
+    function_deny_list: Optional[FrozenSet[str]] = None,
 ) -> List[Tuple[str, str, str]]:
     """More like: get used sql names
 
     Returns a list of tuples: (database_or_namespace, table_name, table_func).
     """
-    hashable_list = frozenset() if function_allow_list is None else function_allow_list
+    function_allow_hashable_list = frozenset() if function_allow_list is None else function_allow_list
+    function_deny_hashable_list = frozenset() if function_deny_list is None else function_deny_list
 
     return copy.copy(
-        sql_get_used_tables_cached(sql, raising, default_database, table_functions, function_allow_list=hashable_list)
+        sql_get_used_tables_cached(
+            sql,
+            raising,
+            default_database,
+            table_functions,
+            function_allow_list=function_allow_hashable_list,
+            function_deny_list=function_deny_hashable_list,
+        )
     )
 
 

tinybird/syncasync.py

@@ -73,7 +73,7 @@ class Local:
     def __init__(self, thread_critical: bool = False) -> None:
         self._thread_critical = thread_critical
         self._thread_lock = threading.RLock()
-        self._context_refs: "weakref.WeakSet[object]" = weakref.WeakSet()
+        self._context_refs: weakref.WeakSet[object] = weakref.WeakSet()
         # Random suffixes stop accidental reuse between different Locals,
         # though we try to force deletion as well.
         self._attr_name = "_asgiref_local_impl_{}_{}".format(

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev26'
-__revision__ = 'f0e9917'
+__version__ = '0.0.1.dev27'
+__revision__ = 'a8944c0'

tinybird/tb/cli.py

@@ -6,11 +6,13 @@ if sys.platform == "win32":
 
 import tinybird.tb.modules.auth
 import tinybird.tb.modules.build
+import tinybird.tb.modules.build_server
 import tinybird.tb.modules.cli
 import tinybird.tb.modules.common
 import tinybird.tb.modules.connection
 import tinybird.tb.modules.create
 import tinybird.tb.modules.datasource
+import tinybird.tb.modules.deploy
 import tinybird.tb.modules.fmt
 import tinybird.tb.modules.job
 import tinybird.tb.modules.local

tinybird/tb/modules/build.py

@@ -28,22 +28,20 @@ from tinybird.tb.modules.watch import watch_files
 @click.option(
     "--folder",
     default=".",
-    help="Folder from where to execute the command. By default the current folder",
+    help="Folder from where to execute the command. By default the current folder.",
     hidden=True,
     type=click.types.STRING,
 )
 @click.option(
     "--watch",
     is_flag=True,
-    help="Watch for changes in the files and re-check them.",
+    help="Watch for changes in the files and rebuild them.",
 )
 def build(
     folder: str,
     watch: bool,
 ) -> None:
-    """
-    Watch for changes in the files and re-check them.
-    """
+    """Build the project in Tinybird Local."""
     ignore_sql_errors = FeatureFlags.ignore_sql_errors()
     context.disable_template_security_validation.set(True)
     is_internal = has_internal_datafiles(folder)
@@ -78,23 +76,40 @@ def build(
                 ignore_sql_errors=ignore_sql_errors,
                 is_internal=is_internal,
                 watch=watch,
+                folder=folder,
             )
         if len(filenames) > 0:
             filename = filenames[0]
             if filename.endswith(".ndjson"):
                 fixture_path = Path(filename)
-                name = "_".join(fixture_path.stem.split("_")[:-1])
-                ds_path = Path(folder) / "datasources" / f"{name}.datasource"
+                datasources_path = Path(folder) / "datasources"
+                ds_name = fixture_path.stem
+                ds_path = datasources_path / f"{ds_name}.datasource"
+
+                if not ds_path.exists():
+                    try:
+                        ds_name = "_".join(fixture_path.stem.split("_")[:-1])
+                        ds_path = datasources_path / f"{ds_name}.datasource"
+                    except Exception:
+                        pass
+
                 if ds_path.exists():
-                    await append_datasource(tb_client, name, str(fixture_path), silent=True)
+                    await append_datasource(tb_client, ds_name, str(fixture_path))
 
             if watch:
                 if filename.endswith(".datasource"):
                     ds_path = Path(filename)
-                    name = build_fixture_name(filename, ds_path.stem, ds_path.read_text())
-                    fixture_path = get_fixture_dir() / f"{name}.ndjson"
+                    ds_name = ds_path.stem
+                    name = build_fixture_name(filename, ds_name, ds_path.read_text())
+                    fixture_folder = get_fixture_dir(folder)
+                    fixture_path = fixture_folder / f"{name}.ndjson"
+
+                    if not fixture_path.exists():
+                        fixture_path = fixture_folder / f"{ds_name}.ndjson"
+
                     if fixture_path.exists():
-                        await append_datasource(tb_client, ds_path.stem, str(fixture_path), silent=True)
+                        await append_datasource(tb_client, ds_name, str(fixture_path))
+
                 if not filename.endswith(".ndjson"):
                     await build_and_print_resource(tb_client, filename)
 
@@ -113,13 +128,26 @@ def build(
             for filename in filenames:
                 if filename.endswith(".datasource"):
                     ds_path = Path(filename)
-                    name = build_fixture_name(filename, ds_path.stem, ds_path.read_text())
-                    fixture_path = get_fixture_dir() / f"{name}.ndjson"
+                    ds_name = ds_path.stem
+                    name = build_fixture_name(filename, ds_name, ds_path.read_text())
+                    fixture_folder = get_fixture_dir(folder)
+                    fixture_path = fixture_folder / f"{name}.ndjson"
+
+                    if not fixture_path.exists():
+                        fixture_path = fixture_folder / f"{ds_name}.ndjson"
+
                     if fixture_path.exists():
-                        await append_datasource(tb_client, ds_path.stem, str(fixture_path), silent=True)
+                        await append_datasource(tb_client, ds_name, str(fixture_path))
             click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s"))
             ok = True
         except Exception as e:
+            error_path = Path(".tb_error.txt")
+            if error_path.exists():
+                content = error_path.read_text()
+                content += f"\n\n{str(e)}"
+                error_path.write_text(content)
+            else:
+                error_path.write_text(str(e))
             click.echo(FeedbackManager.error_exception(error=e))
             ok = False
         return ok
@@ -174,13 +202,13 @@ async def append_datasource(
     tb_client: TinyB,
     datasource_name: str,
     url: str,
-    silent: bool = False,
 ):
+    await tb_client.datasource_truncate(datasource_name)
     await push_data(
         tb_client,
         datasource_name,
         url,
         mode="append",
         concurrency=1,
-        silent=silent,
+        silent=True,
     )

tinybird/tb/modules/build_server.py

@@ -0,0 +1,75 @@
+import asyncio
+import glob
+import json
+import logging
+from pathlib import Path
+from typing import List
+
+import click
+import requests
+
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.local_common import get_tinybird_local_client
+
+
+def project_files(project_path: Path) -> List[str]:
+    project_file_extensions = ("datasource", "pipe")
+    project_files = []
+    for extension in project_file_extensions:
+        for project_file in glob.glob(f"{project_path}/**/*.{extension}", recursive=True):
+            logging.debug(f"Found project file: {project_file}")
+            project_files.append(project_file)
+    return project_files
+
+
+@cli.command()
+@click.argument("project_path", type=click.Path(exists=True), default=Path.cwd())
+def build_server(project_path: Path) -> None:
+    """
+    Validate and build the project server side.
+    """
+
+    MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
+    DATAFILE_TYPE_TO_CONTENT_TYPE = {
+        ".datasource": "text/plain",
+        ".pipe": "text/plain",
+    }
+
+    tb_client = asyncio.run(get_tinybird_local_client(str(project_path)))
+    TINYBIRD_API_URL = tb_client.host + "/v1/build"
+    TINYBIRD_API_KEY = tb_client.token
+
+    files = [
+        ("context://", ("cli-version", "1.0.0", "text/plain")),
+    ]
+    fds = []
+    for file_path in project_files(project_path):
+        relative_path = str(Path(file_path).relative_to(project_path))
+        fd = open(file_path, "rb")
+        fds.append(fd)
+        content_type = DATAFILE_TYPE_TO_CONTENT_TYPE.get(Path(file_path).suffix, "application/unknown")
+        files.append((MULTIPART_BOUNDARY_DATA_PROJECT, (relative_path, fd, content_type)))
+
+    try:
+        HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
+
+        r = requests.post(TINYBIRD_API_URL, files=files, headers=HEADERS)
+        result = r.json()
+        logging.debug(json.dumps(result, indent=2))
+
+        build_result = result.get("result")
+        if build_result == "success":
+            click.echo(FeedbackManager.success(message="Build completed successfully"))
+        elif build_result == "failed":
+            click.echo(FeedbackManager.error(message="Build failed"))
+            build_errors = result.get("errors")
+            for build_error in build_errors:
+                click.echo(
+                    FeedbackManager.error(message=f"{build_error.get('filename')}\n\n{build_error.get('error')}")
+                )
+        else:
+            click.echo(FeedbackManager.error(message=f"Unknown build result. Error: {result.get('error')}"))
+    finally:
+        for fd in fds:
+            fd.close()

tinybird/tb/modules/cli.py

@@ -48,6 +48,7 @@ from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
 from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 from tinybird.tb.modules.datafile.pull import folder_pull
 from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.llm import LLM
 from tinybird.tb.modules.local_common import get_tinybird_local_client
 
 __old_click_echo = click.echo
@@ -486,6 +487,27 @@ async def sql(
         click.echo(FeedbackManager.info_no_rows())
 
 
+@cli.command(hidden=True)
+@click.argument("prompt")
+@click.option("--model", default="gpt-4o", help="The model to use for the LLM")
+@click.option("--folder", default=".", help="The folder to use for the project")
+@coro
+async def llm(prompt: str, model: str, folder: str) -> None:
+    """Run a prompt against the LLM api."""
+    try:
+        config = CLIConfig.get_project_config(folder)
+        user_token = config.get_user_token()
+
+        if not user_token:
+            raise CLIException("This action requires authentication. Run 'tb login' first.")
+
+        client = config.get_client()
+        llm = LLM(user_token=user_token, client=client)
+        click.echo(await llm.ask(prompt, model=model))
+    except Exception as e:
+        raise CLIException(FeedbackManager.error_exception(error=e))
+
+
 def __patch_click_output():
     import re
 

tinybird/tb/modules/common.py

@@ -175,7 +175,7 @@ def generate_datafile(
     if not f.exists() or force:
         with open(f"{f}", "w") as ds_file:
             ds_file.write(datafile)
-        click.echo(FeedbackManager.info_file_created(file=str(f)))
+        click.echo(FeedbackManager.info_file_created(file=f.relative_to(folder)))
 
         if data and (base / "fixtures").exists():
             # Generating a fixture for Parquet files is not so trivial, since Parquet format
@@ -289,7 +289,7 @@ def load_connector_config(ctx: Context, connector_name: str, debug: bool, check_
                 click.echo(FeedbackManager.warning_connector_not_installed(connector=connector_name))
                 return
             ctx.ensure_object(dict)[connector_name] = create_connector(connector_name, config)
-    except IOError:
+    except OSError:
         if debug:
             click.echo(f"** {connector_name} connector not configured")
         pass

tinybird/tb/modules/config.py

@@ -43,12 +43,10 @@ def compare_versions(a: str, b: str) -> int:
 
 
 class ConfigValueOrigin(Enum):
-    # Sources for config values (environment variables, .tinyb file or default value)
-
-    ENVIRONMENT: str = "env"
-    CONFIG: str = "conf"
-    DEFAULT: str = "default"
-    NONE: str = ""
+    ENVIRONMENT = "env"
+    CONFIG = "conf"
+    DEFAULT = "default"
+    NONE = ""
 
 
 @dataclass
@@ -152,7 +150,7 @@ class CLIConfig:
                 for k, v in values.items():
                     self[k] = v
             return True
-        except IOError:
+        except OSError:
             return False
 
     def override_with_environment(self) -> None:
@@ -306,13 +304,14 @@ class CLIConfig:
 
         Note: the `_path` argument is mainly intended to help during testing.
         """
-        if not CLIConfig._global:
-            path: Optional[str] = _path or os.environ.get("XDG_CONFIG_HOME", None)
-            if not path:
-                path = os.path.join(os.environ.get("HOME", "~"), ".config")
-            path = os.path.join(path, APP_CONFIG_NAME, ".tinyb")
-            CLIConfig._global = CLIConfig(path, parent=None)
-        return CLIConfig._global
+        path: Optional[str] = _path or os.environ.get("XDG_CONFIG_HOME", None)
+        if not path:
+            path = os.path.join(os.environ.get("HOME", "~"), ".config")
+        path = os.path.join(path, APP_CONFIG_NAME, ".tinyb")
+        exists_path = os.path.exists(path)
+        if not exists_path:
+            path = os.path.join(os.getcwd(), ".tinyb")
+        return CLIConfig(path, parent=None)
 
     @staticmethod
     def get_project_config(working_dir: Optional[str] = None) -> "CLIConfig":