tinybird 0.0.1.dev18__py3-none-any.whl → 0.0.1.dev20__py3-none-any.whl

tinybird/client.py

@@ -103,7 +103,7 @@ class TinyB:
         self.send_telemetry = send_telemetry
         self.semver = semver
 
-    async def _req(
+    async def _req_raw(
         self,
         endpoint: str,
         data=None,
@@ -155,10 +155,15 @@ class TinyB:
                     response = await sync_to_async(session.get, thread_sensitive=False)(
                         url, verify=verify_ssl, **kwargs
                     )
-
         except Exception as e:
             raise e
 
+        if self.send_telemetry:
+            try:
+                add_telemetry_event("api_request", endpoint=url, token=self.token, status_code=response.status_code)
+            except Exception as ex:
+                logging.exception(f"Can't send telemetry: {ex}")
+
         logging.debug("== server response ==")
         logging.debug(response.content)
         logging.debug("==      end        ==")
@@ -169,6 +174,21 @@ class TinyB:
             except Exception as ex:
                 logging.exception(f"Can't send telemetry: {ex}")
 
+        return response
+
+    async def _req(
+        self,
+        endpoint: str,
+        data=None,
+        files=None,
+        method: str = "GET",
+        retries: int = LIMIT_RETRIES,
+        use_token: Optional[str] = None,
+        **kwargs,
+    ):
+        token_to_use = use_token if use_token else self.token
+        response = await self._req_raw(endpoint, data, files, method, retries, use_token, **kwargs)
+
         if response.status_code == 403:
             error = parse_error_response(response)
             if not token_to_use:
@@ -500,7 +520,7 @@ class TinyB:
         payload = {"datasource_a": datasource_a, "datasource_b": datasource_b}
         return await self._req("/v0/datasources/exchange", method="POST", data=payload)
 
-    async def datasource_events(self, datasource_name: str, data: Dict[str, Any]):
+    async def datasource_events(self, datasource_name: str, data: str):
         return await self._req(f"/v0/events?name={datasource_name}", method="POST", data=data)
 
     async def analyze_pipe_node(
@@ -816,7 +836,7 @@ class TinyB:
         params = {"with_token": "true" if with_token else "false"}
         return await self._req(f"/v0/workspaces/{workspace_id}?{urlencode(params)}")
 
-    async def workspace_info(self):
+    async def workspace_info(self) -> Dict[str, Any]:
         return await self._req("/v0/workspace")
 
     async def wait_for_job(

tinybird/config.py

@@ -20,7 +20,7 @@ CURRENT_VERSION = f"{__cli__.__version__}"
 VERSION = f"{__cli__.__version__} (rev {__revision__})"
 DEFAULT_UI_HOST = "https://app.tinybird.co"
 SUPPORTED_CONNECTORS = ["bigquery", "snowflake"]
-PROJECT_PATHS = ["datasources", "pipes", "tests"]
+PROJECT_PATHS = ["datasources", "datasources/fixtures", "endpoints", "pipes", "tests", "scripts", "deploy"]
 DEPRECATED_PROJECT_PATHS = ["endpoints"]
 MIN_WORKSPACE_ID_LENGTH = 36
 LEGACY_HOSTS = {

tinybird/feedback_manager.py

@@ -1,6 +1,5 @@
 from collections import namedtuple
 from typing import Any, Callable
-import os
 
 FeedbackMessage = namedtuple("FeedbackMessage", "message")
 
@@ -27,16 +26,6 @@ def print_message(message: str, color: str = bcolors.ENDC) -> Callable[..., str]
 def error_message(message: str) -> Callable[..., str]:
     return print_message(f"\n** {message}", bcolors.FAIL)
 
-def error_exception(message: Exception) -> Callable[..., str]:
-    def formatter(**kwargs: Any) -> str:
-        color = bcolors.FAIL
-        if isinstance(kwargs.get('error', ''), Exception) and os.environ.get('TB_DEBUG', '') != '':
-            import traceback
-            stack_trace = f"Traceback:\n{''.join(traceback.format_tb(kwargs['error'].__traceback__))}"
-            return f"{color}{message.format(**kwargs)}{stack_trace}{bcolors.ENDC}"
-        return f"{color}{message.format(**kwargs)}{bcolors.ENDC}"
-    return formatter
-
 
 def simple_error_message(message: str) -> Callable[..., str]:
     return print_message(f"{message}", bcolors.FAIL)
@@ -62,12 +51,8 @@ def prompt_message(message: str) -> Callable[..., str]:
     return print_message(message, bcolors.HEADER)
 
 
-def gray_message(message: str) -> Callable[..., str]:
-    return print_message(message, bcolors.CGREY)
-
-
 class FeedbackManager:
-    error_exception = error_exception("{error}")
+    error_exception = error_message("{error}")
     simple_error_exception = simple_error_message("{error}")
     error_exception_trace = error_message("{error}\n** Trace:\n{trace}")
     error_notoken = error_message(
@@ -576,9 +561,9 @@ Ready? """
     )
     warning_fixture_not_found = warning_message("** Warning: No fixture found for the datasource {datasource_name}")
     warning_update_version = warning_message(
-        '** UPDATE AVAILABLE: please run "pip install tinybird=={latest_version}" to update or `export TB_VERSION_WARNING=0` to skip the check.'
+        '** UPDATE AVAILABLE: please run "pip install tinybird-cli=={latest_version}" to update or `export TB_VERSION_WARNING=0` to skip the check.'
     )
-    warning_current_version = warning_message("** current: tinybird {current_version}\n")
+    warning_current_version = warning_message("** current: tinybird-cli {current_version}\n")
     warning_confirm_truncate_datasource = prompt_message(
         "Do you want to truncate {datasource}? Once truncated, your data can't be recovered"
     )
@@ -686,7 +671,6 @@ Ready? """
     info_building_dependencies = info_message("** Building dependencies")
     info_processing_new_resource = info_message("** Running '{name}' {version}")
     info_dry_processing_new_resource = info_message("** [DRY RUN] Running '{name}' {version}")
-    info_building_resource = info_message("** Building {name}")
     info_processing_resource = info_message(
         "** Running '{name}' => v{version} (remote latest version: v{latest_version})"
     )
@@ -727,9 +711,9 @@ Ready? """
     info_removing_pipe = info_message("** Removing pipe {pipe}")
     info_removing_pipe_not_found = info_message("** {pipe} not found")
     info_dry_removing_pipe = info_message("** [DRY RUN] Removing pipe {pipe}")
-    info_path_created = info_message("✓ /{path}")
-    info_file_created = info_message("✓ /{file}")
-    info_path_already_exists = info_message("✓ /{path} already exists, skipping")
+    info_path_created = info_message("** - /{path} created")
+    info_file_created = info_message("** - {file} created")
+    info_path_already_exists = info_message("** - /{path} already exists, skipping")
     info_dottinyb_already_ignored = info_message("** - '.tinyb' already in .gitignore, skipping")
     info_dotdifftemp_already_ignored = info_message("** - '.diff_tmp' not found or already in .gitignore, skipping")
     info_dottinyenv_already_exists = info_message("** - '.tinyenv' already exists, skipping")
@@ -803,7 +787,7 @@ Ready? """
     info_diff_resources_for_git_init = info_message(
         "** Checking diffs between remote Workspace and local. Hint: use 'tb diff' to check if your Data Project and Workspace synced"
     )
-    info_cicd_file_generated = info_message("✓ {file_path}")
+    info_cicd_file_generated = info_message("** File {file_path} generated for CI/CD")
     info_available_git_providers = info_message("** List of available providers:")
     info_git_release_init_without_diffs = info_message("** No diffs detected for '{workspace}'")
     info_deployment_detecting_changes_header = info_message("\n** Detecting changes from last commit ...")
@@ -950,7 +934,7 @@ Ready? """
     success_create = success_message("** '{name}' created")
     success_delete = success_message("** '{name}' deleted")
     success_dynamodb_initial_load = success_message("** Initial load of DynamoDB table started: {job_url}")
-    success_progress_blocks = success_message("✓ Done!")
+    success_progress_blocks = success_message("** \N{FRONT-FACING BABY CHICK} done")
     success_now_using_config = success_message("** Now using {name} ({id})")
     success_connector_config = success_message(
         "** {connector} configuration written to {file_name} file, consider adding it to .gitignore"
@@ -1036,9 +1020,3 @@ Ready? """
     success_tag_removed = success_message("** Tag '{tag_name}' removed!")
 
     debug_running_file = print_message("** Running {file}", bcolors.CGREY)
-
-    success = success_message("{message}")
-    info = info_message("{message}")
-    highlight = info_highlight_message("{message}")
-    error = error_message("{message}")
-    gray = gray_message("{message}")

tinybird/tb/__cli__.py

@@ -0,0 +1,8 @@
+
+__name__ = 'tinybird'
+__description__ = 'Tinybird Command Line Tool'
+__url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
+__author__ = 'Tinybird'
+__author_email__ = 'support@tinybird.co'
+__version__ = '0.0.1.dev20'
+__revision__ = '6125327'

tinybird/tb/modules/auth.py

@@ -10,7 +10,6 @@ import click
 import humanfriendly.tables
 
 from tinybird.config import get_display_host
-from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import (
     configure_connector,
@@ -23,6 +22,7 @@ from tinybird.tb.modules.common import (
 )
 from tinybird.tb.modules.config import CLIConfig, ConfigValueOrigin
 from tinybird.tb.modules.exceptions import CLIAuthException
+from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.regions import Region
 
 

tinybird/tb/modules/build.py

@@ -3,17 +3,13 @@ import os
 import threading
 import time
 from pathlib import Path
-from typing import Any, Awaitable, Callable, List, Union
+from typing import List
 
 import click
-from watchdog.events import FileSystemEventHandler
-from watchdog.observers import Observer
 
 import tinybird.context as context
 from tinybird.client import TinyB
 from tinybird.config import FeatureFlags
-from tinybird.feedback_manager import FeedbackManager
-from tinybird.tb.modules.build_shell import BuildShell, print_table_formatted
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import push_data
 from tinybird.tb.modules.datafile.build import folder_build
@@ -22,69 +18,10 @@ from tinybird.tb.modules.datafile.exceptions import ParseException
 from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
 from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
 from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
+from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.local_common import get_tinybird_local_client
-
-
-class FileChangeHandler(FileSystemEventHandler):
-    def __init__(self, filenames: List[str], process: Callable[[List[str]], None], build_ok: bool):
-        self.filenames = filenames
-        self.process = process
-        self.build_ok = build_ok
-
-    def on_modified(self, event: Any) -> None:
-        is_not_vendor = "vendor/" not in event.src_path
-        if (
-            is_not_vendor
-            and not event.is_directory
-            and any(event.src_path.endswith(ext) for ext in [".datasource", ".pipe", ".ndjson"])
-        ):
-            filename = event.src_path.split("/")[-1]
-            click.echo(FeedbackManager.highlight(message=f"\n\n⟲ Changes detected in {filename}\n"))
-            try:
-                to_process = [event.src_path] if self.build_ok else self.filenames
-                self.process(to_process)
-                self.build_ok = True
-            except Exception as e:
-                click.echo(FeedbackManager.error_exception(error=e))
-
-
-def watch_files(
-    filenames: List[str],
-    process: Union[Callable[[List[str]], None], Callable[[List[str]], Awaitable[None]]],
-    shell: BuildShell,
-    folder: str,
-    build_ok: bool,
-) -> None:
-    # Handle both sync and async process functions
-    async def process_wrapper(files: List[str]) -> None:
-        click.echo("⚡ Rebuilding...")
-        time_start = time.time()
-        if asyncio.iscoroutinefunction(process):
-            await process(files, watch=True)
-        else:
-            process(files, watch=True)
-        time_end = time.time()
-        elapsed_time = time_end - time_start
-        click.echo(
-            FeedbackManager.success(message="\n✓ ")
-            + FeedbackManager.gray(message=f"Rebuild completed in {elapsed_time:.1f}s")
-        )
-        shell.reprint_prompt()
-
-    event_handler = FileChangeHandler(filenames, lambda f: asyncio.run(process_wrapper(f)), build_ok)
-    observer = Observer()
-
-    observer.schedule(event_handler, path=folder, recursive=True)
-
-    observer.start()
-
-    try:
-        while True:
-            time.sleep(1)
-    except KeyboardInterrupt:
-        observer.stop()
-
-    observer.join()
+from tinybird.tb.modules.shell import Shell, print_table_formatted
+from tinybird.tb.modules.watch import watch_files
 
 
 @cli.command()
@@ -119,7 +56,7 @@ def build(
 
         for filename in filenames:
             if os.path.isdir(filename):
-                process(filenames=get_project_filenames(filename))
+                check_filenames(filenames=get_project_filenames(filename))
 
             file_suffix = Path(filename).suffix
             if file_suffix == incl_suffix:
@@ -149,7 +86,7 @@ def build(
                 name = "_".join(fixture_path.stem.split("_")[:-1])
                 ds_path = Path(folder) / "datasources" / f"{name}.datasource"
                 if ds_path.exists():
-                    await append_datasource({}, tb_client, name, str(fixture_path), silent=True)
+                    await append_datasource(tb_client, name, str(fixture_path), silent=True)
 
             if watch:
                 if filename.endswith(".datasource"):
@@ -157,7 +94,7 @@ def build(
                     name = build_fixture_name(filename, ds_path.stem, ds_path.read_text())
                     fixture_path = get_fixture_dir() / f"{name}.ndjson"
                     if fixture_path.exists():
-                        await append_datasource({}, tb_client, ds_path.stem, str(fixture_path), silent=True)
+                        await append_datasource(tb_client, ds_path.stem, str(fixture_path), silent=True)
                 if not filename.endswith(".ndjson"):
                     await build_and_print_resource(tb_client, filename)
 
@@ -179,7 +116,7 @@ def build(
                     name = build_fixture_name(filename, ds_path.stem, ds_path.read_text())
                     fixture_path = get_fixture_dir() / f"{name}.ndjson"
                     if fixture_path.exists():
-                        await append_datasource({}, tb_client, ds_path.stem, str(fixture_path), silent=True)
+                        await append_datasource(tb_client, ds_path.stem, str(fixture_path), silent=True)
             click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s\n"))
             ok = True
         except Exception as e:
@@ -193,22 +130,36 @@ def build(
         paths = [Path(f) for f in get_project_filenames(folder, with_vendor=True)]
 
         def is_vendor(f: Path) -> bool:
-            return "vendor/" in f.parts
+            return f.parts[0] == "vendor"
 
         def get_vendor_workspace(f: Path) -> str:
             return f.parts[1]
 
+        def is_endpoint(f: Path) -> bool:
+            return f.suffix == ".pipe" and not is_vendor(f) and f.parts[0] == "endpoints"
+
+        def is_pipe(f: Path) -> bool:
+            return f.suffix == ".pipe" and not is_vendor(f)
+
         datasource_paths = [f for f in paths if f.suffix == ".datasource"]
         datasources = [f.stem for f in datasource_paths if not is_vendor(f)]
         shared_datasources = [f"{get_vendor_workspace(f)}.{f.stem}" for f in datasource_paths if is_vendor(f)]
-        pipes = [f.stem for f in paths if f.suffix == ".pipe" and not is_vendor(f)]
-        shell = BuildShell(folder=folder, client=tb_client, datasources=datasources + shared_datasources, pipes=pipes)
+        pipes = [f.stem for f in paths if is_pipe(f) and not is_endpoint(f)]
+        endpoints = [f.stem for f in paths if is_endpoint(f)]
+        shell = Shell(
+            folder=folder,
+            client=tb_client,
+            datasources=datasources,
+            shared_datasources=shared_datasources,
+            pipes=pipes,
+            endpoints=endpoints,
+        )
         click.echo(FeedbackManager.highlight(message="◎ Watching for changes..."))
         watcher_thread = threading.Thread(
             target=watch_files, args=(filenames, process, shell, folder, build_ok), daemon=True
         )
         watcher_thread.start()
-        shell.run_shell()
+        shell.run()
 
 
 async def build_and_print_resource(tb_client: TinyB, filename: str):
@@ -220,21 +171,16 @@ async def build_and_print_resource(tb_client: TinyB, filename: str):
 
 
 async def append_datasource(
-    ctx: click.Context,
     tb_client: TinyB,
     datasource_name: str,
     url: str,
     silent: bool = False,
 ):
     await push_data(
-        ctx,
         tb_client,
         datasource_name,
         url,
-        connector=None,
-        sql=None,
         mode="append",
-        ignore_empty=False,
         concurrency=1,
         silent=silent,
     )

tinybird/tb/modules/cicd.py

@@ -6,7 +6,7 @@ from typing import Any, Dict, List, Optional, Type, Union
 import click
 from tornado.template import Template
 
-from tinybird.feedback_manager import FeedbackManager
+from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 
 class Provider(Enum):