tinybird 0.0.1.dev1__py3-none-any.whl → 0.0.1.dev2__py3-none-any.whl

tinybird/feedback_manager.py

@@ -671,6 +671,7 @@ Ready? """
     info_building_dependencies = info_message("** Building dependencies")
     info_processing_new_resource = info_message("** Running '{name}' {version}")
     info_dry_processing_new_resource = info_message("** [DRY RUN] Running '{name}' {version}")
+    info_building_resource = info_message("** Building {name}")
     info_processing_resource = info_message(
         "** Running '{name}' => v{version} (remote latest version: v{latest_version})"
     )
@@ -1020,3 +1021,8 @@ Ready? """
     success_tag_removed = success_message("** Tag '{tag_name}' removed!")
 
     debug_running_file = print_message("** Running {file}", bcolors.CGREY)
+
+    success = success_message("{message}")
+    info = info_message("{message}")
+    highlight = info_highlight_message("{message}")
+    error = error_message("{message}")

tinybird/tb_cli.py

@@ -6,9 +6,11 @@ if sys.platform == "win32":
 
 import tinybird.tb_cli_modules.auth
 import tinybird.tb_cli_modules.branch
+import tinybird.tb_cli_modules.build
 import tinybird.tb_cli_modules.cli
 import tinybird.tb_cli_modules.common
 import tinybird.tb_cli_modules.connection
+import tinybird.tb_cli_modules.create
 import tinybird.tb_cli_modules.datasource
 import tinybird.tb_cli_modules.fmt
 import tinybird.tb_cli_modules.job

tinybird/tb_cli_modules/build.py

@@ -0,0 +1,221 @@
+import asyncio
+import json
+import os
+import time
+from pathlib import Path
+from typing import Any, Awaitable, Callable, Dict, List, Union
+
+import click
+from watchdog.events import FileSystemEventHandler
+from watchdog.observers import Observer
+
+import tinybird.context as context
+from tinybird.client import TinyB
+from tinybird.config import FeatureFlags
+from tinybird.datafile import (
+    ParseException,
+    folder_build,
+    get_project_filenames,
+    has_internal_datafiles,
+    parse_datasource,
+    parse_pipe,
+)
+from tinybird.feedback_manager import FeedbackManager, info_highlight_message, success_message
+from tinybird.tb_cli_modules.cli import cli
+from tinybird.tb_cli_modules.common import (
+    coro,
+    echo_safe_humanfriendly_tables_format_smart_table,
+)
+from tinybird.tb_cli_modules.create import generate_sample_data_from_columns
+from tinybird.tb_cli_modules.local import (
+    get_docker_client,
+    get_tinybird_local_client,
+    remove_tinybird_local,
+    start_tinybird_local,
+    stop_tinybird_local,
+)
+
+
+class FileChangeHandler(FileSystemEventHandler):
+    def __init__(self, filenames: List[str], process: Callable[[List[str]], None]):
+        self.filenames = filenames
+        self.process = process
+
+    def on_modified(self, event: Any) -> None:
+        if not event.is_directory and any(event.src_path.endswith(ext) for ext in [".datasource", ".pipe"]):
+            filename = event.src_path.split("/")[-1]
+            click.echo(info_highlight_message(f"\n⟲ Changes detected in {filename}\n")())
+            try:
+                self.process([event.src_path])
+            except Exception as e:
+                click.echo(FeedbackManager.error_exception(error=e))
+
+
+def watch_files(
+    filenames: List[str],
+    process: Union[Callable[[List[str]], None], Callable[[List[str]], Awaitable[None]]],
+) -> None:
+    # Handle both sync and async process functions
+    async def process_wrapper(files: List[str]) -> None:
+        click.echo("⚡ Rebuilding...")
+        time_start = time.time()
+        if asyncio.iscoroutinefunction(process):
+            await process(files, watch=True)
+        else:
+            process(files, watch=True)
+        time_end = time.time()
+        elapsed_time = time_end - time_start
+        click.echo(success_message(f"\n✓ Rebuild completed in {elapsed_time:.1f}s")())
+
+    event_handler = FileChangeHandler(filenames, lambda f: asyncio.run(process_wrapper(f)))
+    observer = Observer()
+
+    # Watch each provided path
+    for filename in filenames:
+        path = filename if os.path.isdir(filename) else os.path.dirname(filename)
+        observer.schedule(event_handler, path=path, recursive=True)
+
+    observer.start()
+
+    try:
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        observer.stop()
+
+    observer.join()
+
+
+@cli.command()
+@click.option(
+    "--folder",
+    default=".",
+    help="Folder from where to execute the command. By default the current folder",
+    hidden=True,
+    type=click.types.STRING,
+)
+@click.option(
+    "--watch",
+    is_flag=True,
+    help="Watch for changes in the files and re-check them.",
+)
+@click.option(
+    "--restart",
+    is_flag=True,
+    help="Restart the Tinybird development environment before building the first time.",
+)
+@coro
+async def build(
+    folder: str,
+    watch: bool,
+    restart: bool,
+) -> None:
+    """
+    Watch for changes in the files and re-check them.
+    """
+    docker_client = get_docker_client()
+    if restart:
+        remove_tinybird_local(docker_client)
+        start_tinybird_local(docker_client)
+    ignore_sql_errors = FeatureFlags.ignore_sql_errors()
+    context.disable_template_security_validation.set(True)
+    is_internal = has_internal_datafiles(folder)
+    tb_client = get_tinybird_local_client()
+    workspaces: List[Dict[str, Any]] = (await tb_client.user_workspaces_and_branches()).get("workspaces", [])
+    datasources: List[Dict[str, Any]] = await tb_client.datasources()
+    pipes: List[Dict[str, Any]] = await tb_client.pipes(dependencies=True)
+
+    def check_filenames(filenames: List[str]):
+        parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
+        incl_suffix = ".incl"
+
+        for filename in filenames:
+            if os.path.isdir(filename):
+                process(filenames=get_project_filenames(filename))
+
+            file_suffix = Path(filename).suffix
+            if file_suffix == incl_suffix:
+                continue
+
+            parser = parser_matrix.get(file_suffix)
+            if not parser:
+                raise ParseException(FeedbackManager.error_unsupported_datafile(extension=file_suffix))
+
+            parser(filename)
+
+    async def process(filenames: List[str], watch: bool = False, only_pipes: bool = False):
+        check_filenames(filenames=filenames)
+        await folder_build(
+            tb_client,
+            workspaces,
+            datasources,
+            pipes,
+            filenames,
+            ignore_sql_errors=ignore_sql_errors,
+            is_internal=is_internal,
+            only_pipes=only_pipes,
+        )
+
+        for filename in filenames:
+            if filename.endswith(".datasource"):
+                ds_path = Path(filename)
+                ds_name = ds_path.stem
+                datasource_content = ds_path.read_text()
+                sample_data = await generate_sample_data_from_columns(tb_client, datasource_content)
+                ndjson_data = "\n".join([json.dumps(row) for row in sample_data])
+                await tb_client.datasource_events(ds_name, ndjson_data)
+
+        if watch:
+            filename = filenames[0]
+            if filename.endswith(".pipe"):
+                await build_and_print_pipe(tb_client, filename)
+
+    filenames = get_project_filenames(folder)
+
+    async def build_once(filenames: List[str]):
+        try:
+            click.echo("⚡ Building project...")
+            time_start = time.time()
+            await process(filenames=filenames, watch=False)
+            time_end = time.time()
+            elapsed_time = time_end - time_start
+            click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s\n"))
+        except Exception as e:
+            click.echo(FeedbackManager.error(message=str(e)))
+
+    await build_once(filenames)
+
+    if watch:
+        click.echo(FeedbackManager.highlight(message="◎ Watching for changes..."))
+        watch_files(filenames, process)
+
+
+async def build_and_print_pipe(tb_client: TinyB, filename: str):
+    pipe_name = os.path.basename(filename.split(".")[0])
+    res = await tb_client.query(f"SELECT * FROM {pipe_name} LIMIT 5 FORMAT JSON", pipeline=pipe_name)
+    data = []
+    for d in res["data"]:
+        data.append(d.values())
+    meta = res["meta"]
+    column_names = [col["name"] for col in meta]
+    echo_safe_humanfriendly_tables_format_smart_table(data, column_names=column_names)
+
+
+@cli.command()
+@coro
+async def stop() -> None:
+    """Stop Tinybird development environment"""
+    click.echo(FeedbackManager.info(message="Shutting down Tinybird development environment..."))
+    docker_client = get_docker_client()
+    stop_tinybird_local(docker_client)
+    click.echo(FeedbackManager.success(message="Tinybird development environment stopped"))
+
+
+@cli.command()
+@coro
+async def start() -> None:
+    """Start Tinybird development environment"""
+    click.echo(FeedbackManager.info(message="Starting Tinybird development environment..."))
+    docker_client = get_docker_client()
+    start_tinybird_local(docker_client)
+    click.echo(FeedbackManager.success(message="Tinybird development environment started"))

tinybird/tb_cli_modules/cli.py

@@ -10,12 +10,14 @@ import pprint
 import re
 import shutil
 import sys
+from datetime import datetime
 from os import environ, getcwd
 from pathlib import Path
 from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
 
 import click
 import humanfriendly
+import requests
 from click import Context
 from packaging import version
 
@@ -67,6 +69,7 @@ from tinybird.tb_cli_modules.common import (
     try_update_config_with_remote,
 )
 from tinybird.tb_cli_modules.config import CLIConfig
+from tinybird.tb_cli_modules.prompts import sample_data_prompt
 from tinybird.tb_cli_modules.telemetry import add_telemetry_event
 
 __old_click_echo = click.echo
@@ -1569,3 +1572,67 @@ async def deploy(
         raise
     except Exception as e:
         raise CLIException(str(e))
+
+
+@cli.command()
+@click.argument("datasource_file", type=click.Path(exists=True))
+@click.option("--count", type=int, default=10, help="Number of events to send")
+@click.option("--model", type=str, default=None, help="Model to use for data generation")
+@click.option("--print-data", is_flag=True, default=False, help="Print the data being sent")
+@click.pass_context
+def load_sample_data(ctx: Context, datasource_file: str, count: int, model: Optional[str], print_data: bool) -> None:
+    """Load sample data into a datasource.
+
+    Args:
+        ctx: Click context object
+        datasource_file: Path to the datasource file to load sample data into
+    """
+    import llm
+
+    try:
+        # TODO(eclbg): allow passing a datasource name instead of a file
+        datasource_path = Path(datasource_file)
+        if datasource_path.suffix != ".datasource":
+            raise CLIException(FeedbackManager.error_file_extension(filename=datasource_file))
+
+        datasource_name = datasource_path.stem
+
+        response = requests.get("http://localhost:80/tokens")
+        token = response.json()["workspace_admin_token"]
+
+        with open(datasource_file) as f:
+            content = f.read()
+            schema_start = next(i for i, line in enumerate(content.splitlines()) if line.strip().startswith("SCHEMA >"))
+            schema_end = next(
+                i
+                for i, line in enumerate(content.splitlines()[schema_start + 1 :], schema_start + 1)
+                if not line.strip()
+            )
+            schema = "\n".join(content.splitlines()[schema_start:schema_end])
+        llm_model = llm.get_model(model)
+        click.echo(f"Using model: {model}")
+        prompt = sample_data_prompt.format(current_datetime=datetime.now().isoformat(), row_count=count)
+        # prompt = sample_data_with_errors_prompt.format(current_datetime=datetime.now().isoformat())  # This prompt will generate data with errors
+        full_prompt = prompt + "\n\n" + schema
+        sent_events = 0
+        while sent_events < count:
+            click.echo(f"Generating data for '{datasource_name}'")
+            data = llm_model.prompt(full_prompt)
+
+            click.echo(f"Sending data to '{datasource_name}'")
+            headers = {"Authorization": f"Bearer {token}"}
+            if print_data:
+                click.echo(f"Data: {data}")
+            response = requests.post(
+                f"http://localhost:80/v0/events?name={datasource_name}",
+                data=data,
+                headers=headers,
+            )
+            if response.status_code not in (200, 202):
+                raise CLIException(f"Failed to send data: {response.text}")
+            click.echo(f"Response: {response.text}")
+            sent_events += 10
+            click.echo(f"Sent 10 events to datasource '{datasource_name}'")
+
+    except Exception as e:
+        raise CLIException(FeedbackManager.error_exception(error=str(e)))

tinybird/tb_cli_modules/common.py

@@ -153,7 +153,7 @@ def generate_datafile(
     if not f.exists() or force:
         with open(f"{f}", "w") as ds_file:
             ds_file.write(datafile)
-        click.echo(FeedbackManager.success_generated_file(file=f, stem=datasource_name, filename=filename))
+        click.echo(FeedbackManager.success(message=f"** Generated {f}"))
 
         if data and (base / "fixtures").exists():
             # Generating a fixture for Parquet files is not so trivial, since Parquet format
@@ -166,7 +166,6 @@ def generate_datafile(
                 newline = b"\n"  # TODO: guess
                 with open(f, "wb") as fixture_file:
                     fixture_file.write(data[: data.rfind(newline)])
-                click.echo(FeedbackManager.success_generated_fixture(fixture=f))
     else:
         click.echo(FeedbackManager.error_file_already_exists(file=f))
 
@@ -1043,6 +1042,7 @@ def get_format_from_filename_or_url(filename_or_url: str) -> str:
 
 async def push_data(
     ctx: Context,
+    client: TinyB,
     datasource_name: str,
     url,
     connector: Optional[str],
@@ -1055,7 +1055,6 @@ async def push_data(
 ):
     if url and type(url) is tuple:
         url = url[0]
-    client: TinyB = ctx.obj["client"]
 
     if connector and sql:
         load_connector_config(ctx, connector, False, check_uninstalled=False)

tinybird/tb_cli_modules/create.py

@@ -0,0 +1,226 @@
+import json
+import os
+from os import getcwd
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import click
+from click import Context
+from openai import OpenAI
+
+from tinybird.client import TinyB
+from tinybird.datafile import folder_build
+from tinybird.feedback_manager import FeedbackManager
+from tinybird.tb_cli_modules.cli import cli
+from tinybird.tb_cli_modules.common import _generate_datafile, coro, generate_datafile, push_data
+from tinybird.tb_cli_modules.config import CLIConfig
+from tinybird.tb_cli_modules.exceptions import CLIDatasourceException
+from tinybird.tb_cli_modules.llm import LLM
+from tinybird.tb_cli_modules.local import get_docker_client, set_up_tinybird_local
+from tinybird.tb_cli_modules.prompts import sample_data_sql_prompt
+
+
+@cli.command()
+@click.option(
+    "--data",
+    type=click.Path(exists=True),
+    default=None,
+    help="Initial data to be used to create the project",
+)
+@click.option(
+    "--prompt",
+    type=str,
+    default=None,
+    help="Prompt to be used to create the project",
+)
+@click.option(
+    "--folder",
+    default=None,
+    type=click.Path(exists=True, file_okay=False),
+    help="Folder where datafiles will be placed",
+)
+@click.pass_context
+@coro
+async def create(
+    ctx: Context,
+    data: Optional[str],
+    prompt: Optional[str],
+    folder: Optional[str],
+) -> None:
+    """Initialize a new project."""
+    click.echo(FeedbackManager.highlight(message="Setting up Tinybird development environment..."))
+    folder = folder or getcwd()
+    try:
+        docker_client = get_docker_client()
+        tb_client = set_up_tinybird_local(docker_client)
+        await project_create(tb_client, data, prompt, folder)
+        workspaces: List[Dict[str, Any]] = (await tb_client.user_workspaces()).get("workspaces", [])
+        datasources = await tb_client.datasources()
+        pipes = await tb_client.pipes(dependencies=True)
+        await folder_build(
+            tb_client,
+            workspaces,
+            datasources,
+            pipes,
+        )
+        if data:
+            ds_name = os.path.basename(data.split(".")[0])
+            await append_datasource(ctx, tb_client, ds_name, data, None, None, False, 1)
+        elif prompt:
+            datasource_files = [f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")]
+            for datasource_file in datasource_files:
+                datasource_content = Path(folder) / "datasources" / datasource_file
+                sample_data = await generate_sample_data_from_columns(tb_client, datasource_content)
+                ndjson_data = "\n".join([json.dumps(row) for row in sample_data])
+                await tb_client.datasource_events(datasource_file, ndjson_data)
+        click.echo(FeedbackManager.success(message="\n✔ Tinybird development environment is ready"))
+    except Exception as e:
+        click.echo(FeedbackManager.error(message=f"Error: {str(e)}"))
+
+
+async def project_create(
+    client: TinyB,
+    data: Optional[str],
+    prompt: Optional[str],
+    folder: str,
+):
+    project_paths = ["datasources", "endpoints", "copies", "sinks", "playgrounds", "materializations"]
+    force = True
+    for x in project_paths:
+        try:
+            f = Path(folder) / x
+            f.mkdir()
+            click.echo(FeedbackManager.info_path_created(path=x))
+        except FileExistsError:
+            pass
+
+    def generate_pipe_file(name: str, content: str):
+        base = Path("endpoints")
+        if not base.exists():
+            base = Path()
+        f = base / (f"{name}.pipe")
+        with open(f"{f}", "w") as file:
+            file.write(content)
+        click.echo(FeedbackManager.success(message=f"** Generated {f}"))
+
+    if data:
+        path = Path(folder) / data
+        format = path.suffix.lstrip(".")
+        await _generate_datafile(str(path), client, format=format, force=force)
+        name = data.split(".")[0]
+        generate_pipe_file(
+            f"{name}_endpoint",
+            f"""
+NODE endpoint
+SQL >
+    SELECT * from {name}
+TYPE ENDPOINT
+            """,
+        )
+    elif prompt:
+        try:
+            config = CLIConfig.get_project_config()
+            model = config.get("llms", {}).get("openai", {}).get("model", "gpt-4o-mini")
+            api_key = config.get("llms", {}).get("openai", {}).get("api_key", None)
+            llm = LLM(model=model, key=api_key)
+            result = await llm.create_project(prompt)
+            for ds in result.datasources:
+                content = ds.content.replace("```", "")
+                generate_datafile(content, filename=f"{ds.name}.datasource", data=None, _format="ndjson", force=force)
+
+            for pipe in result.pipes:
+                content = pipe.content.replace("```", "")
+                generate_pipe_file(pipe.name, content)
+        except Exception as e:
+            click.echo(FeedbackManager.error(message=f"Error: {str(e)}"))
+    else:
+        events_ds = """
+SCHEMA >
+    `age` Int16 `json:$.age`,
+    `airline` String `json:$.airline`,
+    `email` String `json:$.email`,
+    `extra_bags` Int16 `json:$.extra_bags`,
+    `flight_from` String `json:$.flight_from`,
+    `flight_to` String `json:$.flight_to`,
+    `meal_choice` String `json:$.meal_choice`,
+    `name` String `json:$.name`,
+    `passport_number` Int32 `json:$.passport_number`,
+    `priority_boarding` UInt8 `json:$.priority_boarding`,
+    `timestamp` DateTime `json:$.timestamp`,
+    `transaction_id` String `json:$.transaction_id`
+
+ENGINE "MergeTree"
+ENGINE_PARTITION_KEY "toYear(timestamp)"
+ENGINE_SORTING_KEY "airline, timestamp"
+"""
+        top_airlines = """
+NODE endpoint
+SQL >
+    SELECT airline, count() as bookings FROM events
+    GROUP BY airline
+    ORDER BY bookings DESC
+    LIMIT 5
+TYPE ENDPOINT
+"""
+        generate_datafile(events_ds, filename="events.datasource", data=None, _format="ndjson", force=force)
+        generate_pipe_file("top_airlines", top_airlines)
+
+
+async def append_datasource(
+    ctx: Context,
+    tb_client: TinyB,
+    datasource_name: str,
+    url: str,
+    sql: Optional[str],
+    incremental: Optional[str],
+    ignore_empty: bool,
+    concurrency: int,
+):
+    if incremental:
+        date = None
+        source_column = incremental.split(":")[0]
+        dest_column = incremental.split(":")[-1]
+        result = await tb_client.query(f"SELECT max({dest_column}) as inc from {datasource_name} FORMAT JSON")
+        try:
+            date = result["data"][0]["inc"]
+        except Exception as e:
+            raise CLIDatasourceException(f"{str(e)}")
+        if date:
+            sql = f"{sql} WHERE {source_column} > '{date}'"
+    await push_data(
+        ctx,
+        tb_client,
+        datasource_name,
+        url,
+        None,
+        sql,
+        mode="append",
+        ignore_empty=ignore_empty,
+        concurrency=concurrency,
+    )
+
+
+def generate_sql_sample_data(datasource_content: str, row_count: int, model: str, api_key: str) -> str:
+    client = OpenAI(api_key=api_key)
+
+    response = client.chat.completions.create(
+        model=model,
+        messages=[
+            {"role": "system", "content": sample_data_sql_prompt.format(row_count=row_count)},
+            {"role": "user", "content": datasource_content},
+        ],
+    )
+
+    return response.choices[0].message.content or ""
+
+
+async def generate_sample_data_from_columns(
+    tb_client: TinyB, datasource_content: str, row_count: int = 20
+) -> List[Dict[str, Any]]:
+    config = CLIConfig.get_project_config()
+    model = config.get("llms", {}).get("openai", {}).get("model", "gpt-4o-mini")
+    api_key = config.get("llms", {}).get("openai", {}).get("api_key", None)
+    sql = generate_sql_sample_data(datasource_content, row_count, model, api_key)
+    result = await tb_client.query(f"{sql} FORMAT JSON")
+    data = result.get("data", [])
+    return data

tinybird/tb_cli_modules/datasource.py

@@ -174,11 +174,11 @@ async def datasource_append(
     if incremental and not connector:
         raise CLIDatasourceException(FeedbackManager.error_incremental_not_supported())
 
+    client: TinyB = ctx.obj["client"]
     if incremental:
         date = None
         source_column = incremental.split(":")[0]
         dest_column = incremental.split(":")[-1]
-        client: TinyB = ctx.obj["client"]
         result = await client.query(f"SELECT max({dest_column}) as inc from {datasource_name} FORMAT JSON")
         try:
             date = result["data"][0]["inc"]
@@ -187,7 +187,15 @@ async def datasource_append(
         if date:
             sql = f"{sql} WHERE {source_column} > '{date}'"
     await push_data(
-        ctx, datasource_name, url, connector, sql, mode="append", ignore_empty=ignore_empty, concurrency=concurrency
+        ctx,
+        client,
+        datasource_name,
+        url,
+        connector,
+        sql,
+        mode="append",
+        ignore_empty=ignore_empty,
+        concurrency=concurrency,
     )