tinybird 0.0.1.dev151__py3-none-any.whl → 0.0.1.dev153__py3-none-any.whl

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev151'
-__revision__ = 'd9da445'
+__version__ = '0.0.1.dev153'
+__revision__ = '24d7d4a'

tinybird/tb/modules/build.py

@@ -23,6 +23,7 @@ from tinybird.tb.modules.datafile.fixture import FixtureExtension, get_fixture_d
 from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
 from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 from tinybird.tb.modules.datafile.playground import folder_playground
+from tinybird.tb.modules.dev_server import BuildStatus, start_server
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.project import Project
 from tinybird.tb.modules.shell import Shell, print_table_formatted
@@ -50,16 +51,28 @@ def build(ctx: click.Context, watch: bool) -> None:
 
 @cli.command("dev", help="Build the project server side and watch for changes.")
 @click.option("--data-origin", type=str, default="", help="Data origin: local or cloud")
+@click.option("--ui", is_flag=True, default=False, help="Connect your local project to Tinybird UI")
 @click.pass_context
-def dev(ctx: click.Context, data_origin: str) -> None:
+def dev(ctx: click.Context, data_origin: str, ui: bool) -> None:
     if data_origin == "cloud":
         return dev_cloud(ctx)
-
     project: Project = ctx.ensure_object(dict)["project"]
     tb_client: TinyB = ctx.ensure_object(dict)["client"]
+    build_status = BuildStatus()
+    if ui:
+        server_thread = threading.Thread(
+            target=start_server, args=(project, tb_client, process, build_status), daemon=True
+        )
+        server_thread.start()
+        # Wait for the server to start
+        time.sleep(0.5)
     click.echo(FeedbackManager.highlight_building_project())
-    process(project=project, tb_client=tb_client, watch=True)
-    run_watch(project=project, tb_client=tb_client, process=partial(process, project=project, tb_client=tb_client))
+    process(project=project, tb_client=tb_client, watch=True, build_status=build_status)
+    run_watch(
+        project=project,
+        tb_client=tb_client,
+        process=partial(process, project=project, tb_client=tb_client, build_status=build_status),
+    )
 
 
 def build_project(project: Project, tb_client: TinyB, file_changed: Optional[str] = None, silent: bool = False) -> None:
@@ -266,10 +279,17 @@ def process(
     file_changed: Optional[str] = None,
     diff: Optional[str] = None,
     silent: bool = False,
-) -> None:
+    error: bool = False,
+    build_status: Optional[BuildStatus] = None,
+) -> Optional[str]:
     time_start = time.time()
     build_failed = False
     build_error: Optional[str] = None
+    if build_status:
+        if build_status.building:
+            return build_status.error
+        else:
+            build_status.building = True
     if file_changed and (file_changed.endswith(FixtureExtension.NDJSON) or file_changed.endswith(FixtureExtension.CSV)):
         rebuild_fixture(project, tb_client, file_changed)
     elif file_changed and file_changed.endswith(".sql"):
@@ -277,6 +297,9 @@ def process(
     else:
         try:
             build_project(project, tb_client, file_changed, silent)
+            if build_status:
+                build_status.building = False
+                build_status.error = None
         except click.ClickException as e:
             click.echo(FeedbackManager.info(message=str(e)))
             build_error = str(e)
@@ -284,7 +307,8 @@ def process(
         try:
             if file_changed and not build_failed:
                 asyncio.run(folder_build(project, tb_client, filenames=[file_changed]))
-                show_data(tb_client, file_changed, diff)
+                if not build_status:
+                    show_data(tb_client, file_changed, diff)
         except Exception:
             pass
 
@@ -296,14 +320,19 @@ def process(
         click.echo(FeedbackManager.error(message=f"✗ {rebuild_str} failed"))
         if not watch:
             sys_exit("build_error", build_error or "Unknown error")
+        build_error = build_error or "Unknown error"
+        if build_status:
+            build_status.error = build_error
+            build_status.building = False
+        return build_error
     else:
         if not silent:
             click.echo(FeedbackManager.success(message=f"\n✓ {rebuild_str} completed in {elapsed_time:.1f}s"))
 
+    return None
 
-def run_watch(
-    project: Project, tb_client: TinyB, process: Callable[[bool, Optional[str], Optional[str]], None]
-) -> None:
+
+def run_watch(project: Project, tb_client: TinyB, process: Callable) -> None:
     shell = Shell(project=project, tb_client=tb_client, playground=False)
     click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
     watcher_thread = threading.Thread(

tinybird/tb/modules/cli.py

@@ -141,7 +141,7 @@ async def cli(
     config = await get_config(host, token, user_token=user_token, config_file=config_temp._path)
     client = _get_tb_client(config.get("token", None), config["host"])
     folder = os.path.join(config_temp._path.replace(".tinyb", ""), config.get("cwd", os.getcwd()))
-    project = Project(folder=folder)
+    project = Project(folder=folder, workspace_name=config.get("name", ""))
     config["path"] = str(project.path)
     # If they have passed a token or host as parameter and it's different that record in .tinyb, refresh the workspace id
     if token or host:

tinybird/tb/modules/create.py

@@ -1,3 +1,4 @@
+import glob
 import os
 import re
 from pathlib import Path
@@ -5,7 +6,7 @@ from typing import Any, Dict, Optional, Tuple
 
 import click
 
-from tinybird.prompts import create_prompt, mock_prompt, readme_prompt, rules_prompt
+from tinybird.prompts import create_prompt, readme_prompt, rules_prompt
 from tinybird.tb.client import TinyB
 from tinybird.tb.modules.cicd import init_cicd
 from tinybird.tb.modules.cli import cli
@@ -17,6 +18,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.llm import LLM
 from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
 from tinybird.tb.modules.local_common import get_tinybird_local_client
+from tinybird.tb.modules.mock import create_mock_data
 from tinybird.tb.modules.project import Project
 
 
@@ -83,7 +85,7 @@ async def create(
 
         if data or (prompt and user_token):
             click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
-            result, created_something = await create_resources(tb_client, user_token, data, prompt, folder, ctx_config)
+            result, created_something = await create_resources(tb_client, user_token, data, prompt, project, ctx_config)
             click.echo(FeedbackManager.success(message="✓ Done!\n"))
             if prompt:
                 readme_path = Path(root_folder) / "README.md"
@@ -132,22 +134,26 @@ async def create(
                 persist_fixture(ds_name, data_content, folder, format=data_format)
                 created_something = True
             elif prompt and user_token:
-                datasource_files = [f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")]
-                for datasource_file in datasource_files:
-                    datasource_path = Path(folder) / "datasources" / datasource_file
-                    llm = LLM(user_token=user_token, host=tb_client.host)
+                for datasource_file in project.get_datasource_files():
+                    datasource_path = Path(datasource_file)
                     datasource_name = datasource_path.stem
                     datasource_content = datasource_path.read_text()
                     has_json_path = "`json:" in datasource_content
                     if has_json_path:
-                        prompt = f"<datasource_schema>{datasource_content}</datasource_schema>"
-                        response = llm.ask(system_prompt=mock_prompt(rows), prompt=prompt)
-                        sql = extract_xml(response, "sql")
-                        sql = sql.split("FORMAT")[0]
-                        query_result = await tb_client.query(f"{sql} FORMAT JSON")
-                        data = query_result.get("data", [])
-                        if data:
-                            persist_fixture(datasource_name, data, folder, format="ndjson")
+                        mock_data = await create_mock_data(
+                            datasource_name,
+                            datasource_content,
+                            rows,
+                            prompt,
+                            config,
+                            ctx_config,
+                            user_token,
+                            tb_client,
+                            format_="ndjson",
+                            folder=project.folder,
+                        )
+                        if mock_data:
+                            persist_fixture(datasource_name, mock_data, folder, format="ndjson")
                             click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}"))
                             created_something = True
 
@@ -204,12 +210,12 @@ async def create_resources(
     user_token: Optional[str],
     data: Optional[str],
     prompt: Optional[str],
-    folder: str,
+    project: Project,
     config: Dict[str, Any],
 ) -> Tuple[str, bool]:
     result = ""
     created_any_resource = False
-    folder_path = Path(folder)
+    folder_path = project.path
     if data:
         local_client = await get_tinybird_local_client(config)
         path = folder_path / data
@@ -224,7 +230,7 @@ SQL >
     SELECT * from {name}
 TYPE ENDPOINT
             """,
-            folder,
+            project.folder,
         )
         result = (
             f"<response><resource><type>datasource</type><name>{name}</name><content></content></resource></response>"
@@ -232,19 +238,9 @@ TYPE ENDPOINT
         created_any_resource = True
 
     elif prompt and user_token:
-        datasource_paths = [
-            Path(folder) / "datasources" / f
-            for f in os.listdir(Path(folder) / "datasources")
-            if f.endswith(".datasource")
-        ]
-        pipes_paths = [
-            Path(folder) / "endpoints" / f for f in os.listdir(Path(folder) / "endpoints") if f.endswith(".pipe")
-        ]
-        connections_paths = [
-            Path(folder) / "connections" / f
-            for f in os.listdir(Path(folder) / "connections")
-            if f.endswith(".connection")
-        ]
+        datasource_paths = [Path(ds_file) for ds_file in project.get_datasource_files()]
+        pipes_paths = [Path(pipe_file) for pipe_file in project.get_pipe_files()]
+        connections_paths = [Path(conn_file) for conn_file in project.get_connection_files()]
         resources_xml = "\n".join(
             [
                 f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
@@ -300,19 +296,19 @@ TYPE ENDPOINT
                 data=None,
                 _format="ndjson",
                 force=True,
-                folder=folder,
+                folder=project.folder,
             )
             created_any_resource = True
 
         for pipe in pipes:
             content = pipe["content"].replace("```", "")
-            generate_pipe_file(pipe["name"], content, folder)
+            generate_pipe_file(pipe["name"], content, project.folder)
             created_any_resource = True
 
         for conn in connections:
             content = conn["content"].replace("```", "")
             filename = f"{conn['name']}.connection"
-            generate_connection_file(conn["name"], content, folder)
+            generate_connection_file(conn["name"], content, project.folder)
             created_any_resource = True
 
     return result, created_any_resource
@@ -348,21 +344,25 @@ def generate_pipe_file(name: str, content: str, folder: str) -> Path:
     def is_endpoint(content: str) -> bool:
         return re.search(r"TYPE endpoint", content, re.IGNORECASE) is not None
 
-    if is_copy(content):
-        pathname = "copies"
-    elif is_materialization(content):
-        pathname = "materializations"
-    elif is_sink(content):
-        pathname = "sinks"
-    elif is_endpoint(content):
-        pathname = "endpoints"
+    already_exists = glob.glob(f"{folder}/**/{name}.pipe")
+    if already_exists:
+        f = Path(already_exists[0])
     else:
-        pathname = "pipes"
+        if is_copy(content):
+            pathname = "copies"
+        elif is_materialization(content):
+            pathname = "materializations"
+        elif is_sink(content):
+            pathname = "sinks"
+        elif is_endpoint(content):
+            pathname = "endpoints"
+        else:
+            pathname = "pipes"
 
-    base = Path(folder) / pathname
-    if not base.exists():
-        base.mkdir()
-    f = base / (f"{name}.pipe")
+        base = Path(folder) / pathname
+        if not base.exists():
+            base.mkdir()
+        f = base / (f"{name}.pipe")
     with open(f"{f}", "w") as file:
         file.write(content)
     click.echo(FeedbackManager.info_file_created(file=f.relative_to(folder)))
@@ -370,10 +370,14 @@ def generate_pipe_file(name: str, content: str, folder: str) -> Path:
 
 
 def generate_connection_file(name: str, content: str, folder: str, skip_feedback: bool = False) -> Path:
-    base = Path(folder) / "connections"
-    if not base.exists():
-        base.mkdir()
-    f = base / (f"{name}.connection")
+    already_exists = glob.glob(f"{folder}/**/{name}.connection")
+    if already_exists:
+        f = Path(already_exists[0])
+    else:
+        base = Path(folder) / "connections"
+        if not base.exists():
+            base.mkdir()
+        f = base / (f"{name}.connection")
     with open(f"{f}", "w") as file:
         file.write(content)
     if not skip_feedback:

tinybird/tb/modules/datafile/common.py

@@ -846,6 +846,14 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
                     lineno=line,
                     pos=pos,
                 )
+            if detected_type in ("Int", "UInt"):
+                t = detected_type
+                raise SchemaSyntaxError(
+                    message=f"Precision is mandatory for {t} types",
+                    hint=f"Hint: use one of {t}8, {t}16, {t}32, {t}64, {t}128, {t}256",
+                    lineno=line,
+                    pos=pos - len(t),
+                )
 
             try:
                 # Imported in the body to be compatible with the CLI

tinybird/tb/modules/deployment.py

@@ -68,7 +68,6 @@ def download_github_template(url: str) -> Optional[Path]:
     """
     # Parse GitHub URL components
     # From: https://github.com/owner/repo/tree/branch/path
-    # To: api.github.com/repos/owner/repo/contents/path?ref=branch
     parts = url.replace("https://github.com/", "").split("/")
     if len(parts) < 5 or "tree" not in parts:
         click.echo(
@@ -83,16 +82,50 @@ def download_github_template(url: str) -> Optional[Path]:
     branch = parts[parts.index("tree") + 1]
     path = "/".join(parts[parts.index("tree") + 2 :])
 
-    api_url = f"https://api.github.com/repos/{owner}/{repo}/contents/{path}?ref={branch}"
-
-    # Create temporary directory for template
+    try:
+        import shutil
+        import subprocess
+        import tempfile
+
+        # Create a temporary directory for cloning
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Clone the specific branch with minimum depth
+            repo_url = f"https://github.com/{owner}/{repo}.git"
+            subprocess.run(
+                ["git", "clone", "--depth", "1", "--branch", branch, repo_url, temp_dir],
+                check=True,
+                capture_output=True,
+            )
 
-    dir = Path(".")
+            # Copy the specific path to current directory
+            source_path = Path(temp_dir) / path
+            if not source_path.exists():
+                click.echo(FeedbackManager.error(message=f"Path {path} not found in repository"))
+                return None
+
+            dir = Path(".")
+            if source_path.is_dir():
+                # Copy directory contents
+                for item in source_path.iterdir():
+                    dest = dir / item.name
+                    if item.is_dir():
+                        shutil.copytree(item, dest)
+                    else:
+                        shutil.copy2(item, dest)
+                    click.echo(FeedbackManager.info(message=f"Downloaded {item.name}"))
+            else:
+                # Copy single file
+                shutil.copy2(source_path, dir / source_path.name)
+                click.echo(FeedbackManager.info(message=f"Downloaded {source_path.name}"))
 
-    # Recursively download all files
-    download_github_contents(api_url, dir)
+            return dir
 
-    return dir
+    except subprocess.CalledProcessError as e:
+        click.echo(FeedbackManager.error(message=f"Git clone failed: {e.stderr.decode()}"))
+        return None
+    except Exception as e:
+        click.echo(FeedbackManager.error(message=f"Error downloading template: {str(e)}"))
+        return None
 
 
 # TODO(eclbg): This should eventually end up in client.py, but we're not using it here yet.

tinybird/tb/modules/dev_server.py

@@ -0,0 +1,257 @@
+import glob
+import http.server
+import json
+import time
+from pathlib import Path
+from typing import Callable, Optional
+
+import click
+
+from tinybird.tb.client import TinyB
+from tinybird.tb.modules.common import sys_exit
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.local_common import TB_LOCAL_PORT
+from tinybird.tb.modules.project import Project
+
+
+class BuildStatus:
+    def __init__(self):
+        self.last_build_time = 0
+        self.building = False
+        self.error: Optional[str] = None
+        self.result: Optional[str] = None
+
+
+class DevServer(http.server.HTTPServer):
+    project: Project
+    tb_client: TinyB
+    process: Callable
+
+    def __init__(
+        self,
+        process: Callable,
+        project: Project,
+        build_status: BuildStatus,
+        tb_client: TinyB,
+    ):
+        port = 49161
+        self.project = project
+        self.tb_client = tb_client
+        self.process = process
+        self.build_status = build_status
+        self.port = port
+
+        try:
+            super().__init__(("", port), DevHandler)
+            click.echo(FeedbackManager.success(message=f"✓ Dev server running on http://localhost:{port}"))
+            click.echo(
+                FeedbackManager.info(
+                    message=f"* Access your project at https://cloud.tinybird.co/local/{TB_LOCAL_PORT}/{project.workspace_name}/project"
+                )
+            )
+        except OSError as e:
+            if e.errno == 48:  # Address already in use
+                click.echo(FeedbackManager.error(message=f"Port {port} is already in use. Try a different port."))
+                sys_exit("port_in_use", f"Port {port} is already in use")
+            else:
+                click.echo(FeedbackManager.error_exception(error=e))
+                sys_exit("server_error", str(e))
+
+
+class DevHandler(http.server.SimpleHTTPRequestHandler):
+    server: DevServer
+
+    def send_cors_headers(self):
+        self.send_header("Access-Control-Allow-Origin", "*")
+        self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
+        self.send_header("Access-Control-Allow-Headers", "Content-Type, Authorization")
+
+    def do_OPTIONS(self):
+        self.send_response(200)
+        self.send_cors_headers()
+        self.end_headers()
+
+    def do_GET(self):
+        try:
+            project: Project = self.server.project
+
+            if self.path == "/":
+                self.send_response(200)
+                self.send_header("Content-Type", "text/html")
+                self.send_cors_headers()
+                self.end_headers()
+
+                html = f"""
+                <!DOCTYPE html>
+                <html>
+                <head>
+                    <title>Tinybird Dev Server</title>
+                    <style>
+                        body {{ font-family: Arial, sans-serif; margin: 40px; line-height: 1.6; }}
+                        h1 {{ color: #333; }}
+                        ul {{ list-style-type: none; padding: 0; }}
+                        li {{ margin-bottom: 10px; }}
+                        a {{ color: #0066cc; text-decoration: none; }}
+                        a:hover {{ text-decoration: underline; }}
+                    </style>
+                </head>
+                <body>
+                    <h1>Tinybird Dev Server</h1>
+                    <p>Server running for project: {project.path}</p>
+                    <p>Access your local environment at <a href="https://cloud.tinybird.co/local/{TB_LOCAL_PORT}/{project.workspace_name}/project">https://cloud.tinybird.co/local/{TB_LOCAL_PORT}/{project.workspace_name}/project</a> and edit your project.</p>
+                    <h2>API Endpoints:</h2>
+                    <ul>
+                        <li><a href="/files">/files</a> - List all project files</li>
+                    </ul>
+                </body>
+                </html>
+                """
+                self.wfile.write(html.encode())
+                return
+
+            elif "/files/" in self.path:
+                file_path = self.path.split("/files/")[1].split("?")[0]
+                project_files = project.get_project_files()
+                project_file = next((Path(f) for f in project_files if f.endswith(file_path)), None)
+                if not project_file:
+                    self.send_response(404)
+                    self.send_header("Content-Type", "application/json")
+                    self.send_cors_headers()
+                    self.end_headers()
+                    self.wfile.write(json.dumps({"status": "error", "error": "Resource not found"}).encode())
+                    return
+
+                self.send_response(200)
+                self.send_header("Content-Type", "application/json")
+                self.send_cors_headers()
+                self.end_headers()
+                self.wfile.write(json.dumps({"path": str(project_file), "content": project_file.read_text()}).encode())
+                return
+
+            elif "/files" in self.path:
+                project_files = project.get_project_files()
+                self.send_response(200)
+                self.send_header("Content-Type", "application/json")
+                self.send_cors_headers()
+                self.end_headers()
+                self.wfile.write(
+                    json.dumps(
+                        {
+                            "files": [f.replace(f"{project.folder}/", "") for f in project_files],
+                            "root": project.folder,
+                            "workspace_name": project.workspace_name,
+                        }
+                    ).encode()
+                )
+                return
+
+            # Handle all other paths
+            self.send_response(404)
+            self.send_header("Content-Type", "application/json")
+            self.send_cors_headers()
+            self.end_headers()
+            self.wfile.write(json.dumps({"status": "error", "message": "Not found"}).encode())
+
+        except Exception as e:
+            self.send_response(400)
+            self.send_header("Content-Type", "application/json")
+            self.send_cors_headers()
+            self.end_headers()
+            self.wfile.write(json.dumps({"status": "error", "message": str(e)}).encode())
+
+    def do_POST(self):
+        try:
+            # Parse the request body
+            content_length = int(self.headers["Content-Length"])
+            body = self.rfile.read(content_length)
+            data = json.loads(body)
+
+            # Get the file path from the request body
+            file_path = data.get("path")
+            if not file_path:
+                self.send_response(400)
+                self.send_header("Content-Type", "application/json")
+                self.send_cors_headers()
+                self.end_headers()
+                self.wfile.write(json.dumps({"status": "error", "message": "No file path provided"}).encode())
+                return
+
+            exists = glob.glob(f"{self.server.project.folder}/**/{file_path}", recursive=True)
+            if not exists:
+                self.send_response(400)
+                self.send_header("Content-Type", "application/json")
+                self.send_cors_headers()
+                self.end_headers()
+                self.wfile.write(json.dumps({"status": "error", "message": "File does not exist"}).encode())
+                return
+
+            # Get the content from the request body
+            content = data.get("content")
+            if not content:
+                self.send_response(400)
+                self.send_header("Content-Type", "application/json")
+                self.send_cors_headers()
+                self.end_headers()
+                self.wfile.write(json.dumps({"status": "error", "message": "No content provided"}).encode())
+                return
+
+            # Write the content to the file
+            file_path = exists[0]
+            with open(file_path, "w") as f:
+                f.write(content)
+            # sleep for 0.2 seconds to ensure the file is written
+            time.sleep(0.2)
+
+            def check_build_status(attempts: int = 0):
+                if attempts > 10:
+                    return "Build timeout. Check the console for more details."
+                if self.server.build_status.building:
+                    time.sleep(0.5)
+                    return check_build_status(attempts + 1)
+                else:
+                    return self.server.build_status.error
+
+            build_error = check_build_status()
+            if build_error:
+                raise Exception(build_error)
+
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json")
+            self.send_cors_headers()
+            self.end_headers()
+            self.wfile.write(json.dumps({"status": "success", "message": "File written successfully"}).encode())
+
+        except Exception as e:
+            self.send_response(400)
+            self.send_header("Content-Type", "application/json")
+            self.send_cors_headers()
+            self.end_headers()
+            self.wfile.write(json.dumps({"status": "error", "error": str(e)}).encode())
+
+    def log_message(self, format, *args):
+        pass
+
+
+def start_server(project: Project, tb_client: TinyB, process: Callable, build_status: BuildStatus):
+    """Start a development server for the project.
+
+    Args:
+        project: The project instance to serve.
+    """
+
+    try:
+        click.echo(FeedbackManager.highlight(message="\n» Starting Tinybird dev server...\n"))
+
+        # Create and start the server
+        server = DevServer(process, project, build_status, tb_client)
+        server.serve_forever()
+
+        # Run the server in the main thread
+        click.echo(FeedbackManager.gray(message="\nWatching for changes...\n"))
+        click.echo(FeedbackManager.highlight(message="Press Ctrl+C to stop the server\n"))
+
+    except KeyboardInterrupt:
+        click.echo(FeedbackManager.highlight(message="\n» Stopping Tinybird dev server...\n"))
+    except Exception as e:
+        click.echo(FeedbackManager.error_exception(error=e))
+        sys_exit("server_error", str(e))

tinybird/tb/modules/mock.py

@@ -1,5 +1,6 @@
 import glob
 from pathlib import Path
+from typing import Any, Dict, List
 
 import click
 
@@ -65,41 +66,23 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, form
 
         datasource_content = datasource_path.read_text()
         config = CLIConfig.get_project_config()
-        user_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
         user_token = ctx_config.get("user_token")
 
         if not user_token:
             raise Exception("This action requires authentication. Run 'tb login' first.")
 
-        llm = LLM(user_token=user_token, host=user_client.host)
-        prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
-        sql = ""
-        attempts = 0
-        data = []
-        error = ""
-        sql_path = None
-        while True:
-            try:
-                response = llm.ask(system_prompt=mock_prompt(rows, error), prompt=prompt)
-                sql = extract_xml(response, "sql")
-                sql_path = persist_fixture_sql(datasource_name, sql, folder)
-                sql_format = "JSON" if format_ == "ndjson" else "CSV"
-                result = await tb_client.query(f"SELECT * FROM ({sql}) LIMIT {rows} FORMAT {sql_format}")
-                if sql_format == "JSON":
-                    data = result.get("data", [])[:rows]
-                    error_response = result.get("error", None)
-                    if error_response:
-                        raise Exception(error_response)
-                else:
-                    data = result
-                break
-            except Exception as e:
-                error = str(e)
-                attempts += 1
-                if attempts > 5:
-                    raise Exception(f"Failed to generate a valid solution. Check {str(sql_path)} and try again.")
-                else:
-                    continue
+        data = await create_mock_data(
+            datasource_name,
+            datasource_content,
+            rows,
+            prompt,
+            config,
+            ctx_config,
+            user_token,
+            tb_client,
+            format_,
+            folder,
+        )
 
         fixture_path = persist_fixture(datasource_name, data, folder, format=format_)
         click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}.{format_} created"))
@@ -125,3 +108,48 @@ async def append_fixture(
         concurrency=1,
         silent=True,
     )
+
+
+async def create_mock_data(
+    datasource_name: str,
+    datasource_content: str,
+    rows: int,
+    prompt: str,
+    config: CLIConfig,
+    ctx_config: Dict[str, Any],
+    user_token: str,
+    tb_client: TinyB,
+    format_: str,
+    folder: str,
+) -> List[Dict[str, Any]]:
+    user_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
+    llm = LLM(user_token=user_token, host=user_client.host)
+    prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
+    sql = ""
+    attempts = 0
+    data = []
+    error = ""
+    sql_path = None
+    while True:
+        try:
+            response = llm.ask(system_prompt=mock_prompt(rows, error), prompt=prompt)
+            sql = extract_xml(response, "sql")
+            sql_path = persist_fixture_sql(datasource_name, sql, folder)
+            sql_format = "JSON" if format_ == "ndjson" else "CSV"
+            result = await tb_client.query(f"SELECT * FROM ({sql}) LIMIT {rows} FORMAT {sql_format}")
+            if sql_format == "JSON":
+                data = result.get("data", [])[:rows]
+                error_response = result.get("error", None)
+                if error_response:
+                    raise Exception(error_response)
+            else:
+                data = result
+            break
+        except Exception as e:
+            error = str(e)
+            attempts += 1
+            if attempts > 5:
+                raise Exception(f"Failed to generate a valid solution. Check {str(sql_path)} and try again.")
+            else:
+                continue
+    return data

tinybird/tb/modules/project.py

@@ -11,8 +11,9 @@ from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 class Project:
     extensions = ("datasource", "pipe", "connection")
 
-    def __init__(self, folder: str):
+    def __init__(self, folder: str, workspace_name: str):
         self.folder = folder
+        self.workspace_name = workspace_name
 
     @property
     def path(self) -> Path:
@@ -55,6 +56,15 @@ class Project:
     def connections(self) -> List[str]:
         return sorted([Path(f).stem for f in glob.glob(f"{self.path}/**/*.connection", recursive=False)])
 
+    def get_datasource_files(self) -> List[str]:
+        return glob.glob(f"{self.path}/**/*.datasource", recursive=False)
+
+    def get_pipe_files(self) -> List[str]:
+        return glob.glob(f"{self.path}/**/*.pipe", recursive=False)
+
+    def get_connection_files(self) -> List[str]:
+        return glob.glob(f"{self.path}/**/*.connection", recursive=False)
+
     def get_pipe_datafile(self, filename: str) -> Optional[Datafile]:
         try:
             return parse_pipe(filename).datafile

{tinybird-0.0.1.dev151.dist-info → tinybird-0.0.1.dev153.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev151
+Version: 0.0.1.dev153
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev151.dist-info → tinybird-0.0.1.dev153.dist-info}/RECORD

@@ -12,23 +12,24 @@ tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
 tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
 tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
 tinybird/ch_utils/engine.py,sha256=BZuPM7MFS7vaEKK5tOMR2bwSAgJudPrJt27uVEwZmTY,40512
-tinybird/tb/__cli__.py,sha256=aj33xy8_AEPy1i5FQnKXG-Z1k76v879JPRjaUTlX_28,252
+tinybird/tb/__cli__.py,sha256=V2P7LUM2Vw2frXY-02zxDbU0UWJW6e_-DJj6Csa4P5Y,252
 tinybird/tb/check_pypi.py,sha256=rW4QmDRbtgKdUUwJCnBkVjmTjZSZGN-XgZhx7vMkC0w,1009
 tinybird/tb/cli.py,sha256=uPV6pvi4aYVfaiGs0DQO-eoi1g9dHlrgvhutkXkdJko,1075
 tinybird/tb/client.py,sha256=aaPKq5C77e72kR7IMv9WrvnvNki8mKMOTi9EsCp0eUc,55962
 tinybird/tb/config.py,sha256=jT9xndpeCY_g0HdB5qE2EquC0TFRRnkPnQFWZWd04jo,3998
 tinybird/tb/modules/auth.py,sha256=_OeYnmTH83lnqCgQEdS6K0bx1KBUeRmZk2M7JnRmWpk,9037
-tinybird/tb/modules/build.py,sha256=5SX59ssq0KaRrgzk21KmYUnB6xOdQ_O59n7rxFBQQ1Y,17393
+tinybird/tb/modules/build.py,sha256=PJxsI4Wqw8lRkSw0RrQqkwAHYClikJ4ZKd9i1_gRdkk,18488
 tinybird/tb/modules/cicd.py,sha256=Cn1DyE28w2maQgIJF4IjZU7J7ULPvMG11DjN2OH-cqE,7161
-tinybird/tb/modules/cli.py,sha256=AJooz6uiluC7eMO2ZlXluWL-NNLwTKb9kqkv3yqg6S4,14227
+tinybird/tb/modules/cli.py,sha256=TCms3gyokv3ES1yD6WuL_mLNEqBcAFQHSvgOnr6RYDY,14266
 tinybird/tb/modules/common.py,sha256=WzF_zRtAl3FKqbzK6yi_IjeeGIjQNu6SKnP-PX-o3Kk,85196
 tinybird/tb/modules/config.py,sha256=ziqW_t_mRVvWOd85VoB4vKyvgMkEfpXDf9H4v38p2xc,11422
 tinybird/tb/modules/connection.py,sha256=7oOR7x4PhBcm1ETFFCH2YJ_3oeGXjAbmx1cnZX9_L70,9014
 tinybird/tb/modules/copy.py,sha256=2Mm4FWKehOG7CoOhiF1m9UZJgJn0W1_cMolqju8ONYg,5805
-tinybird/tb/modules/create.py,sha256=H06HmD-4CCYVMvM6doUvvKF3wU_6bBjefkwXW4ZVhE8,17378
+tinybird/tb/modules/create.py,sha256=OHUvuHuvP0iecPPGI4eVOHOgR20qy7a_Sw7sbJKuG8g,17411
 tinybird/tb/modules/datasource.py,sha256=V314rkpdVxVMjsp5qcSCTqDlmp4Vu--qM07BoWh-aqs,17783
-tinybird/tb/modules/deployment.py,sha256=T0lwhAT8FwZZcRnA-JDMfPUrj5MOXcQ8blxYIK2igbc,24363
+tinybird/tb/modules/deployment.py,sha256=a6CZrYqAM-t6WxGKjgg16ZvncpZBta5gBq7YEBPBoQc,25811
 tinybird/tb/modules/deprecations.py,sha256=nI_VPCB0Seyhlk2Foomoj3oUnx6GV_8Q5DMYVDi9L-o,3751
+tinybird/tb/modules/dev_server.py,sha256=57FCKuWpErwYUYgHspYDkLWEm9F4pbvVOtMrFXX1fVU,10129
 tinybird/tb/modules/endpoint.py,sha256=XySDt3pk66vxOZ0egUfz4bY8bEk3BjOXkv-L0OIJ3sc,12083
 tinybird/tb/modules/exceptions.py,sha256=5jK91w1LPmtqIUfDpHe_Op5OxGz8-p1BPgtLREMIni0,5217
 tinybird/tb/modules/feedback_manager.py,sha256=NoflMdTcUs6JF2IUZS8TatItxWDNCfwaIJnKtbY8e3w,77447
@@ -41,10 +42,10 @@ tinybird/tb/modules/local_common.py,sha256=jNbVG_nk5sNduPo3JDSNtop1H2X1Tdd5j4q3x
 tinybird/tb/modules/login.py,sha256=fmXPSdvJnKPv03chptGuu3_Fm6LhP6kUsUKhrmT8rJc,8269
 tinybird/tb/modules/logout.py,sha256=ULooy1cDBD02-r7voZmhV7udA0ML5tVuflJyShrh56Y,1022
 tinybird/tb/modules/materialization.py,sha256=QJX5kCPhhm6IXBO1JsalVfbQdypCe_eOUDZ_WHJZWS8,5478
-tinybird/tb/modules/mock.py,sha256=tyhs6izqVGmDzMHdW5yHiprNPjeJDa2I-cLwrLRDC_U,4816
+tinybird/tb/modules/mock.py,sha256=z_mYNVAu5aeq-5qGB0kSkliOX4j62fgUuLi1-ZcTamA,5304
 tinybird/tb/modules/open.py,sha256=OuctINN77oexpSjth9uoIZPCelKO4Li-yyVxeSnk1io,1371
 tinybird/tb/modules/pipe.py,sha256=AQKEDagO6e3psPVjJkS_MDbn8aK-apAiLp26k7jgAV0,2432
-tinybird/tb/modules/project.py,sha256=RdVOzJiuFPqkCzPhzGetbgcFeiMEn9IMviFCG58XKgI,3142
+tinybird/tb/modules/project.py,sha256=PwEG8Ob9bW4HZxh1jQq2LyDfIoitqF69Jdu0WsWHMCg,3568
 tinybird/tb/modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb/modules/secret.py,sha256=WsqzxxLh9W_jkuHL2JofMXdIJy0lT5WEI-7bQSIDgAc,2921
 tinybird/tb/modules/shell.py,sha256=Zd_4Ak_5tKVX-cw6B4ag36xZeEGHeh-jZpAsIXkoMoE,14116
@@ -59,7 +60,7 @@ tinybird/tb/modules/datafile/build.py,sha256=d_h3pRFDPFrDKGhpFx2iejY25GuB2k8yfNo
 tinybird/tb/modules/datafile/build_common.py,sha256=LU24kAQmxDJIyoIapDaYG-SU3P4FrMG9UBf8m9PgVSI,4565
 tinybird/tb/modules/datafile/build_datasource.py,sha256=nXEQ0qHdq2ai7jJTv8H2d7eeDPBYzLn8VY7zMtOYb8M,17382
 tinybird/tb/modules/datafile/build_pipe.py,sha256=6Cwjf3BKEF3-oQ9PipsQfK-Z43nSwtA4qJAUoysI7Uc,11385
-tinybird/tb/modules/datafile/common.py,sha256=-F-7SHlMJ7cumJ3msAeI9_RbA7L-8D6aWerzthR6Oco,89401
+tinybird/tb/modules/datafile/common.py,sha256=1xM8R2NVmk8uQpU910Iqp_YGM1SHYoXaLZgjBYIJl54,89771
 tinybird/tb/modules/datafile/diff.py,sha256=MTmj53RYjER4neLgWVjabn-FKVFgh8h8uYiBo55lFQg,6757
 tinybird/tb/modules/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
 tinybird/tb/modules/datafile/fixture.py,sha256=DrRWivcvo_1rn7LlVUnHcXccdgx9yVj63mzBkUwCzk8,1420
@@ -79,8 +80,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev151.dist-info/METADATA,sha256=mEbvyHyE45_UqqUueYfSUhRzd9tzpwxSg8aHLS-VeYg,1612
-tinybird-0.0.1.dev151.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev151.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev151.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev151.dist-info/RECORD,,
+tinybird-0.0.1.dev153.dist-info/METADATA,sha256=ssrKpwmrdIOxtjrzdNFLCOKObE89ZhkF0hO0VSOC-tA,1612
+tinybird-0.0.1.dev153.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev153.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev153.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev153.dist-info/RECORD,,