tinybird 0.0.1.dev112__tar.gz → 0.0.1.dev114__tar.gz

{tinybird-0.0.1.dev112 → tinybird-0.0.1.dev114}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev112
+Version: 0.0.1.dev114
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev112 → tinybird-0.0.1.dev114}/tinybird/prompts.py

@@ -954,3 +954,35 @@ FORWARD_QUERY >
     </forward_query_example>
 </deployment_instruction>
 """
+
+
+def readme_prompt(readme: str, host: str, token: str, resources_xml: str) -> str:
+    return f"""
+You are an expert in SQL and Tinybird. Follow these instructions to generate a new README.md file for a tinybird project:
+Current README.md file:
+<current_resources_xml>
+{resources_xml}
+</current_resources_xml>
+<readme>{readme}</readme>
+<readme_instructions>
+- If it is not present in the current readme, generate a new ## Tinybird section with the following content:
+    - ### Overview section:
+        - Explaining the purpose of the project.
+    - ### Data sources section: 
+        - Explaining the purpose of each datasource.
+        - Add a snippet of how to ingest data into each datasource like the following (where the payload example matches the datasource schema respecting non-nullable types):
+        curl -X POST "{host}/v0/events?name=events" \
+    -H "Authorization: Bearer {token}" \
+    -d '{{"date":"2025-01-31","id":"123","user_id":"abc"}}'
+    - ### Endpoints section:
+        - Explaining the purpose of each endpoint.
+        - Add a snippet of how to use each endpoint like the following:
+        curl -X GET "{host}/v0/pipes/pipe_name.json?token={token}"
+        - DateTime parameters must be formatted as YYYY-MM-DD HH:MM:SS, or else will fail.
+- Do not include any other extra info related to Tinybird just maintain the existing one in the <readme> tag.
+- Make sure the API host is correct, {host}.
+- It is mandatory to return a <resource> tag with type "readme" in the response.
+- The response must follow the following format:
+<readme>[readme content here]</readme>
+</readme_instructions>
+"""

{tinybird-0.0.1.dev112 → tinybird-0.0.1.dev114}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev112'
-__revision__ = 'eb9e376'
+__version__ = '0.0.1.dev114'
+__revision__ = '9878107'

{tinybird-0.0.1.dev112 → tinybird-0.0.1.dev114}/tinybird/tb/cli.py

@@ -24,7 +24,6 @@ import tinybird.tb.modules.materialization
 import tinybird.tb.modules.mock
 import tinybird.tb.modules.open
 import tinybird.tb.modules.pipe
-import tinybird.tb.modules.playground
 import tinybird.tb.modules.secret
 import tinybird.tb.modules.tag
 import tinybird.tb.modules.test

{tinybird-0.0.1.dev112 → tinybird-0.0.1.dev114}/tinybird/tb/modules/build.py

@@ -4,23 +4,30 @@ import logging
 import sys
 import threading
 import time
+from copy import deepcopy
 from functools import partial
 from pathlib import Path
-from typing import Callable, Optional
+from typing import Callable, List, Optional
 from urllib.parse import urlencode
 
 import click
 import requests
 
+import tinybird.context as context
 from tinybird.client import TinyB
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import push_data
+from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.datafile.build import folder_build
+from tinybird.tb.modules.datafile.exceptions import ParseException
 from tinybird.tb.modules.datafile.fixture import get_fixture_dir, persist_fixture
+from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
+from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
+from tinybird.tb.modules.datafile.playground import folder_playground
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.project import Project
 from tinybird.tb.modules.shell import Shell, print_table_formatted
-from tinybird.tb.modules.watch import watch_project
+from tinybird.tb.modules.watch import watch_files, watch_project
 
 
 @cli.command()
@@ -43,8 +50,12 @@ def build(ctx: click.Context, watch: bool) -> None:
 
 
 @cli.command("dev", help="Build the project server side and watch for changes.")
+@click.option("--data-origin", type=str, default="", help="Data origin: local or cloud")
 @click.pass_context
-def dev(ctx: click.Context) -> None:
+def dev(ctx: click.Context, data_origin: str) -> None:
+    if data_origin == "cloud":
+        return dev_cloud(ctx)
+
     project: Project = ctx.ensure_object(dict)["project"]
     tb_client: TinyB = ctx.ensure_object(dict)["client"]
     click.echo(FeedbackManager.highlight_building_project())
@@ -284,3 +295,113 @@ def rebuild_fixture_sql(project: Project, tb_client: TinyB, sql_file: str) -> No
     result = asyncio.run(tb_client.query(f"{sql} FORMAT JSON"))
     data = result.get("data", [])
     persist_fixture(datasource_name, data, project.folder)
+
+
+def is_vendor(f: Path) -> bool:
+    return f.parts[0] == "vendor"
+
+
+def get_vendor_workspace(f: Path) -> str:
+    return f.parts[1]
+
+
+def is_endpoint(f: Path) -> bool:
+    return f.suffix == ".pipe" and not is_vendor(f) and f.parts[0] == "endpoints"
+
+
+def is_pipe(f: Path) -> bool:
+    return f.suffix == ".pipe" and not is_vendor(f)
+
+
+def check_filenames(filenames: List[str]):
+    parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
+    incl_suffix = ".incl"
+
+    for filename in filenames:
+        file_suffix = Path(filename).suffix
+        if file_suffix == incl_suffix:
+            continue
+
+        parser = parser_matrix.get(file_suffix)
+        if not parser:
+            raise ParseException(FeedbackManager.error_unsupported_datafile(extension=file_suffix))
+
+        parser(filename)
+
+
+def dev_cloud(
+    ctx: click.Context,
+) -> None:
+    project: Project = ctx.ensure_object(dict)["project"]
+    config = CLIConfig.get_project_config()
+    tb_client: TinyB = config.get_client()
+    context.disable_template_security_validation.set(True)
+
+    async def process(filenames: List[str], watch: bool = False):
+        datafiles = [f for f in filenames if f.endswith(".datasource") or f.endswith(".pipe")]
+        if len(datafiles) > 0:
+            check_filenames(filenames=datafiles)
+            await folder_playground(
+                project, config, tb_client, filenames=datafiles, is_internal=False, current_ws=None, local_ws=None
+            )
+        if len(filenames) > 0 and watch:
+            filename = filenames[0]
+            await build_and_print_resource(config, tb_client, filename)
+
+    datafiles = project.get_project_files()
+    filenames = datafiles
+
+    async def build_once(filenames: List[str]):
+        ok = False
+        try:
+            click.echo(FeedbackManager.highlight(message="» Building project...\n"))
+            time_start = time.time()
+            await process(filenames=filenames, watch=False)
+            time_end = time.time()
+            elapsed_time = time_end - time_start
+
+            click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s"))
+            ok = True
+        except Exception as e:
+            error_path = Path(".tb_error.txt")
+            if error_path.exists():
+                content = error_path.read_text()
+                content += f"\n\n{str(e)}"
+                error_path.write_text(content)
+            else:
+                error_path.write_text(str(e))
+            click.echo(FeedbackManager.error_exception(error=e))
+            ok = False
+        return ok
+
+    build_ok = asyncio.run(build_once(filenames))
+
+    shell = Shell(project=project, tb_client=tb_client, playground=True)
+    click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
+    watcher_thread = threading.Thread(
+        target=watch_files, args=(filenames, process, shell, project, build_ok), daemon=True
+    )
+    watcher_thread.start()
+    shell.run()
+
+
+async def build_and_print_resource(config: CLIConfig, tb_client: TinyB, filename: str):
+    resource_path = Path(filename)
+    name = resource_path.stem
+    playground_name = name if filename.endswith(".pipe") else None
+    user_client = deepcopy(tb_client)
+    user_client.token = config.get_user_token() or ""
+    cli_params = {}
+    cli_params["workspace_id"] = config.get("id", None)
+    data = await user_client._req(f"/v0/playgrounds?{urlencode(cli_params)}")
+    playgrounds = data["playgrounds"]
+    playground = next((p for p in playgrounds if p["name"] == (f"{playground_name}" + "__tb__playground")), None)
+    if not playground:
+        return
+    playground_id = playground["id"]
+    last_node = playground["nodes"][-1]
+    if not last_node:
+        return
+    node_sql = last_node["sql"]
+    res = await tb_client.query(f"{node_sql} FORMAT JSON", playground=playground_id)
+    print_table_formatted(res, name)

{tinybird-0.0.1.dev112 → tinybird-0.0.1.dev114}/tinybird/tb/modules/cicd.py

@@ -120,7 +120,9 @@ tinybird_ci_workflow:
   services:
     - name: tinybirdco/tinybird-local:beta
       alias: tinybird-local
+"""
 
+GITLAB_CD_YML = """
 tinybird_cd_workflow:
   image: ubuntu:latest
   stage: deploy
@@ -217,6 +219,11 @@ class GitLabCICDGenerator(CICDGeneratorBase):
             file_name="tinybird-ci.yml",
             dir_path=".gitlab/tinybird",
         ),
+        CICDFile(
+            template=GITLAB_CD_YML,
+            file_name="tinybird-cd.yml",
+            dir_path=".gitlab/tinybird",
+        ),
     ]
 
 

{tinybird-0.0.1.dev112 → tinybird-0.0.1.dev114}/tinybird/tb/modules/cli.py

@@ -372,7 +372,7 @@ async def create_ctx_client(ctx: Context, config: Dict[str, Any], cloud: bool, b
     if command in commands_without_ctx_client:
         return None
 
-    commands_always_cloud = ["pull", "playground", "infra"]
+    commands_always_cloud = ["pull", "infra"]
     commands_always_build = ["build", "test", "dev", "create"]
     commands_always_local: List[str] = []
     if (

{tinybird-0.0.1.dev112 → tinybird-0.0.1.dev114}/tinybird/tb/modules/create.py

@@ -6,7 +6,7 @@ from typing import Optional, Tuple
 import click
 
 from tinybird.client import TinyB
-from tinybird.prompts import create_prompt, mock_prompt, rules_prompt
+from tinybird.prompts import create_prompt, mock_prompt, readme_prompt, rules_prompt
 from tinybird.tb.modules.cicd import init_cicd
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import _generate_datafile, coro, generate_datafile
@@ -88,12 +88,32 @@ async def create(
             click.echo(FeedbackManager.success(message="✓ Scaffolding completed!\n"))
             created_something = True
         result = ""
-        if data or prompt:
+        if (data or prompt) and user_token:
             click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
             result, created_something = await create_resources(
                 local_client, tb_client, user_token, data, prompt, folder
             )
             click.echo(FeedbackManager.success(message="✓ Done!\n"))
+            if prompt:
+                readme_path = Path(root_folder) / "README.md"
+                if readme_path.exists():
+                    click.echo(FeedbackManager.highlight(message="\n» Updating README.md..."))
+                else:
+                    click.echo(FeedbackManager.highlight(message="\n» Creating README.md..."))
+                    readme_path.touch()
+                llm = LLM(user_token=str(user_token), host=tb_client.host)
+                readme_user_prompt = prompt or ""
+                all_resources_xml = get_resources_xml(project)
+                readme_response = llm.ask(
+                    system_prompt=readme_prompt(
+                        readme_path.read_text(), tb_client.host, tb_client.token, all_resources_xml
+                    ),
+                    prompt=readme_user_prompt,
+                )
+                readme_result = extract_xml(readme_response, "readme")
+                readme_path.write_text(readme_result)
+                click.echo(FeedbackManager.success(message="✓ Done!\n"))
+                created_something = True
 
         if not already_has_cicd(root_folder):
             click.echo(FeedbackManager.highlight(message="\n» Creating CI/CD files for GitHub and GitLab..."))
@@ -390,3 +410,30 @@ def get_context() -> str:
 def save_context(prompt: str, feedback: str):
     context_file = get_context_file()
     context_file.write_text(f"- {prompt}\n{feedback}")
+
+
+def get_resources_xml(project: Project) -> str:
+    folder = project.folder
+    datasource_paths = [
+        Path(folder) / "datasources" / f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")
+    ]
+    pipes_paths = [
+        Path(folder) / "endpoints" / f for f in os.listdir(Path(folder) / "endpoints") if f.endswith(".pipe")
+    ]
+    resources_xml = "\n".join(
+        [
+            f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
+            for resource_type, resource_name, resource_content in [
+                ("datasource", ds.stem, ds.read_text()) for ds in datasource_paths
+            ]
+            + [
+                (
+                    "pipe",
+                    pipe.stem,
+                    pipe.read_text(),
+                )
+                for pipe in pipes_paths
+            ]
+        ]
+    )
+    return resources_xml

{tinybird-0.0.1.dev112 → tinybird-0.0.1.dev114}/tinybird/tb/modules/datafile/fixture.py

@@ -5,7 +5,9 @@ from tinybird.tb.modules.common import format_data_to_ndjson
 
 
 def get_fixture_dir(folder: str) -> Path:
-    return Path(folder) / "fixtures"
+    fixture_dir = Path(folder) / "fixtures"
+    fixture_dir.mkdir(exist_ok=True)
+    return fixture_dir
 
 
 def persist_fixture_sql(fixture_name: str, sql: str, folder: str) -> Path:

{tinybird-0.0.1.dev112 → tinybird-0.0.1.dev114}/tinybird/tb/modules/infra.py

@@ -371,7 +371,7 @@ async def infra_init(
 
     # Create infra directory if it doesn't exist
     infra_dir = Path(f"infra/{cloud_provider}")
-    infra_dir.mkdir(exist_ok=True)
+    infra_dir.mkdir(parents=True, exist_ok=True)
     yaml_path = infra_dir / "k8s.yaml"
     tf_path = infra_dir / "main.tf"
     config_path = infra_dir / "config.json"

{tinybird-0.0.1.dev112 → tinybird-0.0.1.dev114}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev112
+Version: 0.0.1.dev114
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev112 → tinybird-0.0.1.dev114}/tinybird.egg-info/SOURCES.txt

@@ -53,7 +53,6 @@ tinybird/tb/modules/materialization.py
 tinybird/tb/modules/mock.py
 tinybird/tb/modules/open.py
 tinybird/tb/modules/pipe.py
-tinybird/tb/modules/playground.py
 tinybird/tb/modules/project.py
 tinybird/tb/modules/regions.py
 tinybird/tb/modules/secret.py

tinybird-0.0.1.dev112/tinybird/tb/modules/playground.py

@@ -1,135 +0,0 @@
-import asyncio
-import threading
-import time
-from copy import deepcopy
-from pathlib import Path
-from typing import List
-from urllib.parse import urlencode
-
-import click
-
-import tinybird.context as context
-from tinybird.client import TinyB
-from tinybird.tb.modules.cli import cli
-from tinybird.tb.modules.config import CLIConfig
-from tinybird.tb.modules.datafile.exceptions import ParseException
-from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
-from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
-from tinybird.tb.modules.datafile.playground import folder_playground
-from tinybird.tb.modules.feedback_manager import FeedbackManager
-from tinybird.tb.modules.project import Project
-from tinybird.tb.modules.shell import Shell, print_table_formatted
-from tinybird.tb.modules.watch import watch_files
-
-
-def is_vendor(f: Path) -> bool:
-    return f.parts[0] == "vendor"
-
-
-def get_vendor_workspace(f: Path) -> str:
-    return f.parts[1]
-
-
-def is_endpoint(f: Path) -> bool:
-    return f.suffix == ".pipe" and not is_vendor(f) and f.parts[0] == "endpoints"
-
-
-def is_pipe(f: Path) -> bool:
-    return f.suffix == ".pipe" and not is_vendor(f)
-
-
-def check_filenames(filenames: List[str]):
-    parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
-    incl_suffix = ".incl"
-
-    for filename in filenames:
-        file_suffix = Path(filename).suffix
-        if file_suffix == incl_suffix:
-            continue
-
-        parser = parser_matrix.get(file_suffix)
-        if not parser:
-            raise ParseException(FeedbackManager.error_unsupported_datafile(extension=file_suffix))
-
-        parser(filename)
-
-
-@cli.command()
-@click.pass_context
-def playground(
-    ctx: click.Context,
-) -> None:
-    """Build the project in Tinybird Local."""
-    project: Project = ctx.ensure_object(dict)["project"]
-    tb_client: TinyB = ctx.ensure_object(dict)["client"]
-    config = CLIConfig.get_project_config()
-    context.disable_template_security_validation.set(True)
-
-    async def process(filenames: List[str], watch: bool = False):
-        datafiles = [f for f in filenames if f.endswith(".datasource") or f.endswith(".pipe")]
-        if len(datafiles) > 0:
-            check_filenames(filenames=datafiles)
-            await folder_playground(
-                project, config, tb_client, filenames=datafiles, is_internal=False, current_ws=None, local_ws=None
-            )
-        if len(filenames) > 0 and watch:
-            filename = filenames[0]
-            await build_and_print_resource(config, tb_client, filename)
-
-    datafiles = project.get_project_files()
-    filenames = datafiles
-
-    async def build_once(filenames: List[str]):
-        ok = False
-        try:
-            click.echo(FeedbackManager.highlight(message="» Building project...\n"))
-            time_start = time.time()
-            await process(filenames=filenames, watch=False)
-            time_end = time.time()
-            elapsed_time = time_end - time_start
-
-            click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s"))
-            ok = True
-        except Exception as e:
-            error_path = Path(".tb_error.txt")
-            if error_path.exists():
-                content = error_path.read_text()
-                content += f"\n\n{str(e)}"
-                error_path.write_text(content)
-            else:
-                error_path.write_text(str(e))
-            click.echo(FeedbackManager.error_exception(error=e))
-            ok = False
-        return ok
-
-    build_ok = asyncio.run(build_once(filenames))
-
-    shell = Shell(project=project, tb_client=tb_client, playground=True)
-    click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
-    watcher_thread = threading.Thread(
-        target=watch_files, args=(filenames, process, shell, project, build_ok), daemon=True
-    )
-    watcher_thread.start()
-    shell.run()
-
-
-async def build_and_print_resource(config: CLIConfig, tb_client: TinyB, filename: str):
-    resource_path = Path(filename)
-    name = resource_path.stem
-    playground_name = name if filename.endswith(".pipe") else None
-    user_client = deepcopy(tb_client)
-    user_client.token = config.get_user_token() or ""
-    cli_params = {}
-    cli_params["workspace_id"] = config.get("id", None)
-    data = await user_client._req(f"/v0/playgrounds?{urlencode(cli_params)}")
-    playgrounds = data["playgrounds"]
-    playground = next((p for p in playgrounds if p["name"] == (f"{playground_name}" + "__tb__playground")), None)
-    if not playground:
-        return
-    playground_id = playground["id"]
-    last_node = playground["nodes"][-1]
-    if not last_node:
-        return
-    node_sql = last_node["sql"]
-    res = await tb_client.query(f"{node_sql} FORMAT JSON", playground=playground_id)
-    print_table_formatted(res, name)