tinybird 0.0.1.dev175__tar.gz → 0.0.1.dev176__tar.gz

{tinybird-0.0.1.dev175 → tinybird-0.0.1.dev176}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev175
+Version: 0.0.1.dev176
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -14,7 +14,7 @@ Requires-Dist: click<8.2,>=8.1.6
 Requires-Dist: clickhouse-toolset==0.34.dev0
 Requires-Dist: colorama==0.4.6
 Requires-Dist: cryptography~=41.0.0
-Requires-Dist: croniter==1.3.8
+Requires-Dist: croniter==1.3.15
 Requires-Dist: docker==7.1.0
 Requires-Dist: GitPython~=3.1.32
 Requires-Dist: humanfriendly~=8.2

{tinybird-0.0.1.dev175 → tinybird-0.0.1.dev176}/tinybird/prompts.py

@@ -461,8 +461,7 @@ Use the following feedback to regenerate the response:
 """
 
     return f"""
-Given the schema for a Tinybird datasource, return a can you create a clickhouse sql query to generate some random data that matches that schema.
-
+Given the schema for a Tinybird datasource, return a valid clickhouse sql query to generate some random data that matches that schema.
 Response format MUST be just a valid clickhouse sql query.
 
 <example>
@@ -722,6 +721,7 @@ datasource_instructions = """
     - Use MergeTree engine by default.
     - Use AggregatingMergeTree engine when the datasource is the target of a materialized pipe.
     - Use always json paths to define the schema. Example: `user_id` String `json:$.user_id`,
+    - Array columns are supported with a special syntax. Example: `items` Array(String) `json:$.items[:]`
     - If the datasource is using an S3 or GCS connection, they need to set IMPORT_CONNECTION_NAME, IMPORT_BUCKET_URI and IMPORT_SCHEDULE (GCS @on-demand only, S3 supports @auto too)
 </datasource_file_instructions>
 """

{tinybird-0.0.1.dev175 → tinybird-0.0.1.dev176}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev175'
-__revision__ = '943d7b4'
+__version__ = '0.0.1.dev176'
+__revision__ = '09be7b9'

{tinybird-0.0.1.dev175 → tinybird-0.0.1.dev176}/tinybird/tb/modules/create.py

@@ -2,7 +2,7 @@ import glob
 import os
 import re
 from pathlib import Path
-from typing import Any, Dict, Optional, Tuple
+from typing import Any, Dict, List, Optional
 
 import click
 
@@ -69,7 +69,7 @@ async def create(
 
     try:
         tb_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
-        user_token: Optional[str] = None
+        user_token: str = ""
         created_something = False
         if prompt:
             user_token = ctx_config.get("user_token")
@@ -81,32 +81,40 @@ async def create(
             create_project_structure(folder)
             click.echo(FeedbackManager.success(message="✓ Scaffolding completed!\n"))
             created_something = True
-        result = ""
-
-        if data or (prompt and user_token):
+        result: List[Path] = []
+        if data or prompt:
             click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
-            result, created_something = await create_resources(tb_client, user_token, data, prompt, project, ctx_config)
+
+        data_result: List[Path] = []
+        if data:
+            data_result = await create_resources_from_data(data, project, ctx_config)
+            result.extend(data_result)
+
+        prompt_result: List[Path] = []
+        if prompt:
+            prompt_result = await create_resources_from_prompt(tb_client, user_token, prompt, project)
+            result.extend(prompt_result)
+            readme_path = Path(root_folder) / "README.md"
+            if readme_path.exists():
+                click.echo(FeedbackManager.highlight(message="\n» Updating README.md..."))
+            else:
+                click.echo(FeedbackManager.highlight(message="\n» Creating README.md..."))
+                readme_path.touch()
+            llm = LLM(user_token=str(user_token), host=tb_client.host)
+            readme_user_prompt = prompt or ""
+            all_resources_xml = get_resources_xml(project)
+            readme_response = llm.ask(
+                system_prompt=readme_prompt(
+                    readme_path.read_text(), tb_client.host, "$TB_ADMIN_TOKEN", all_resources_xml
+                ),
+                prompt=readme_user_prompt,
+            )
+            readme_result = extract_xml(readme_response, "readme")
+            readme_path.write_text(readme_result)
+            created_something = True
+
+        if data or prompt:
             click.echo(FeedbackManager.success(message="✓ Done!\n"))
-            if prompt:
-                readme_path = Path(root_folder) / "README.md"
-                if readme_path.exists():
-                    click.echo(FeedbackManager.highlight(message="\n» Updating README.md..."))
-                else:
-                    click.echo(FeedbackManager.highlight(message="\n» Creating README.md..."))
-                    readme_path.touch()
-                llm = LLM(user_token=str(user_token), host=tb_client.host)
-                readme_user_prompt = prompt or ""
-                all_resources_xml = get_resources_xml(project)
-                readme_response = llm.ask(
-                    system_prompt=readme_prompt(
-                        readme_path.read_text(), tb_client.host, "$TB_ADMIN_TOKEN", all_resources_xml
-                    ),
-                    prompt=readme_user_prompt,
-                )
-                readme_result = extract_xml(readme_response, "readme")
-                readme_path.write_text(readme_result)
-                click.echo(FeedbackManager.success(message="✓ Done!\n"))
-                created_something = True
 
         if not already_has_cicd(root_folder):
             click.echo(FeedbackManager.highlight(message="\n» Creating CI/CD files for GitHub and GitLab..."))
@@ -125,17 +133,18 @@ async def create(
             click.echo(FeedbackManager.highlight(message="\n» Generating fixtures..."))
 
             if data:
-                ds_name = os.path.basename(data.split(".")[0])
-                data_path = Path(data)
-                data_content = data_path.read_text()
-                data_format = data_path.suffix.lstrip(".")
-                datasource_path = Path(folder) / "datasources" / f"{ds_name}.datasource"
-                click.echo(FeedbackManager.info(message=f"✓ /fixtures/{ds_name}"))
-                persist_fixture(ds_name, data_content, folder, format=data_format)
-                created_something = True
-            elif prompt and user_token:
-                for datasource_file in project.get_datasource_files():
-                    datasource_path = Path(datasource_file)
+                for ds_path in [ds for ds in data_result if ds.suffix == ".datasource"]:
+                    data_path = Path(data)
+                    data_content = data_path.read_text()
+                    ds_name = ds_path.stem
+                    data_format = data_path.suffix.lstrip(".")
+                    datasource_path = Path(folder) / "datasources" / f"{ds_name}.datasource"
+                    click.echo(FeedbackManager.info(message=f"✓ /fixtures/{ds_name}"))
+                    persist_fixture(ds_name, data_content, folder, format=data_format)
+                    created_something = True
+            elif prompt and prompt_result:
+                ds_results = [path for path in prompt_result if path.suffix == ".datasource"]
+                for datasource_path in ds_results:
                     datasource_name = datasource_path.stem
                     datasource_content = datasource_path.read_text()
                     has_json_path = "`json:" in datasource_content
@@ -157,7 +166,7 @@ async def create(
                             click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}"))
                             created_something = True
 
-        if not created_something:
+        if not created_something and not len(result) > 0:
             click.echo(FeedbackManager.warning(message="△ No resources created\n"))
     except Exception as e:
         raise CLICreateException(FeedbackManager.error(message=str(e)))
@@ -178,10 +187,19 @@ def validate_project_structure(folder: str) -> bool:
     return len(datasources) > 0 or len(pipes) > 0
 
 
-def should_generate_fixtures(result: str) -> bool:
+def should_generate_fixtures(result: List[Path]) -> List[Path]:
     if not result:
-        return False
-    return "<type>datasource</type>" in result
+        return []
+    return [
+        path
+        for path in result
+        if path.suffix == ".datasource"
+        # we only want to generate fixtures for MergeTree or Null engines
+        and (
+            re.search(r'ENGINE\s+(?:"MergeTree"|MergeTree|"Null"|Null)', path.read_text())
+            or not re.search(r"ENGINE\s+", path.read_text())
+        )
+    ]
 
 
 def already_has_cicd(folder: str) -> bool:
@@ -205,113 +223,86 @@ def create_project_structure(folder: str):
         click.echo(FeedbackManager.info_path_created(path=x))
 
 
-async def create_resources(
+async def create_resources_from_prompt(
     tb_client: TinyB,
-    user_token: Optional[str],
-    data: Optional[str],
-    prompt: Optional[str],
+    user_token: str,
+    prompt: str,
     project: Project,
-    config: Dict[str, Any],
-) -> Tuple[str, bool]:
-    result = ""
-    created_any_resource = False
-    folder_path = project.path
-    if data:
-        local_client = await get_tinybird_local_client(config)
-        path = folder_path / data
-        format = path.suffix.lstrip(".")
-        await _generate_datafile(str(path), local_client, format=format, force=True)
-        name = path.stem
-        generate_pipe_file(
-            f"{name}_endpoint",
-            f"""
-NODE endpoint
-SQL >
-    SELECT * from {name}
-TYPE ENDPOINT
-            """,
-            project.folder,
-        )
-        result = (
-            f"<response><resource><type>datasource</type><name>{name}</name><content></content></resource></response>"
-        )
-        created_any_resource = True
-
-    elif prompt and user_token:
-        datasource_paths = [Path(ds_file) for ds_file in project.get_datasource_files()]
-        pipes_paths = [Path(pipe_file) for pipe_file in project.get_pipe_files()]
-        connections_paths = [Path(conn_file) for conn_file in project.get_connection_files()]
-        resources_xml = "\n".join(
-            [
-                f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
-                for resource_type, resource_name, resource_content in [
-                    ("datasource", ds.stem, ds.read_text()) for ds in datasource_paths
-                ]
-                + [
-                    (
-                        "pipe",
-                        pipe.stem,
-                        pipe.read_text(),
-                    )
-                    for pipe in pipes_paths
-                ]
-                + [
-                    (
-                        "connection",
-                        conn.stem,
-                        conn.read_text(),
-                    )
-                    for conn in connections_paths
-                ]
+) -> List[Path]:
+    result: List[Path] = []
+    datasource_paths = [Path(ds_file) for ds_file in project.get_datasource_files()]
+    pipes_paths = [Path(pipe_file) for pipe_file in project.get_pipe_files()]
+    connections_paths = [Path(conn_file) for conn_file in project.get_connection_files()]
+    resources_xml = "\n".join(
+        [
+            f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
+            for resource_type, resource_name, resource_content in [
+                ("datasource", ds.stem, ds.read_text()) for ds in datasource_paths
+            ]
+            + [
+                (
+                    "pipe",
+                    pipe.stem,
+                    pipe.read_text(),
+                )
+                for pipe in pipes_paths
             ]
+            + [
+                (
+                    "connection",
+                    conn.stem,
+                    conn.read_text(),
+                )
+                for conn in connections_paths
+            ]
+        ]
+    )
+    llm = LLM(user_token=user_token, host=tb_client.host)
+    prompt_result = llm.ask(system_prompt=create_prompt(resources_xml), prompt=prompt)
+    prompt_result = extract_xml(prompt_result, "response")
+    resources = parse_xml(prompt_result, "resource")
+    datasources = []
+    pipes = []
+    connections = []
+    for resource_xml in resources:
+        resource_type = extract_xml(resource_xml, "type")
+        name = extract_xml(resource_xml, "name")
+        content = extract_xml(resource_xml, "content")
+        resource = {
+            "name": name,
+            "content": content,
+        }
+        if resource_type.lower() == "datasource":
+            datasources.append(resource)
+        elif resource_type.lower() == "pipe":
+            pipes.append(resource)
+        elif resource_type.lower() == "connection":
+            connections.append(resource)
+
+    for ds in datasources:
+        content = ds["content"].replace("```", "")
+        filename = f"{ds['name']}.datasource"
+        ds_file = generate_datafile(
+            content,
+            filename=filename,
+            data=None,
+            _format="ndjson",
+            force=True,
+            folder=project.folder,
         )
-        llm = LLM(user_token=user_token, host=tb_client.host)
-        result = llm.ask(system_prompt=create_prompt(resources_xml), prompt=prompt)
-        result = extract_xml(result, "response")
-        resources = parse_xml(result, "resource")
-        datasources = []
-        pipes = []
-        connections = []
-        for resource_xml in resources:
-            resource_type = extract_xml(resource_xml, "type")
-            name = extract_xml(resource_xml, "name")
-            content = extract_xml(resource_xml, "content")
-            resource = {
-                "name": name,
-                "content": content,
-            }
-            if resource_type.lower() == "datasource":
-                datasources.append(resource)
-            elif resource_type.lower() == "pipe":
-                pipes.append(resource)
-            elif resource_type.lower() == "connection":
-                connections.append(resource)
-
-        for ds in datasources:
-            content = ds["content"].replace("```", "")
-            filename = f"{ds['name']}.datasource"
-            generate_datafile(
-                content,
-                filename=filename,
-                data=None,
-                _format="ndjson",
-                force=True,
-                folder=project.folder,
-            )
-            created_any_resource = True
+        result.append(ds_file)
+    for pipe in pipes:
+        content = pipe["content"].replace("```", "")
+        pipe_file = generate_pipe_file(pipe["name"], content, project.folder)
+        result.append(pipe_file)
 
-        for pipe in pipes:
-            content = pipe["content"].replace("```", "")
-            generate_pipe_file(pipe["name"], content, project.folder)
-            created_any_resource = True
+    for conn in connections:
+        content = conn["content"].replace("```", "")
+        filename = f"{conn['name']}.connection"
+        conn_file = generate_connection_file(conn["name"], content, project.folder)
+        result.append(conn_file)
 
-        for conn in connections:
-            content = conn["content"].replace("```", "")
-            filename = f"{conn['name']}.connection"
-            generate_connection_file(conn["name"], content, project.folder)
-            created_any_resource = True
-
-    return result, created_any_resource
+    return result
 
 
 def init_git(folder: str):
@@ -454,3 +445,32 @@ def get_resources_xml(project: Project) -> str:
         ]
     )
     return resources_xml
+
+
+async def create_resources_from_data(
+    data: str,
+    project: Project,
+    config: Dict[str, Any],
+) -> List[Path]:
+    folder_path = project.path
+    local_client = await get_tinybird_local_client(config)
+    path = folder_path / data
+    result: List[Path] = []
+    format = path.suffix.lstrip(".")
+    ds_file = await _generate_datafile(str(path), local_client, format=format, force=True)
+    result.append(ds_file)
+    name = ds_file.stem
+    no_pipes = len(project.get_pipe_files()) == 0
+    if no_pipes:
+        pipe_file = generate_pipe_file(
+            f"{name}_endpoint",
+            f"""
+    NODE endpoint
+    SQL >
+    SELECT * from {name}
+    TYPE ENDPOINT
+            """,
+            project.folder,
+        )
+        result.append(pipe_file)
+    return result

{tinybird-0.0.1.dev175 → tinybird-0.0.1.dev176}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev175
+Version: 0.0.1.dev176
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -14,7 +14,7 @@ Requires-Dist: click<8.2,>=8.1.6
 Requires-Dist: clickhouse-toolset==0.34.dev0
 Requires-Dist: colorama==0.4.6
 Requires-Dist: cryptography~=41.0.0
-Requires-Dist: croniter==1.3.8
+Requires-Dist: croniter==1.3.15
 Requires-Dist: docker==7.1.0
 Requires-Dist: GitPython~=3.1.32
 Requires-Dist: humanfriendly~=8.2

{tinybird-0.0.1.dev175 → tinybird-0.0.1.dev176}/tinybird.egg-info/requires.txt

@@ -5,7 +5,7 @@ click<8.2,>=8.1.6
 clickhouse-toolset==0.34.dev0
 colorama==0.4.6
 cryptography~=41.0.0
-croniter==1.3.8
+croniter==1.3.15
 docker==7.1.0
 GitPython~=3.1.32
 humanfriendly~=8.2