tinybird 0.0.1.dev151__tar.gz → 0.0.1.dev152__tar.gz

{tinybird-0.0.1.dev151 → tinybird-0.0.1.dev152}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev151
+Version: 0.0.1.dev152
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev151 → tinybird-0.0.1.dev152}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev151'
-__revision__ = 'd9da445'
+__version__ = '0.0.1.dev152'
+__revision__ = '100f32b'

{tinybird-0.0.1.dev151 → tinybird-0.0.1.dev152}/tinybird/tb/modules/create.py

@@ -1,3 +1,4 @@
+import glob
 import os
 import re
 from pathlib import Path
@@ -5,7 +6,7 @@ from typing import Any, Dict, Optional, Tuple
 
 import click
 
-from tinybird.prompts import create_prompt, mock_prompt, readme_prompt, rules_prompt
+from tinybird.prompts import create_prompt, readme_prompt, rules_prompt
 from tinybird.tb.client import TinyB
 from tinybird.tb.modules.cicd import init_cicd
 from tinybird.tb.modules.cli import cli
@@ -17,6 +18,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.llm import LLM
 from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
 from tinybird.tb.modules.local_common import get_tinybird_local_client
+from tinybird.tb.modules.mock import create_mock_data
 from tinybird.tb.modules.project import Project
 
 
@@ -83,7 +85,7 @@ async def create(
 
         if data or (prompt and user_token):
             click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
-            result, created_something = await create_resources(tb_client, user_token, data, prompt, folder, ctx_config)
+            result, created_something = await create_resources(tb_client, user_token, data, prompt, project, ctx_config)
             click.echo(FeedbackManager.success(message="✓ Done!\n"))
             if prompt:
                 readme_path = Path(root_folder) / "README.md"
@@ -132,22 +134,26 @@ async def create(
                 persist_fixture(ds_name, data_content, folder, format=data_format)
                 created_something = True
             elif prompt and user_token:
-                datasource_files = [f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")]
-                for datasource_file in datasource_files:
-                    datasource_path = Path(folder) / "datasources" / datasource_file
-                    llm = LLM(user_token=user_token, host=tb_client.host)
+                for datasource_file in project.get_datasource_files():
+                    datasource_path = Path(datasource_file)
                     datasource_name = datasource_path.stem
                     datasource_content = datasource_path.read_text()
                     has_json_path = "`json:" in datasource_content
                     if has_json_path:
-                        prompt = f"<datasource_schema>{datasource_content}</datasource_schema>"
-                        response = llm.ask(system_prompt=mock_prompt(rows), prompt=prompt)
-                        sql = extract_xml(response, "sql")
-                        sql = sql.split("FORMAT")[0]
-                        query_result = await tb_client.query(f"{sql} FORMAT JSON")
-                        data = query_result.get("data", [])
-                        if data:
-                            persist_fixture(datasource_name, data, folder, format="ndjson")
+                        mock_data = await create_mock_data(
+                            datasource_name,
+                            datasource_content,
+                            rows,
+                            prompt,
+                            config,
+                            ctx_config,
+                            user_token,
+                            tb_client,
+                            format_="ndjson",
+                            folder=project.folder,
+                        )
+                        if mock_data:
+                            persist_fixture(datasource_name, mock_data, folder, format="ndjson")
                             click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}"))
                             created_something = True
 
@@ -204,12 +210,12 @@ async def create_resources(
     user_token: Optional[str],
     data: Optional[str],
     prompt: Optional[str],
-    folder: str,
+    project: Project,
     config: Dict[str, Any],
 ) -> Tuple[str, bool]:
     result = ""
     created_any_resource = False
-    folder_path = Path(folder)
+    folder_path = project.path
     if data:
         local_client = await get_tinybird_local_client(config)
         path = folder_path / data
@@ -224,7 +230,7 @@ SQL >
     SELECT * from {name}
 TYPE ENDPOINT
             """,
-            folder,
+            project.folder,
         )
         result = (
             f"<response><resource><type>datasource</type><name>{name}</name><content></content></resource></response>"
@@ -232,19 +238,9 @@ TYPE ENDPOINT
         created_any_resource = True
 
     elif prompt and user_token:
-        datasource_paths = [
-            Path(folder) / "datasources" / f
-            for f in os.listdir(Path(folder) / "datasources")
-            if f.endswith(".datasource")
-        ]
-        pipes_paths = [
-            Path(folder) / "endpoints" / f for f in os.listdir(Path(folder) / "endpoints") if f.endswith(".pipe")
-        ]
-        connections_paths = [
-            Path(folder) / "connections" / f
-            for f in os.listdir(Path(folder) / "connections")
-            if f.endswith(".connection")
-        ]
+        datasource_paths = [Path(ds_file) for ds_file in project.get_datasource_files()]
+        pipes_paths = [Path(pipe_file) for pipe_file in project.get_pipe_files()]
+        connections_paths = [Path(conn_file) for conn_file in project.get_connection_files()]
         resources_xml = "\n".join(
             [
                 f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
@@ -300,19 +296,19 @@ TYPE ENDPOINT
                 data=None,
                 _format="ndjson",
                 force=True,
-                folder=folder,
+                folder=project.folder,
             )
             created_any_resource = True
 
         for pipe in pipes:
             content = pipe["content"].replace("```", "")
-            generate_pipe_file(pipe["name"], content, folder)
+            generate_pipe_file(pipe["name"], content, project.folder)
             created_any_resource = True
 
         for conn in connections:
             content = conn["content"].replace("```", "")
             filename = f"{conn['name']}.connection"
-            generate_connection_file(conn["name"], content, folder)
+            generate_connection_file(conn["name"], content, project.folder)
             created_any_resource = True
 
     return result, created_any_resource
@@ -348,21 +344,25 @@ def generate_pipe_file(name: str, content: str, folder: str) -> Path:
     def is_endpoint(content: str) -> bool:
         return re.search(r"TYPE endpoint", content, re.IGNORECASE) is not None
 
-    if is_copy(content):
-        pathname = "copies"
-    elif is_materialization(content):
-        pathname = "materializations"
-    elif is_sink(content):
-        pathname = "sinks"
-    elif is_endpoint(content):
-        pathname = "endpoints"
+    already_exists = glob.glob(f"{folder}/**/{name}.pipe")
+    if already_exists:
+        f = Path(already_exists[0])
     else:
-        pathname = "pipes"
+        if is_copy(content):
+            pathname = "copies"
+        elif is_materialization(content):
+            pathname = "materializations"
+        elif is_sink(content):
+            pathname = "sinks"
+        elif is_endpoint(content):
+            pathname = "endpoints"
+        else:
+            pathname = "pipes"
 
-    base = Path(folder) / pathname
-    if not base.exists():
-        base.mkdir()
-    f = base / (f"{name}.pipe")
+        base = Path(folder) / pathname
+        if not base.exists():
+            base.mkdir()
+        f = base / (f"{name}.pipe")
     with open(f"{f}", "w") as file:
         file.write(content)
     click.echo(FeedbackManager.info_file_created(file=f.relative_to(folder)))
@@ -370,10 +370,14 @@ def generate_pipe_file(name: str, content: str, folder: str) -> Path:
 
 
 def generate_connection_file(name: str, content: str, folder: str, skip_feedback: bool = False) -> Path:
-    base = Path(folder) / "connections"
-    if not base.exists():
-        base.mkdir()
-    f = base / (f"{name}.connection")
+    already_exists = glob.glob(f"{folder}/**/{name}.connection")
+    if already_exists:
+        f = Path(already_exists[0])
+    else:
+        base = Path(folder) / "connections"
+        if not base.exists():
+            base.mkdir()
+        f = base / (f"{name}.connection")
     with open(f"{f}", "w") as file:
         file.write(content)
     if not skip_feedback:

{tinybird-0.0.1.dev151 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datafile/common.py

@@ -846,6 +846,14 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
                     lineno=line,
                     pos=pos,
                 )
+            if detected_type in ("Int", "UInt"):
+                t = detected_type
+                raise SchemaSyntaxError(
+                    message=f"Precision is mandatory for {t} types",
+                    hint=f"Hint: use one of {t}8, {t}16, {t}32, {t}64, {t}128, {t}256",
+                    lineno=line,
+                    pos=pos - len(t),
+                )
 
             try:
                 # Imported in the body to be compatible with the CLI

{tinybird-0.0.1.dev151 → tinybird-0.0.1.dev152}/tinybird/tb/modules/mock.py

@@ -1,5 +1,6 @@
 import glob
 from pathlib import Path
+from typing import Any, Dict, List
 
 import click
 
@@ -65,41 +66,23 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, form
 
         datasource_content = datasource_path.read_text()
         config = CLIConfig.get_project_config()
-        user_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
         user_token = ctx_config.get("user_token")
 
         if not user_token:
             raise Exception("This action requires authentication. Run 'tb login' first.")
 
-        llm = LLM(user_token=user_token, host=user_client.host)
-        prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
-        sql = ""
-        attempts = 0
-        data = []
-        error = ""
-        sql_path = None
-        while True:
-            try:
-                response = llm.ask(system_prompt=mock_prompt(rows, error), prompt=prompt)
-                sql = extract_xml(response, "sql")
-                sql_path = persist_fixture_sql(datasource_name, sql, folder)
-                sql_format = "JSON" if format_ == "ndjson" else "CSV"
-                result = await tb_client.query(f"SELECT * FROM ({sql}) LIMIT {rows} FORMAT {sql_format}")
-                if sql_format == "JSON":
-                    data = result.get("data", [])[:rows]
-                    error_response = result.get("error", None)
-                    if error_response:
-                        raise Exception(error_response)
-                else:
-                    data = result
-                break
-            except Exception as e:
-                error = str(e)
-                attempts += 1
-                if attempts > 5:
-                    raise Exception(f"Failed to generate a valid solution. Check {str(sql_path)} and try again.")
-                else:
-                    continue
+        data = await create_mock_data(
+            datasource_name,
+            datasource_content,
+            rows,
+            prompt,
+            config,
+            ctx_config,
+            user_token,
+            tb_client,
+            format_,
+            folder,
+        )
 
         fixture_path = persist_fixture(datasource_name, data, folder, format=format_)
         click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}.{format_} created"))
@@ -125,3 +108,48 @@ async def append_fixture(
         concurrency=1,
         silent=True,
     )
+
+
+async def create_mock_data(
+    datasource_name: str,
+    datasource_content: str,
+    rows: int,
+    prompt: str,
+    config: CLIConfig,
+    ctx_config: Dict[str, Any],
+    user_token: str,
+    tb_client: TinyB,
+    format_: str,
+    folder: str,
+) -> List[Dict[str, Any]]:
+    user_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
+    llm = LLM(user_token=user_token, host=user_client.host)
+    prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
+    sql = ""
+    attempts = 0
+    data = []
+    error = ""
+    sql_path = None
+    while True:
+        try:
+            response = llm.ask(system_prompt=mock_prompt(rows, error), prompt=prompt)
+            sql = extract_xml(response, "sql")
+            sql_path = persist_fixture_sql(datasource_name, sql, folder)
+            sql_format = "JSON" if format_ == "ndjson" else "CSV"
+            result = await tb_client.query(f"SELECT * FROM ({sql}) LIMIT {rows} FORMAT {sql_format}")
+            if sql_format == "JSON":
+                data = result.get("data", [])[:rows]
+                error_response = result.get("error", None)
+                if error_response:
+                    raise Exception(error_response)
+            else:
+                data = result
+            break
+        except Exception as e:
+            error = str(e)
+            attempts += 1
+            if attempts > 5:
+                raise Exception(f"Failed to generate a valid solution. Check {str(sql_path)} and try again.")
+            else:
+                continue
+    return data

{tinybird-0.0.1.dev151 → tinybird-0.0.1.dev152}/tinybird/tb/modules/project.py

@@ -55,6 +55,15 @@ class Project:
     def connections(self) -> List[str]:
         return sorted([Path(f).stem for f in glob.glob(f"{self.path}/**/*.connection", recursive=False)])
 
+    def get_datasource_files(self) -> List[str]:
+        return glob.glob(f"{self.path}/**/*.datasource", recursive=False)
+
+    def get_pipe_files(self) -> List[str]:
+        return glob.glob(f"{self.path}/**/*.pipe", recursive=False)
+
+    def get_connection_files(self) -> List[str]:
+        return glob.glob(f"{self.path}/**/*.connection", recursive=False)
+
     def get_pipe_datafile(self, filename: str) -> Optional[Datafile]:
         try:
             return parse_pipe(filename).datafile

{tinybird-0.0.1.dev151 → tinybird-0.0.1.dev152}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev151
+Version: 0.0.1.dev152
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird