tinybird 0.0.1.dev123__tar.gz → 0.0.1.dev125__tar.gz

{tinybird-0.0.1.dev123 → tinybird-0.0.1.dev125}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev123
+Version: 0.0.1.dev125
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev123 → tinybird-0.0.1.dev125}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev123'
-__revision__ = '01f52fe'
+__version__ = '0.0.1.dev125'
+__revision__ = '57c6bd7'

{tinybird-0.0.1.dev123 → tinybird-0.0.1.dev125}/tinybird/tb/config.py

@@ -65,6 +65,10 @@ async def get_config(
     config["host"] = host or config.get("host", DEFAULT_API_HOST)
     config["workspaces"] = config.get("workspaces", [])
     config["cwd"] = config.get("cwd", getcwd())
+    config["user_email"] = config.get("user_email", None)
+    config["user_id"] = config.get("user_id", None)
+    config["workspace_id"] = config.get("id", None)
+    config["workspace_name"] = config.get("name", None)
     return config
 
 

{tinybird-0.0.1.dev123 → tinybird-0.0.1.dev125}/tinybird/tb/modules/build.py

@@ -20,7 +20,7 @@ from tinybird.tb.modules.common import push_data
 from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.datafile.build import folder_build
 from tinybird.tb.modules.datafile.exceptions import ParseException
-from tinybird.tb.modules.datafile.fixture import get_fixture_dir, persist_fixture
+from tinybird.tb.modules.datafile.fixture import FixtureExtension, get_fixture_dir, persist_fixture
 from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
 from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 from tinybird.tb.modules.datafile.playground import folder_playground
@@ -145,8 +145,21 @@ def build_project(project: Project, tb_client: TinyB, file_changed: Optional[str
                         ds_path = Path(filename)
                         ds_name = ds_path.stem
                         fixture_folder = get_fixture_dir(project.folder)
-                        fixture_path = fixture_folder / f"{ds_name}.ndjson"
-                        if fixture_path.exists():
+                        fixture_extensions = [FixtureExtension.NDJSON, FixtureExtension.CSV]
+                        fixture_path = next(
+                            (
+                                fixture_folder / f"{ds_name}{ext}"
+                                for ext in fixture_extensions
+                                if (fixture_folder / f"{ds_name}{ext}").exists()
+                            ),
+                            None,
+                        )
+                        if not fixture_path:
+                            sql_path = fixture_folder / f"{ds_name}.sql"
+                            if sql_path.exists():
+                                fixture_path = rebuild_fixture_sql(project, tb_client, str(sql_path))
+
+                        if fixture_path:
                             append_fixture(tb_client, ds_name, str(fixture_path))
 
             except Exception as e:
@@ -252,7 +265,7 @@ def process(
 ) -> None:
     time_start = time.time()
     build_failed = False
-    if file_changed and file_changed.endswith(".ndjson"):
+    if file_changed and (file_changed.endswith(FixtureExtension.NDJSON) or file_changed.endswith(FixtureExtension.CSV)):
         rebuild_fixture(project, tb_client, file_changed)
     elif file_changed and file_changed.endswith(".sql"):
         rebuild_fixture_sql(project, tb_client, file_changed)
@@ -295,13 +308,25 @@ def run_watch(
     shell.run()
 
 
-def rebuild_fixture_sql(project: Project, tb_client: TinyB, sql_file: str) -> None:
+def rebuild_fixture_sql(project: Project, tb_client: TinyB, sql_file: str) -> Path:
     sql_path = Path(sql_file)
     datasource_name = sql_path.stem
+    valid_extensions = [FixtureExtension.NDJSON, FixtureExtension.CSV]
+    fixtures_path = get_fixture_dir(project.folder)
+    current_fixture_path = next(
+        (
+            fixtures_path / f"{datasource_name}{extension}"
+            for extension in valid_extensions
+            if (fixtures_path / f"{datasource_name}{extension}").exists()
+        ),
+        None,
+    )
+    fixture_format = current_fixture_path.suffix.lstrip(".") if current_fixture_path else "ndjson"
     sql = sql_path.read_text()
-    result = asyncio.run(tb_client.query(f"{sql} FORMAT JSON"))
+    sql_format = "CSV" if fixture_format == "csv" else "JSON"
+    result = asyncio.run(tb_client.query(f"{sql} FORMAT {sql_format}"))
     data = result.get("data", [])
-    persist_fixture(datasource_name, data, project.folder)
+    return persist_fixture(datasource_name, data, project.folder, format=fixture_format)
 
 
 def is_vendor(f: Path) -> bool:

{tinybird-0.0.1.dev123 → tinybird-0.0.1.dev125}/tinybird/tb/modules/common.py

@@ -243,7 +243,7 @@ class CatchAuthExceptions(AliasedGroup):
         formatter.write_heading("Telemetry")
         formatter.write_text(
             """
-  Tinybird collects anonymous usage data and errors to improve the command
+  Tinybird collects usage data and errors to improve the command
 line experience. To opt-out, set TB_CLI_TELEMETRY_OPTOUT to '1' or 'true'."""
         )
         formatter.write_paragraph()

{tinybird-0.0.1.dev123 → tinybird-0.0.1.dev125}/tinybird/tb/modules/create.py

@@ -88,7 +88,8 @@ async def create(
             click.echo(FeedbackManager.success(message="✓ Scaffolding completed!\n"))
             created_something = True
         result = ""
-        if (data or prompt) and user_token:
+
+        if data or (prompt and user_token):
             click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
             result, created_something = await create_resources(
                 local_client, tb_client, user_token, data, prompt, folder
@@ -133,10 +134,12 @@ async def create(
 
             if data:
                 ds_name = os.path.basename(data.split(".")[0])
-                data_content = Path(data).read_text()
+                data_path = Path(data)
+                data_content = data_path.read_text()
+                data_format = data_path.suffix.lstrip(".")
                 datasource_path = Path(folder) / "datasources" / f"{ds_name}.datasource"
                 click.echo(FeedbackManager.info(message=f"✓ /fixtures/{ds_name}"))
-                persist_fixture(ds_name, data_content, folder)
+                persist_fixture(ds_name, data_content, folder, format=data_format)
                 created_something = True
             elif prompt and user_token:
                 datasource_files = [f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")]
@@ -154,7 +157,7 @@ async def create(
                         query_result = await local_client.query(f"{sql} FORMAT JSON")
                         data = query_result.get("data", [])
                         if data:
-                            persist_fixture(datasource_name, data, folder)
+                            persist_fixture(datasource_name, data, folder, format="ndjson")
                             click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}"))
                             created_something = True
 
@@ -221,7 +224,7 @@ async def create_resources(
         path = folder_path / data
         format = path.suffix.lstrip(".")
         await _generate_datafile(str(path), local_client, format=format, force=True)
-        name = data.split(".")[0]
+        name = path.stem
         generate_pipe_file(
             f"{name}_endpoint",
             f"""
@@ -354,7 +357,7 @@ def generate_pipe_file(name: str, content: str, folder: str) -> Path:
 
     base = Path(folder) / pathname
     if not base.exists():
-        base = Path()
+        base.mkdir()
     f = base / (f"{name}.pipe")
     with open(f"{f}", "w") as file:
         file.write(content)

{tinybird-0.0.1.dev123 → tinybird-0.0.1.dev125}/tinybird/tb/modules/datafile/fixture.py

@@ -17,10 +17,14 @@ def persist_fixture_sql(fixture_name: str, sql: str, folder: str) -> Path:
     return fixture_file
 
 
-def persist_fixture(fixture_name: str, data: Union[List[Dict[str, Any]], str], folder: str, format="ndjson") -> Path:
+def persist_fixture(fixture_name: str, data: Union[List[Dict[str, Any]], str], folder: str, format: str) -> Path:
     fixture_dir = get_fixture_dir(folder)
     fixture_file = fixture_dir / f"{fixture_name}.{format}"
-    fixture_file.write_text(data if isinstance(data, str) else format_data_to_ndjson(data))
+    extension = f".{format}"
+    if extension == FixtureExtension.NDJSON:
+        fixture_file.write_text(data if isinstance(data, str) else format_data_to_ndjson(data))
+    elif extension == FixtureExtension.CSV:
+        fixture_file.write_text(data if isinstance(data, str) else str(data))  ## this should not happen
     return fixture_file
 
 
@@ -34,3 +38,8 @@ def load_fixture(
     if not fixture_file.exists():
         return None
     return fixture_file
+
+
+class FixtureExtension:
+    NDJSON = ".ndjson"
+    CSV = ".csv"

{tinybird-0.0.1.dev123 → tinybird-0.0.1.dev125}/tinybird/tb/modules/datasource.py

@@ -445,7 +445,7 @@ async def datasource_export(
 
         res = await client.query(query)
 
-        fixture_path = persist_fixture(datasource, res, project.folder)
+        fixture_path = persist_fixture(datasource, res, project.folder, format=format_)
         file_size = os.path.getsize(fixture_path)
 
         click.echo(

{tinybird-0.0.1.dev123 → tinybird-0.0.1.dev125}/tinybird/tb/modules/mock.py

@@ -24,9 +24,16 @@ from tinybird.tb.modules.project import Project
     default="",
     help="Extra context to use for data generation",
 )
+@click.option(
+    "--format",
+    "format_",
+    type=click.Choice(["ndjson", "csv"], case_sensitive=False),
+    default="ndjson",
+    help="Format of the fixture to create",
+)
 @click.pass_context
 @coro
-async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str) -> None:
+async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, format_: str) -> None:
     """Generate sample data for a data source.
 
     Args:
@@ -79,13 +86,16 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str) -> N
                 response = llm.ask(system_prompt=mock_prompt(rows, error), prompt=prompt)
                 sql = extract_xml(response, "sql")
                 sql_path = persist_fixture_sql(datasource_name, sql, folder)
-                result = await tb_client.query(f"{sql} FORMAT JSON")
-                data = result.get("data", [])[:rows]
-                error_response = result.get("error", None)
-                if error_response:
-                    raise CLIException(error_response)
+                sql_format = "JSON" if format_ == "ndjson" else "CSV"
+                result = await tb_client.query(f"SELECT * FROM ({sql}) LIMIT {rows} FORMAT {sql_format}")
+                if sql_format == "JSON":
+                    data = result.get("data", [])[:rows]
+                    error_response = result.get("error", None)
+                    if error_response:
+                        raise CLIException(error_response)
                 else:
-                    break
+                    data = result
+                break
             except Exception as e:
                 error = str(e)
                 attempts += 1
@@ -94,8 +104,8 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str) -> N
                 else:
                     continue
 
-        fixture_path = persist_fixture(datasource_name, data, folder)
-        click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}.ndjson created"))
+        fixture_path = persist_fixture(datasource_name, data, folder, format=format_)
+        click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}.{format_} created"))
         if env == "cloud":
             await append_fixture(tb_client, datasource_name, str(fixture_path))
 

{tinybird-0.0.1.dev123 → tinybird-0.0.1.dev125}/tinybird/tb/modules/telemetry.py

@@ -1,4 +1,5 @@
 import functools
+import hashlib
 import json
 import os
 import platform
@@ -13,7 +14,8 @@ from urllib.parse import urlencode
 
 import requests
 
-from tinybird.tb.config import CURRENT_VERSION
+from tinybird.syncasync import async_to_sync
+from tinybird.tb.config import CURRENT_VERSION, get_config
 
 TELEMETRY_TIMEOUT: int = 1
 TELEMETRY_DATASOURCE: str = "tb_cli_telemetry"
@@ -80,7 +82,11 @@ def _hide_tokens(text: str) -> str:
 
 
 class TelemetryHelper:
-    def __init__(self, tb_host: Optional[str] = None, max_enqueued_events: int = 5) -> None:
+    def __init__(
+        self,
+        tb_host: Optional[str] = None,
+        max_enqueued_events: int = 5,
+    ) -> None:
         self.tb_host = tb_host or os.getenv("TB_CLI_TELEMETRY_HOST", "https://api.tinybird.co")
         self.max_enqueued_events: int = max_enqueued_events
 
@@ -112,6 +118,15 @@ class TelemetryHelper:
             self.log("Not sending events in local development mode")
             return
 
+        event_data["fingerprint_id"] = hashlib.sha256(platform.node().encode()).hexdigest()
+
+        config = async_to_sync(get_config)(None, None)
+        if config:
+            event_data["workspace_id"] = config.get("workspace_id", "")
+            event_data["workspace_name"] = config.get("workspace_name", "")
+            event_data["user_email"] = config.get("user_email", "")
+            event_data["user_id"] = config.get("user_id", "")
+
         # Let's save deep copies to not interfere with original objects
         event_dict: Dict[str, Any] = deepcopy(self._defaults)
         event_dict["event"] = event

{tinybird-0.0.1.dev123 → tinybird-0.0.1.dev125}/tinybird/tb/modules/watch.py

@@ -16,13 +16,14 @@ from watchdog.events import (
 from watchdog.observers import Observer
 
 from tinybird.tb.modules.datafile.common import Datafile, DatafileKind
+from tinybird.tb.modules.datafile.fixture import FixtureExtension
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.project import Project
 from tinybird.tb.modules.shell import Shell
 
 
 class WatchProjectHandler(PatternMatchingEventHandler):
-    valid_extensions = [".datasource", ".pipe", "connection", ".ndjson", ".sql"]
+    valid_extensions = [".datasource", ".pipe", "connection", FixtureExtension.CSV, FixtureExtension.NDJSON, ".sql"]
 
     def __init__(self, shell: Shell, project: Project, process: Callable):
         self.shell = shell
@@ -163,7 +164,9 @@ class FileChangeHandler(FileSystemEventHandler):
             is_vendor = "vendor/" in path
             if is_vendor:
                 return False
-            return any(path.endswith(ext) for ext in [".datasource", ".pipe", ".ndjson"])
+            return any(
+                path.endswith(ext) for ext in [".datasource", ".pipe", FixtureExtension.NDJSON, FixtureExtension.CSV]
+            )
 
         if should_process_path(event.src_path):
             return event.src_path

{tinybird-0.0.1.dev123 → tinybird-0.0.1.dev125}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev123
+Version: 0.0.1.dev125
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird