tinybird 0.0.1.dev124__py3-none-any.whl → 0.0.1.dev125__py3-none-any.whl

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev124'
-__revision__ = '325c83b'
+__version__ = '0.0.1.dev125'
+__revision__ = '57c6bd7'

tinybird/tb/modules/build.py

@@ -20,7 +20,7 @@ from tinybird.tb.modules.common import push_data
 from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.datafile.build import folder_build
 from tinybird.tb.modules.datafile.exceptions import ParseException
-from tinybird.tb.modules.datafile.fixture import get_fixture_dir, persist_fixture
+from tinybird.tb.modules.datafile.fixture import FixtureExtension, get_fixture_dir, persist_fixture
 from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
 from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 from tinybird.tb.modules.datafile.playground import folder_playground
@@ -145,8 +145,21 @@ def build_project(project: Project, tb_client: TinyB, file_changed: Optional[str
                         ds_path = Path(filename)
                         ds_name = ds_path.stem
                         fixture_folder = get_fixture_dir(project.folder)
-                        fixture_path = fixture_folder / f"{ds_name}.ndjson"
-                        if fixture_path.exists():
+                        fixture_extensions = [FixtureExtension.NDJSON, FixtureExtension.CSV]
+                        fixture_path = next(
+                            (
+                                fixture_folder / f"{ds_name}{ext}"
+                                for ext in fixture_extensions
+                                if (fixture_folder / f"{ds_name}{ext}").exists()
+                            ),
+                            None,
+                        )
+                        if not fixture_path:
+                            sql_path = fixture_folder / f"{ds_name}.sql"
+                            if sql_path.exists():
+                                fixture_path = rebuild_fixture_sql(project, tb_client, str(sql_path))
+
+                        if fixture_path:
                             append_fixture(tb_client, ds_name, str(fixture_path))
 
             except Exception as e:
@@ -252,7 +265,7 @@ def process(
 ) -> None:
     time_start = time.time()
     build_failed = False
-    if file_changed and file_changed.endswith(".ndjson"):
+    if file_changed and (file_changed.endswith(FixtureExtension.NDJSON) or file_changed.endswith(FixtureExtension.CSV)):
         rebuild_fixture(project, tb_client, file_changed)
     elif file_changed and file_changed.endswith(".sql"):
         rebuild_fixture_sql(project, tb_client, file_changed)
@@ -295,13 +308,25 @@ def run_watch(
     shell.run()
 
 
-def rebuild_fixture_sql(project: Project, tb_client: TinyB, sql_file: str) -> None:
+def rebuild_fixture_sql(project: Project, tb_client: TinyB, sql_file: str) -> Path:
     sql_path = Path(sql_file)
     datasource_name = sql_path.stem
+    valid_extensions = [FixtureExtension.NDJSON, FixtureExtension.CSV]
+    fixtures_path = get_fixture_dir(project.folder)
+    current_fixture_path = next(
+        (
+            fixtures_path / f"{datasource_name}{extension}"
+            for extension in valid_extensions
+            if (fixtures_path / f"{datasource_name}{extension}").exists()
+        ),
+        None,
+    )
+    fixture_format = current_fixture_path.suffix.lstrip(".") if current_fixture_path else "ndjson"
     sql = sql_path.read_text()
-    result = asyncio.run(tb_client.query(f"{sql} FORMAT JSON"))
+    sql_format = "CSV" if fixture_format == "csv" else "JSON"
+    result = asyncio.run(tb_client.query(f"{sql} FORMAT {sql_format}"))
     data = result.get("data", [])
-    persist_fixture(datasource_name, data, project.folder)
+    return persist_fixture(datasource_name, data, project.folder, format=fixture_format)
 
 
 def is_vendor(f: Path) -> bool:

tinybird/tb/modules/create.py

@@ -134,10 +134,12 @@ async def create(
 
             if data:
                 ds_name = os.path.basename(data.split(".")[0])
-                data_content = Path(data).read_text()
+                data_path = Path(data)
+                data_content = data_path.read_text()
+                data_format = data_path.suffix.lstrip(".")
                 datasource_path = Path(folder) / "datasources" / f"{ds_name}.datasource"
                 click.echo(FeedbackManager.info(message=f"✓ /fixtures/{ds_name}"))
-                persist_fixture(ds_name, data_content, folder)
+                persist_fixture(ds_name, data_content, folder, format=data_format)
                 created_something = True
             elif prompt and user_token:
                 datasource_files = [f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")]
@@ -155,7 +157,7 @@ async def create(
                         query_result = await local_client.query(f"{sql} FORMAT JSON")
                         data = query_result.get("data", [])
                         if data:
-                            persist_fixture(datasource_name, data, folder)
+                            persist_fixture(datasource_name, data, folder, format="ndjson")
                             click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}"))
                             created_something = True
 
@@ -222,7 +224,7 @@ async def create_resources(
         path = folder_path / data
         format = path.suffix.lstrip(".")
         await _generate_datafile(str(path), local_client, format=format, force=True)
-        name = data.split(".")[0]
+        name = path.stem
         generate_pipe_file(
             f"{name}_endpoint",
             f"""
@@ -355,7 +357,7 @@ def generate_pipe_file(name: str, content: str, folder: str) -> Path:
 
     base = Path(folder) / pathname
     if not base.exists():
-        base = Path()
+        base.mkdir()
     f = base / (f"{name}.pipe")
     with open(f"{f}", "w") as file:
         file.write(content)

tinybird/tb/modules/datafile/fixture.py

@@ -17,10 +17,14 @@ def persist_fixture_sql(fixture_name: str, sql: str, folder: str) -> Path:
     return fixture_file
 
 
-def persist_fixture(fixture_name: str, data: Union[List[Dict[str, Any]], str], folder: str, format="ndjson") -> Path:
+def persist_fixture(fixture_name: str, data: Union[List[Dict[str, Any]], str], folder: str, format: str) -> Path:
     fixture_dir = get_fixture_dir(folder)
     fixture_file = fixture_dir / f"{fixture_name}.{format}"
-    fixture_file.write_text(data if isinstance(data, str) else format_data_to_ndjson(data))
+    extension = f".{format}"
+    if extension == FixtureExtension.NDJSON:
+        fixture_file.write_text(data if isinstance(data, str) else format_data_to_ndjson(data))
+    elif extension == FixtureExtension.CSV:
+        fixture_file.write_text(data if isinstance(data, str) else str(data))  ## this should not happen
     return fixture_file
 
 
@@ -34,3 +38,8 @@ def load_fixture(
     if not fixture_file.exists():
         return None
     return fixture_file
+
+
+class FixtureExtension:
+    NDJSON = ".ndjson"
+    CSV = ".csv"

tinybird/tb/modules/datasource.py

@@ -445,7 +445,7 @@ async def datasource_export(
 
         res = await client.query(query)
 
-        fixture_path = persist_fixture(datasource, res, project.folder)
+        fixture_path = persist_fixture(datasource, res, project.folder, format=format_)
         file_size = os.path.getsize(fixture_path)
 
         click.echo(

tinybird/tb/modules/mock.py

@@ -24,9 +24,16 @@ from tinybird.tb.modules.project import Project
     default="",
     help="Extra context to use for data generation",
 )
+@click.option(
+    "--format",
+    "format_",
+    type=click.Choice(["ndjson", "csv"], case_sensitive=False),
+    default="ndjson",
+    help="Format of the fixture to create",
+)
 @click.pass_context
 @coro
-async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str) -> None:
+async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, format_: str) -> None:
     """Generate sample data for a data source.
 
     Args:
@@ -79,13 +86,16 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str) -> N
                 response = llm.ask(system_prompt=mock_prompt(rows, error), prompt=prompt)
                 sql = extract_xml(response, "sql")
                 sql_path = persist_fixture_sql(datasource_name, sql, folder)
-                result = await tb_client.query(f"{sql} FORMAT JSON")
-                data = result.get("data", [])[:rows]
-                error_response = result.get("error", None)
-                if error_response:
-                    raise CLIException(error_response)
+                sql_format = "JSON" if format_ == "ndjson" else "CSV"
+                result = await tb_client.query(f"SELECT * FROM ({sql}) LIMIT {rows} FORMAT {sql_format}")
+                if sql_format == "JSON":
+                    data = result.get("data", [])[:rows]
+                    error_response = result.get("error", None)
+                    if error_response:
+                        raise CLIException(error_response)
                 else:
-                    break
+                    data = result
+                break
             except Exception as e:
                 error = str(e)
                 attempts += 1
@@ -94,8 +104,8 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str) -> N
                 else:
                     continue
 
-        fixture_path = persist_fixture(datasource_name, data, folder)
-        click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}.ndjson created"))
+        fixture_path = persist_fixture(datasource_name, data, folder, format=format_)
+        click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}.{format_} created"))
         if env == "cloud":
             await append_fixture(tb_client, datasource_name, str(fixture_path))
 

tinybird/tb/modules/watch.py

@@ -16,13 +16,14 @@ from watchdog.events import (
 from watchdog.observers import Observer
 
 from tinybird.tb.modules.datafile.common import Datafile, DatafileKind
+from tinybird.tb.modules.datafile.fixture import FixtureExtension
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.project import Project
 from tinybird.tb.modules.shell import Shell
 
 
 class WatchProjectHandler(PatternMatchingEventHandler):
-    valid_extensions = [".datasource", ".pipe", "connection", ".ndjson", ".sql"]
+    valid_extensions = [".datasource", ".pipe", "connection", FixtureExtension.CSV, FixtureExtension.NDJSON, ".sql"]
 
     def __init__(self, shell: Shell, project: Project, process: Callable):
         self.shell = shell
@@ -163,7 +164,9 @@ class FileChangeHandler(FileSystemEventHandler):
             is_vendor = "vendor/" in path
             if is_vendor:
                 return False
-            return any(path.endswith(ext) for ext in [".datasource", ".pipe", ".ndjson"])
+            return any(
+                path.endswith(ext) for ext in [".datasource", ".pipe", FixtureExtension.NDJSON, FixtureExtension.CSV]
+            )
 
         if should_process_path(event.src_path):
             return event.src_path

{tinybird-0.0.1.dev124.dist-info → tinybird-0.0.1.dev125.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev124
+Version: 0.0.1.dev125
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev124.dist-info → tinybird-0.0.1.dev125.dist-info}/RECORD

@@ -12,20 +12,20 @@ tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
 tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
 tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
 tinybird/ch_utils/engine.py,sha256=BZuPM7MFS7vaEKK5tOMR2bwSAgJudPrJt27uVEwZmTY,40512
-tinybird/tb/__cli__.py,sha256=yRgIZndvTr1rfUOaK4rUcrgoA1LsLCDprVp8u7BjOKs,252
+tinybird/tb/__cli__.py,sha256=dV409bozEIxzsWY-MNKKruKjTqMQqrXLEV7AQlZNHEE,252
 tinybird/tb/cli.py,sha256=uDLwcbwSJfVFw-pceijZJqaq26z5jNsey0QaUGFjt7w,1097
 tinybird/tb/client.py,sha256=oa0pR46WvzDoqpyaDU-lRu33I0cZhtZVDqDQz2RHMWQ,56085
 tinybird/tb/config.py,sha256=iIia7F67UI9p51Ja4jALonyttIi4eH7PwOn3HvcvOb8,4008
 tinybird/tb/modules/auth.py,sha256=_OeYnmTH83lnqCgQEdS6K0bx1KBUeRmZk2M7JnRmWpk,9037
-tinybird/tb/modules/build.py,sha256=9O6SzOSe2kPWFSjvSYlBV17Db14MeVQpHY952knArx0,15932
+tinybird/tb/modules/build.py,sha256=b1fjWrurDWLwNdWjcNaEaSJh6Iko98uSExw4k_CRoTc,17244
 tinybird/tb/modules/cicd.py,sha256=A7zJZF9HkJ6NPokplgNjmefMrpUlRbFxBbjMZhq5OTI,7110
 tinybird/tb/modules/cli.py,sha256=Y_5hu9xwyTIZw4bQoe0MYLnRIzmR7hUjql_oZBxd4Qg,13407
 tinybird/tb/modules/common.py,sha256=bdKNjHtQKcsZFyZIsLY8vEoTD0Xf-1UNBnh1pAyWPZU,83456
 tinybird/tb/modules/config.py,sha256=ziqW_t_mRVvWOd85VoB4vKyvgMkEfpXDf9H4v38p2xc,11422
 tinybird/tb/modules/connection.py,sha256=ctrcZCksfBKJHv3pMoDzsmgmx-rlkfeyn2GkxM0Zlw0,6892
 tinybird/tb/modules/copy.py,sha256=2Mm4FWKehOG7CoOhiF1m9UZJgJn0W1_cMolqju8ONYg,5805
-tinybird/tb/modules/create.py,sha256=CGQBcHk6HVUguCEZ_7VHgnfEOkB38kuC0YvzuUufU1k,16577
-tinybird/tb/modules/datasource.py,sha256=-aQCK_-wEmoo92Ki7zr2wm424RchDgWvT6yhfFY5kko,16909
+tinybird/tb/modules/create.py,sha256=PDAuzeBiqN9q7GY2p4zcMWDojdNjz6_aJgI5dxR-FVM,16701
+tinybird/tb/modules/datasource.py,sha256=XvY7iTRefG9iEBLQkqBSF47bK-CrLu2AzDnigGI2mSc,16925
 tinybird/tb/modules/deployment.py,sha256=4Zt7jPbqt18fB5kPx7DbO91Bh6xzBBTEUFY7O89shuU,19560
 tinybird/tb/modules/endpoint.py,sha256=XySDt3pk66vxOZ0egUfz4bY8bEk3BjOXkv-L0OIJ3sc,12083
 tinybird/tb/modules/exceptions.py,sha256=4A2sSjCEqKUMqpP3WI00zouCWW4uLaghXXLZBSw04mY,3363
@@ -40,7 +40,7 @@ tinybird/tb/modules/local_common.py,sha256=DqTHHcq4JitohYvHiqCz2Av4REPhmwwS6naQT
 tinybird/tb/modules/login.py,sha256=rLYdV7MLCTT1F4VBAWGv6SO9efMiM9-977ayLecUzjI,6842
 tinybird/tb/modules/logout.py,sha256=ULooy1cDBD02-r7voZmhV7udA0ML5tVuflJyShrh56Y,1022
 tinybird/tb/modules/materialization.py,sha256=yIns8ypdFVLWwuCcvAZPBChsuJl2DIxJe6M8UCBHNsU,5752
-tinybird/tb/modules/mock.py,sha256=Z_1nYMO8mmjZkBjikqHNqSd4ssdmcfaXUqIh8jY-z6o,4519
+tinybird/tb/modules/mock.py,sha256=KScG2WohfCjxakBS2Rfb4-7QKS4OSkoVgiMyUvkCWWE,4924
 tinybird/tb/modules/open.py,sha256=OuctINN77oexpSjth9uoIZPCelKO4Li-yyVxeSnk1io,1371
 tinybird/tb/modules/pipe.py,sha256=AQKEDagO6e3psPVjJkS_MDbn8aK-apAiLp26k7jgAV0,2432
 tinybird/tb/modules/project.py,sha256=RdVOzJiuFPqkCzPhzGetbgcFeiMEn9IMviFCG58XKgI,3142
@@ -52,7 +52,7 @@ tinybird/tb/modules/tag.py,sha256=anPmMUBc-TbFovlpFi8GPkKA18y7Y0GczMsMms5TZsU,35
 tinybird/tb/modules/telemetry.py,sha256=C-aEXY3yQMHzFFySwefwyigprcmjoz_CiCXv17jpCPA,11001
 tinybird/tb/modules/test.py,sha256=HxkSpdLOW8brGUNsqdBoSW8RixxyC7OCDmYwEZsIpfs,11459
 tinybird/tb/modules/token.py,sha256=2fmKwu10_M0pqs6YmJVeILR9ZQB0ejRAET86agASbKM,13488
-tinybird/tb/modules/watch.py,sha256=poNJOUNDESDNn80H2dHvE6X6pIu-t9MZFi59_TxVN2U,8822
+tinybird/tb/modules/watch.py,sha256=-2DGSShUGAPDYpLVkiDiONC-RqGdZbTiXOCsDtU5udw,8990
 tinybird/tb/modules/workspace.py,sha256=SlK08psp0tu5t_URHyIczm696buW6KD6FPs9Lg1aNRE,6614
 tinybird/tb/modules/workspace_members.py,sha256=RYLpyPM1ECCasHRg3uvpckzXplX0_KgNFsSPZn_i6qk,8744
 tinybird/tb/modules/datafile/build.py,sha256=d_h3pRFDPFrDKGhpFx2iejY25GuB2k8yfNouj6s8caw,50973
@@ -62,7 +62,7 @@ tinybird/tb/modules/datafile/build_pipe.py,sha256=6Cwjf3BKEF3-oQ9PipsQfK-Z43nSwt
 tinybird/tb/modules/datafile/common.py,sha256=FXwHfCknNgZnMGaiju4u7ODYnXticP5cVagxor5F7Dw,85442
 tinybird/tb/modules/datafile/diff.py,sha256=MTmj53RYjER4neLgWVjabn-FKVFgh8h8uYiBo55lFQg,6757
 tinybird/tb/modules/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
-tinybird/tb/modules/datafile/fixture.py,sha256=V3WGfPLIR78el3oCNWNkySWs6LxIufyIM0mDrrT3aWc,1131
+tinybird/tb/modules/datafile/fixture.py,sha256=DrRWivcvo_1rn7LlVUnHcXccdgx9yVj63mzBkUwCzk8,1420
 tinybird/tb/modules/datafile/format_common.py,sha256=WaNV4tXrQU5gjV6MJP-5TGqg_Bre6ilNS8emvFl-X3c,1967
 tinybird/tb/modules/datafile/format_datasource.py,sha256=iWbeXruxC7OBmjNgurWt6ymcJlYzxfKwkGnhpcoSKEo,6190
 tinybird/tb/modules/datafile/format_pipe.py,sha256=DUGdmlzI146YDqTwW-7kSIOXocz4AH-md_LFGUm9hrc,7436
@@ -79,8 +79,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev124.dist-info/METADATA,sha256=0B_zlu-C3jSAtHgjGRjlFZks5aVHgZ1I0zlGhMa2BZE,1612
-tinybird-0.0.1.dev124.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev124.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev124.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev124.dist-info/RECORD,,
+tinybird-0.0.1.dev125.dist-info/METADATA,sha256=Com9G_kAdU7yninwwz7WnOYy83nfiV9mok_rKuyXsUU,1612
+tinybird-0.0.1.dev125.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev125.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev125.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev125.dist-info/RECORD,,