tinybird 0.0.1.dev6__py3-none-any.whl → 0.0.1.dev7__py3-none-any.whl

tinybird/tb/modules/datafile/pull.py

@@ -0,0 +1,157 @@
+import re
+from asyncio import Semaphore, gather
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import aiofiles
+import click
+
+from tinybird.client import AuthNoTokenException, TinyB
+from tinybird.feedback_manager import FeedbackManager
+from tinybird.tb.modules.datafile.common import get_name_version
+from tinybird.tb.modules.datafile.format_datasource import format_datasource
+from tinybird.tb.modules.datafile.format_pipe import format_pipe
+
+
+async def folder_pull(
+    client: TinyB,
+    folder: str,
+    auto: bool,
+    match: Optional[str],
+    force: bool,
+    verbose: bool = True,
+    progress_bar: bool = False,
+    fmt: bool = False,
+):
+    pattern = re.compile(match) if match else None
+
+    def _get_latest_versions(resources: List[str]):
+        versions: Dict[str, Any] = {}
+
+        for x in resources:
+            t = get_name_version(x)
+            t["original_name"] = x
+            if t["version"] is None:
+                t["version"] = -1
+            name = t["name"]
+
+            if name not in versions or name == x or versions[name]["version"] < t["version"]:
+                versions[name] = t
+        return versions
+
+    def get_file_folder(extension: str):
+        if not auto:
+            return None
+        if extension == "datasource":
+            return "datasources"
+        if extension == "pipe":
+            return "pipes"
+        if extension == "token":
+            return "tokens"
+        return None
+
+    async def write_files(
+        versions: Dict[str, Any],
+        resources: List[str],
+        extension: str,
+        get_resource_function: str,
+        progress_bar: bool = False,
+        fmt: bool = False,
+    ):
+        async def write_resource(k: Dict[str, Any]):
+            name = f"{k['name']}.{extension}"
+            try:
+                if pattern and not pattern.search(name):
+                    if verbose:
+                        click.echo(FeedbackManager.info_skipping_resource(resource=name))
+                    return
+
+                resource = await getattr(client, get_resource_function)(k["original_name"])
+                resource_to_write = resource
+
+                if fmt:
+                    if extension == "datasource":
+                        resource_to_write = await format_datasource(name, content=resource)
+                    elif extension == "pipe":
+                        resource_to_write = await format_pipe(name, content=resource)
+
+                dest_folder = folder
+                if "." in k["name"] and extension != "token":
+                    dest_folder = Path(folder) / "vendor" / k["name"].split(".", 1)[0]
+                    name = f"{k['name'].split('.', 1)[1]}.{extension}"
+
+                file_folder = get_file_folder(extension)
+                f = Path(dest_folder) / file_folder if file_folder is not None else Path(dest_folder)
+
+                if not f.exists():
+                    f.mkdir(parents=True)
+
+                f = f / name
+                resource_names = [x.split(".")[-1] for x in resources]
+
+                if verbose:
+                    click.echo(FeedbackManager.info_writing_resource(resource=f))
+                if not f.exists() or force:
+                    async with aiofiles.open(f, "w") as fd:
+                        # versions are a client only thing so
+                        # datafiles from the server do not contains information about versions
+                        if k["version"] >= 0:
+                            if fmt:
+                                resource_to_write = "\n" + resource_to_write  # fmt strips the first line
+
+                            resource_to_write = f"VERSION {k['version']}\n" + resource_to_write
+                        if resource_to_write:
+                            matches = re.findall(r"([^\s\.]*__v\d+)", resource_to_write)
+                            for match in set(matches):
+                                m = match.split("__v")[0]
+                                if m in resources or m in resource_names:
+                                    resource_to_write = resource_to_write.replace(match, m)
+                            await fd.write(resource_to_write)
+                else:
+                    if verbose:
+                        click.echo(FeedbackManager.info_skip_already_exists())
+            except Exception as e:
+                raise click.ClickException(FeedbackManager.error_exception(error=e))
+
+        values = versions.values()
+
+        if progress_bar:
+            with click.progressbar(values, label=f"Pulling {extension}s") as values:  # type: ignore
+                for k in values:
+                    await write_resource(k)
+        else:
+            tasks = [write_resource(k) for k in values]
+            await _gather_with_concurrency(5, *tasks)
+
+    try:
+        datasources = await client.datasources()
+        remote_datasources = sorted([x["name"] for x in datasources])
+        datasources_versions = _get_latest_versions(remote_datasources)
+
+        pipes = await client.pipes()
+        remote_pipes = sorted([x["name"] for x in pipes])
+        pipes_versions = _get_latest_versions(remote_pipes)
+
+        resources = list(datasources_versions.keys()) + list(pipes_versions.keys())
+
+        await write_files(
+            datasources_versions, resources, "datasource", "datasource_file", progress_bar=progress_bar, fmt=fmt
+        )
+        await write_files(pipes_versions, resources, "pipe", "pipe_file", progress_bar=progress_bar, fmt=fmt)
+
+        return
+
+    except AuthNoTokenException:
+        raise
+    except Exception as e:
+        raise click.ClickException(FeedbackManager.error_pull(error=str(e)))
+
+
+async def _gather_with_concurrency(n, *tasks):
+    semaphore = Semaphore(n)
+
+    async def sem_task(task):
+        async with semaphore:
+            return await task
+
+    return await gather(*(sem_task(task) for task in tasks))

tinybird/tb/modules/datasource.py

@@ -41,7 +41,7 @@ from tinybird.tb.modules.common import (
     validate_kafka_topic,
     wait_job,
 )
-from tinybird.tb.modules.datafile import get_name_version
+from tinybird.tb.modules.datafile.common import get_name_version
 from tinybird.tb.modules.exceptions import CLIDatasourceException
 
 

tinybird/tb/modules/fmt.py

@@ -10,7 +10,10 @@ from click import Context
 from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import coro
-from tinybird.tb.modules.datafile import color_diff, format_datasource, format_pipe, is_file_a_datasource, peek
+from tinybird.tb.modules.datafile.common import is_file_a_datasource
+from tinybird.tb.modules.datafile.diff import color_diff, peek
+from tinybird.tb.modules.datafile.format_datasource import format_datasource
+from tinybird.tb.modules.datafile.format_pipe import format_pipe
 
 
 @cli.command()

tinybird/tb/modules/pipe.py

@@ -19,8 +19,14 @@ from tinybird.config import DEFAULT_API_HOST, FeatureFlags
 from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb.modules.branch import warn_if_in_live
 from tinybird.tb.modules.cli import cli
-from tinybird.tb.modules.common import coro, create_tb_client, echo_safe_humanfriendly_tables_format_smart_table
-from tinybird.tb.modules.datafile import PipeNodeTypes, PipeTypes, folder_push, get_name_version, process_file, wait_job
+from tinybird.tb.modules.common import (
+    coro,
+    create_tb_client,
+    echo_safe_humanfriendly_tables_format_smart_table,
+    wait_job,
+)
+from tinybird.tb.modules.datafile.build import folder_push, process_file
+from tinybird.tb.modules.datafile.common import PipeNodeTypes, PipeTypes, get_name_version
 from tinybird.tb.modules.exceptions import CLIPipeException
 
 

tinybird/tb/modules/prompts.py

@@ -44,7 +44,7 @@ SQL >
     - Create multiple pipes to show different use cases over the same datasource.
     - The SQL query must be a valid ClickHouse SQL query that mixes ClickHouse syntax and Tinybird templating syntax.
     - If you need to use dynamic parameters you must start the whole sql query ALWAYS with "%" symbol on top. e.g: SQL >\n    %\n SELECT * FROM <table> WHERE <condition> LIMIT 10
-    - The Parameter functions like this one {{String(<my_param_name>,<default_value>)}} can be one of the following: String, DateTime, Date, Float32, Float64, Int, Integer, UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256
+    - The Parameter functions like this one {{String(my_param_name,default_value)}} can be one of the following: String, DateTime, Date, Float32, Float64, Int, Integer, UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256
     - Parameter names must be different from column names. Pass always the param name and a default value to the function.
     - Nodes can't have the same exact name as the Pipe they belong to.
 </instructions>

tinybird/tb/modules/workspace.py

@@ -26,7 +26,7 @@ from tinybird.tb.modules.common import (
     switch_workspace,
 )
 from tinybird.tb.modules.config import CLIConfig
-from tinybird.tb.modules.datafile import PipeTypes
+from tinybird.tb.modules.datafile.common import PipeTypes
 from tinybird.tb.modules.exceptions import CLIWorkspaceException
 
 

{tinybird-0.0.1.dev6.dist-info → tinybird-0.0.1.dev7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird
-Version: 0.0.1.dev6
+Version: 0.0.1.dev7
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev6.dist-info → tinybird-0.0.1.dev7.dist-info}/RECORD

@@ -17,32 +17,45 @@ tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj
 tinybird/ch_utils/engine.py,sha256=OXkBhlzGjZotjD0vaT-rFIbSGV4tpiHxE8qO_ip0SyQ,40454
 tinybird/tb/cli.py,sha256=6Lu3wsCNepAxjJCWy4c6RhVPArBtm8TlUcSxX--TsBo,783
 tinybird/tb/modules/auth.py,sha256=hynZ-Temot8YBsySUWKSFzZlYadtFPxG3o6lCSu1n6E,9018
-tinybird/tb/modules/branch.py,sha256=X3Ks8SnIeZLgU6wU6tY0-PfLmIJdGho6wlkmC4o4nOg,39119
-tinybird/tb/modules/build.py,sha256=CAZq1BxbGM-uaoGanVEtZT3YwzWN_jJ6Fzgrw8Nwi3g,7798
+tinybird/tb/modules/branch.py,sha256=R1tTUBGyI0p_dt2IAWbuyNOvemhjCIPwYxEmOxL3zOg,38468
+tinybird/tb/modules/build.py,sha256=dAsgm3UyC46aMYFOH9O8qsyYdBJBoyoTV1mXSBcDJIc,7643
 tinybird/tb/modules/cicd.py,sha256=mIMU1gNGpN3f5K3rBYN9rFoOT3RogDxIs_zB3LY-iO4,7463
-tinybird/tb/modules/cli.py,sha256=Kz55MfNv01lcYow2TNRJuGZvUZ_2Sa-jfNqeYVmhuJU,61968
-tinybird/tb/modules/common.py,sha256=KBuSgVKqyUE4OVgMqD7BXro346IxcSNkKCf_I1_D9fw,79086
+tinybird/tb/modules/cli.py,sha256=Yw6ooemOAIUZR-IKk05BG_D7RgoljTgoHBYW7aq4XaM,56674
+tinybird/tb/modules/common.py,sha256=vb69xKJXbRjoYOlCwerXEU3488Od_zfR2N2yP5VyNw8,79372
 tinybird/tb/modules/config.py,sha256=ppWvACHrSLkb5hOoQLYNby2w8jR76-8Kx2NBCst7ntQ,11760
 tinybird/tb/modules/connection.py,sha256=ZSqBGoRiJedjHKEyB_fr1ybucOHtaad8d7uqGa2Q92M,28668
-tinybird/tb/modules/create.py,sha256=E5hs03kDdJj7n0_6H3loUOcfNv71Jbks0EBu9Kdre4Q,6898
-tinybird/tb/modules/datafile.py,sha256=16MTuwXqsruhf_UoniDQSHetMIGeFbiUtZcLDJcqoAk,253563
-tinybird/tb/modules/datasource.py,sha256=Wkxj6QBtTKCgl43JFwMWafEHYnnQjGZZHoL2SbuC3KQ,35811
+tinybird/tb/modules/create.py,sha256=24TWPzRnG8WmDLm8LP2Y5Sb4wwbz4rzarL3YwKQU4i8,6761
+tinybird/tb/modules/datasource.py,sha256=tjcf5o-HYIdTkb_c1ErGUFIE-W6G992vsvCuDGcxb9Q,35818
 tinybird/tb/modules/exceptions.py,sha256=4A2sSjCEqKUMqpP3WI00zouCWW4uLaghXXLZBSw04mY,3363
-tinybird/tb/modules/fmt.py,sha256=ZoYLUkyExudbUr-Phy-I2riPBdQd1NXwA1y-XbSEFB8,3385
+tinybird/tb/modules/fmt.py,sha256=UszEQO15fdzQ49QEj7Unhu68IKwSuKPsOrKhk2p2TAg,3547
 tinybird/tb/modules/job.py,sha256=eoBVyA24lYIPonU88Jn7FF9hBKz1kScy9_w_oWreuc4,2952
 tinybird/tb/modules/llm.py,sha256=D6ShlqCorZpRLjE5srI7Ws4VUTH6kSotTi88CvUwoGw,2446
 tinybird/tb/modules/local.py,sha256=nCaN05ntqIGFbBqyhRwpwdLpcYwsx6Vvp0atU_KKaTo,5893
 tinybird/tb/modules/mock.py,sha256=dYxm8_1zXGZnf4X8OYmG9hK1I-csHakD64Hag8mAn4Q,2069
-tinybird/tb/modules/pipe.py,sha256=LIiEK3zUmLcu24zgHy8qWvDD4yQjwnb--AZ1vbs0554,30984
-tinybird/tb/modules/prompts.py,sha256=12Abh1kZItxMa7r3WbTi_TgTYtlV30HcaG_JiodafIA,7255
+tinybird/tb/modules/pipe.py,sha256=9wnfKbp2FkmLiJgVk3qbra76ktwsUTXghu6j9cCEahQ,31058
+tinybird/tb/modules/prompts.py,sha256=u_-VSsqk8fXJTsLO3sCv9DR1Qx2V_M_Z_gBfiyvn5jA,7251
 tinybird/tb/modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb/modules/table.py,sha256=hG-PRDVuFp2uph41WpoLRV1yjp3RI2fi_iGGiI0rdxU,7695
 tinybird/tb/modules/tag.py,sha256=1qQWyk1p3Btv3LzM8VbJG-k7x2-pFuAlYCg3QL6QewI,3480
 tinybird/tb/modules/telemetry.py,sha256=iEGnMuCuNhvF6ln__j6X9MSTwL_0Hm-GgFHHHvhfknk,10466
 tinybird/tb/modules/test.py,sha256=psINFpSYT1eGgy32-_4q6CJ7LOcdwBpAfasMA0_tNOU,4330
 tinybird/tb/modules/token.py,sha256=r0oeG1RpOOzHtqbUaHBiOmhE55HfNIvReAAWyKl9fJg,12695
-tinybird/tb/modules/workspace.py,sha256=alZFDKy_qaZk5TIe6gduBeNgpp4h5jp6RDj1SwP5ylg,10909
+tinybird/tb/modules/workspace.py,sha256=FVlh-kbiZp5Gvp6dGFxi0UD8ail77rMamXLhqdVwrZ0,10916
 tinybird/tb/modules/workspace_members.py,sha256=08W0onEYkKLEC5TkAI07cxN9XSquEm7HnL7OkHAVDjo,8715
+tinybird/tb/modules/datafile/build.py,sha256=NAvtZS8skAt_8HIdzOJhbp-d9kMIqFQJ9tupQ6M9IK0,90180
+tinybird/tb/modules/datafile/build_common.py,sha256=74547h5ja4C66DAwDMabj75FA_BUTJxTJv-24tSFmrs,4551
+tinybird/tb/modules/datafile/build_datasource.py,sha256=cJee7crf57gAGIa76Du0TznVU9gtk7F88eBCI7930tk,16836
+tinybird/tb/modules/datafile/build_pipe.py,sha256=X4a-UM_GSOmR8ks2kBITgebXz-aE-iEdG5F1eEUzIyc,27555
+tinybird/tb/modules/datafile/common.py,sha256=g5G0jXC-okEiKPnWa8X3p3-VxXoZlQQUwKzhxVA0tfs,33918
+tinybird/tb/modules/datafile/diff.py,sha256=ZaTPGjRFJWokhaad_rMSxfYT92PA96s4WhhvlZubgyA,6769
+tinybird/tb/modules/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
+tinybird/tb/modules/datafile/format_common.py,sha256=zNWDXvwSKC9_T5e9R92LLj9ekDflVWwsllhGQilZsnY,2184
+tinybird/tb/modules/datafile/format_datasource.py,sha256=tsnCjONISvhFuucKNbIHkT__UmlUbcswx5mwI9hiDQc,6216
+tinybird/tb/modules/datafile/format_pipe.py,sha256=R5tnlEccLn3KX6ehtC_H2sGQNrthuJUiVSN9z_-KGCY,7474
+tinybird/tb/modules/datafile/parse_datasource.py,sha256=9pp0fJ2Da-7pznaHq5OV7NvzswK0RuXOPHU4kXNRMnA,1111
+tinybird/tb/modules/datafile/parse_pipe.py,sha256=STgA12LOLUnnb_cvVvZeEE4ka-nfk0jsNzxJhWj94cY,2599
+tinybird/tb/modules/datafile/pipe_checker.py,sha256=cp80Bru41GlyMRvyERpdJNXns2MjmtIAWFnBLF4cPXs,24667
+tinybird/tb/modules/datafile/pull.py,sha256=MleQG9GTo9_ZoMKrneps-QzI4dr6n5IPosHWz2sTz20,5894
 tinybird/tb/modules/tinyunit/tinyunit.py,sha256=IkjRCvb8HnNEE84rtl0I1b9gQVpE_zCE8MvFFet51sg,11716
 tinybird/tb/modules/tinyunit/tinyunit_lib.py,sha256=hGh1ZaXC1af7rKnX7222urkj0QJMhMWclqMy59dOqwE,1922
 tinybird/tb_cli_modules/cicd.py,sha256=0lMkb6CVOFZl5HOwgY8mK4T4mgI7O8335UngLXtCc-c,13851
@@ -51,8 +64,8 @@ tinybird/tb_cli_modules/config.py,sha256=6NTgIdwf0X132A1j6G_YrdPep87ymZ9b5pABabK
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=iEGnMuCuNhvF6ln__j6X9MSTwL_0Hm-GgFHHHvhfknk,10466
-tinybird-0.0.1.dev6.dist-info/METADATA,sha256=wgXkBp0lUmzhrrDmTwxzy0FKVfmEIGfZ6no4CQF3uQc,2404
-tinybird-0.0.1.dev6.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-tinybird-0.0.1.dev6.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev6.dist-info/top_level.txt,sha256=pgw6AzERHBcW3YTi2PW4arjxLkulk2msOz_SomfOEuc,45
-tinybird-0.0.1.dev6.dist-info/RECORD,,
+tinybird-0.0.1.dev7.dist-info/METADATA,sha256=MDbMCTsirb0x3yH4RirvEzBzYvjieV1ucRH1R89aPAw,2404
+tinybird-0.0.1.dev7.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+tinybird-0.0.1.dev7.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev7.dist-info/top_level.txt,sha256=pgw6AzERHBcW3YTi2PW4arjxLkulk2msOz_SomfOEuc,45
+tinybird-0.0.1.dev7.dist-info/RECORD,,