tinybird-cli 5.0.1.dev0__tar.gz → 5.0.1.dev1__tar.gz

{tinybird-cli-5.0.1.dev0 → tinybird-cli-5.0.1.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.0.1.dev0
+Version: 5.0.1.dev1
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,11 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.1.0.dev1
+***********
+
+- `Added` COPY pipes now support `MODE replace`
+
 5.0.0
 ******
 

{tinybird-cli-5.0.1.dev0 → tinybird-cli-5.0.1.dev1}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '5.0.1.dev0'
-__revision__ = '2af0ad5'
+__version__ = '5.0.1.dev1'
+__revision__ = '3d0cfe4'

{tinybird-cli-5.0.1.dev0 → tinybird-cli-5.0.1.dev1}/tinybird/client.py

@@ -471,6 +471,8 @@ class TinyB(object):
         self, pipe_name: str, node: Dict[str, Any], dry_run: str = "false", datasource_name: Optional[str] = None
     ):
         params = {**{"include_datafile": "true", "dry_run": dry_run}, **node.get("params", node)}
+        if "mode" in params:
+            params.pop("mode")
         node_name = node["params"]["name"] if node.get("params", None) else node["name"]
         if datasource_name:
             params["datasource"] = datasource_name
@@ -550,34 +552,33 @@ class TinyB(object):
     async def pipe_remove_endpoint(self, pipe_name_or_uid: str, published_node_uid: str):
         return await self._req(f"/v0/pipes/{pipe_name_or_uid}/nodes/{published_node_uid}/endpoint", method="DELETE")
 
-    async def pipe_create_copy(
-        self, pipe_name_or_id: str, node_id: str, target_datasource: str, schedule_cron: Optional[str] = None
-    ):
-        data = {"target_datasource": target_datasource}
-        if schedule_cron:
-            data["schedule_cron"] = schedule_cron
-
-        return await self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="POST", data=data)
-
     async def pipe_update_copy(
         self,
         pipe_name_or_id: str,
         node_id: str,
         target_datasource: Optional[str] = None,
         schedule_cron: Optional[str] = None,
+        mode: Optional[str] = None,
     ):
         data = {"schedule_cron": schedule_cron}
 
         if target_datasource:
             data["target_datasource"] = target_datasource
 
+        if mode:
+            data["mode"] = mode
+
         return await self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="PUT", data=data)
 
     async def pipe_remove_copy(self, pipe_name_or_id: str, node_id: str):
         return await self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="DELETE")
 
-    async def pipe_run_copy(self, pipe_name_or_id: str, params: Optional[Dict[str, str]] = None):
+    async def pipe_run_copy(
+        self, pipe_name_or_id: str, params: Optional[Dict[str, str]] = None, mode: Optional[str] = None
+    ):
         params = {**params} if params else {}
+        if mode:
+            params["_mode"] = mode
         return await self._req(f"/v0/pipes/{pipe_name_or_id}/copy?{urlencode(params)}", method="POST")
 
     async def pipe_resume_copy(self, pipe_name_or_id: str):

{tinybird-cli-5.0.1.dev0 → tinybird-cli-5.0.1.dev1}/tinybird/datafile.py

@@ -118,6 +118,18 @@ class DataFileExtensions:
 class CopyParameters:
     TARGET_DATASOURCE = "target_datasource"
     COPY_SCHEDULE = "copy_schedule"
+    COPY_MODE = "copy_mode"
+
+
+class CopyModes:
+    APPEND = "append"
+    REPLACE = "replace"
+
+    valid_modes = (APPEND, REPLACE)
+
+    @staticmethod
+    def is_valid(node_mode):
+        return node_mode.lower() in CopyModes.valid_modes
 
 
 PREVIEW_CONNECTOR_SERVICES = ["s3", "s3_iamrole", "gcs"]
@@ -1076,6 +1088,8 @@ def parse(
         "tags": assign_node_var("tags"),
         "target_datasource": assign_node_var("target_datasource"),
         "copy_schedule": assign_node_var(CopyParameters.COPY_SCHEDULE),
+        "copy_mode": assign_node_var("mode"),
+        "mode": assign_node_var("mode"),
         "resource": assign_node_var("resource"),
         "filter": assign_node_var("filter"),
         "token": add_token,
@@ -1486,6 +1500,7 @@ async def process_file(
                 "description": node.get("description", ""),
                 "target_datasource": node.get("target_datasource", None),
                 "copy_schedule": node.get(CopyParameters.COPY_SCHEDULE, None),
+                "mode": node.get("mode", CopyModes.APPEND),
             }
 
             is_export_node = ExportReplacements.is_export_node(node)
@@ -2483,6 +2498,12 @@ async def check_copy_pipe(pipe, copy_node, tb_client: TinyB):
     if not is_valid_cron:
         raise CLIPipeException(FeedbackManager.error_creating_copy_pipe_invalid_cron(schedule_cron=schedule_cron))
 
+    mode = copy_node["params"].get("mode", CopyModes.APPEND)
+    is_valid_mode = CopyModes.is_valid(mode)
+
+    if not is_valid_mode:
+        raise CLIPipeException(FeedbackManager.error_creating_copy_pipe_invalid_mode(mode=mode))
+
     if not pipe:
         return
 
@@ -2764,6 +2785,7 @@ async def new_pipe(
             try:
                 target_datasource = copy_node.get(CopyParameters.TARGET_DATASOURCE, None)
                 schedule_cron = copy_node.get(CopyParameters.COPY_SCHEDULE, None)
+                mode = copy_node.get("mode", CopyModes.APPEND)
                 schedule_cron = None if schedule_cron == ON_DEMAND else schedule_cron
                 current_target_datasource_id = data["copy_target_datasource"]
                 target_datasource_response = await tb_client.get_datasource(target_datasource)
@@ -2777,6 +2799,7 @@ async def new_pipe(
                 current_schedule_cron = current_schedule.get("cron", None) if current_schedule else None
                 schedule_cron_should_be_removed = current_schedule_cron and not schedule_cron
                 copy_params["schedule_cron"] = "None" if schedule_cron_should_be_removed else schedule_cron
+                copy_params["mode"] = mode
                 await tb_client.pipe_update_copy(**copy_params)
             except Exception as e:
                 raise Exception(
@@ -4628,6 +4651,10 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
         file_parts.append(node_type_upper)
         file_parts.append(DATAFILE_NEW_LINE)
         file_parts.append(f'TARGET_DATASOURCE {node["target_datasource"]}')
+        if node.get("mode"):
+            file_parts.append(DATAFILE_NEW_LINE)
+            file_parts.append(f'COPY_MODE {node.get("mode")}')
+
         if CopyParameters.COPY_SCHEDULE in node and node[CopyParameters.COPY_SCHEDULE]:
             is_ondemand = node[CopyParameters.COPY_SCHEDULE].lower() == ON_DEMAND
             file_parts.append(DATAFILE_NEW_LINE)

{tinybird-cli-5.0.1.dev0 → tinybird-cli-5.0.1.dev1}/tinybird/feedback_manager.py

@@ -126,6 +126,9 @@ class FeedbackManager:
     error_creating_copy_pipe_invalid_cron = error_message(
         "Cannot create Copy pipe. Invalid cron expression: '{schedule_cron}'"
     )
+    error_creating_copy_pipe_invalid_mode = error_message(
+        "Cannot create Copy pipe. Invalid MODE expression: '{mode}'. Valid modes are: 'append', 'replace'"
+    )
     error_creating_sink_pipe_invalid_cron = error_message(
         "Cannot create Sink Pipe. Invalid cron expression: '{schedule_cron}'"
     )

{tinybird-cli-5.0.1.dev0 → tinybird-cli-5.0.1.dev1}/tinybird/tb_cli_modules/pipe.py

@@ -604,6 +604,9 @@ async def regression_test(
 @pipe_copy.command(name="run", short_help="Run an on-demand copy job")
 @click.argument("pipe_name_or_id")
 @click.option("--wait", is_flag=True, default=False, help="Wait for the copy job to finish")
+@click.option(
+    "--mode", type=click.Choice(["append", "replace"], case_sensitive=True), default=None, help="Copy strategy"
+)
 @click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
 @click.option(
     "--param",
@@ -615,7 +618,9 @@ async def regression_test(
 )
 @click.pass_context
 @coro
-async def pipe_copy_run(ctx: click.Context, pipe_name_or_id: str, wait: bool, yes: bool, param: Optional[Tuple[str]]):
+async def pipe_copy_run(
+    ctx: click.Context, pipe_name_or_id: str, wait: bool, mode: str, yes: bool, param: Optional[Tuple[str]]
+):
     """Run an on-demand copy job"""
 
     params = dict(key_value.split("=") for key_value in param) if param else {}
@@ -625,7 +630,7 @@ async def pipe_copy_run(ctx: click.Context, pipe_name_or_id: str, wait: bool, ye
         client: TinyB = ctx.ensure_object(dict)["client"]
 
         try:
-            response = await client.pipe_run_copy(pipe_name_or_id, params)
+            response = await client.pipe_run_copy(pipe_name_or_id, params, mode)
 
             job_id = response["job"]["job_id"]
             job_url = response["job"]["job_url"]

{tinybird-cli-5.0.1.dev0 → tinybird-cli-5.0.1.dev1}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.0.1.dev0
+Version: 5.0.1.dev1
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,6 +18,11 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.1.0.dev1
+***********
+
+- `Added` COPY pipes now support `MODE replace`
+
 5.0.0
 ******