remotivelabs-cli 0.0.24__py3-none-any.whl → 0.0.25__py3-none-any.whl

cli/cloud/cloud_cli.py

@@ -2,7 +2,7 @@ import json
 
 import typer
 
-from . import auth, brokers, configs, projects, recordings, sample_recordings, service_accounts
+from . import auth, brokers, configs, filestorage, projects, recordings, sample_recordings, service_accounts
 from . import rest_helper as rest
 
 app = typer.Typer()
@@ -32,6 +32,7 @@ app.add_typer(auth.app, name="auth")
 app.add_typer(brokers.app, name="brokers", help="Manage cloud broker lifecycle")
 app.add_typer(recordings.app, name="recordings", help="Manage recordings")
 app.add_typer(configs.app, name="signal-databases", help="Manage signal databases")
+app.add_typer(filestorage.app, name="storage")
 app.add_typer(service_accounts.app, name="service-accounts", help="Manage project service account keys")
 app.add_typer(sample_recordings.app, name="samples", help="Manage sample recordings")
 

cli/cloud/filestorage.py

@@ -0,0 +1,143 @@
+import os.path
+from pathlib import Path
+
+import typer
+
+from ..errors import ErrorPrinter
+from . import rest_helper as rest
+from . import resumable_upload as upload
+
+app = typer.Typer(
+    rich_markup_mode="rich",
+    help="""
+Manage files ([yellow]Beta feature not available for all customers[/yellow])
+
+Copy file from local to remote storage and vice versa, list and delete files.
+
+""",
+)
+
+
+@app.command(name="ls")
+def list_files(
+    project: str = typer.Option(..., help="Project ID", envvar="REMOTIVE_CLOUD_PROJECT"),
+    prefix: str = typer.Argument(default="rcs://", help="Remote storage path"),
+):
+    """
+    Listing remote files
+
+    This will list files and directories in project top level directory
+    remotive cloud storage ls rcs://
+
+    This will list all files and directories matching the path
+    remotive cloud storage ls rcs://fileOrDirectoryPrefix
+
+    This will list all files and directories in the specified directory
+    remotive cloud storage ls rcs://fileOrDirectory/
+    """
+
+    if prefix.startswith("rcs://"):
+        prefix = __check_rcs_path(prefix)
+    else:
+        ErrorPrinter.print_hint("Path must start with rcs://")
+        exit(1)
+
+    rest.handle_get(
+        f"/api/project/{project}/files/storage{prefix}",
+    )
+
+
+@app.command(name="rm")
+def delete_file(
+    project: str = typer.Option(..., help="Project ID", envvar="REMOTIVE_CLOUD_PROJECT"),
+    path: str = typer.Argument(default=..., help="Remote storage path to file to delete"),
+):
+    """
+    [red]Deletes[/red] a file from remote storage, this cannot be undone :fire:
+
+    [white]remotive cloud storage rm rcs://directory/filename[/white]
+    """
+    if path.startswith("rcs://"):
+        prefix = __check_rcs_path(path)
+    else:
+        ErrorPrinter.print_hint("Path must start with rcs://")
+        exit(1)
+
+    rest.handle_delete(
+        f"/api/project/{project}/files/storage{prefix}",
+    )
+
+
+@app.command(name="cp")
+def copy_file(
+    source: str = typer.Argument(default=..., help="Remote or local path to source file"),
+    dest: str = typer.Argument(default=..., help="Remote or local path to destination file"),
+    project: str = typer.Option(..., help="Project ID", envvar="REMOTIVE_CLOUD_PROJECT"),
+):
+    """
+    Copies a file to or from remote storage
+
+    remotive cloud storage cp rcs://dir/filename .
+    remotive cloud storage cp rcs://dir/filename filename
+
+    remotive cloud storage cp filename rcs://dir/
+    remotive cloud storage cp filename rcs://dir/filename
+    """
+
+    if not source.startswith("rcs://") and not dest.startswith("rcs://"):
+        ErrorPrinter.print_hint("Source or destination path must be an rcs:// path")
+        exit(2)
+
+    if source.startswith("rcs://") and dest.startswith("rcs://"):
+        ErrorPrinter.print_hint("Currently one of source and destination path must be a local path")
+        exit(2)
+
+    if source.startswith("rcs://"):
+        rcs_path = __check_rcs_path(source)
+        filename = source.rsplit("/", 1)[-1]
+        path = Path(dest)
+        if path.is_dir():
+            if not path.exists():
+                ErrorPrinter.print_generic_error("Destination directory does not exist")
+                exit(1)
+            else:
+                dest = os.path.join(path.absolute(), filename)
+
+        else:
+            if not path.parent.is_dir() or not path.parent.exists():
+                ErrorPrinter.print_generic_error("Destination directory does not exist")
+                exit(1)
+        dest = Path(dest).absolute()
+
+        res = rest.handle_get(
+            f"/api/project/{project}/files/storage{rcs_path}?download=true",
+            return_response=True,
+        )
+
+        rest.download_file(save_file_name=dest, url=res.text)
+
+    else:
+        path = Path(source)
+        if not path.exists():
+            ErrorPrinter.print_hint("Source file does not exist")
+            exit(1)
+        filename = source.rsplit("/", 1)[-1]
+        rcs_path = __check_rcs_path(dest)
+        if rcs_path.endswith("/"):
+            rcs_path = rcs_path + filename
+        res = rest.handle_post(f"/api/project/{project}/files/storage{rcs_path}", return_response=True)
+        json = res.json()
+        url = json["url"]
+        content_type = json["contentType"]
+        try:
+            upload.upload_signed_url(url, source, content_type)
+        except IsADirectoryError:
+            ErrorPrinter.print_hint(f"Supplied source file '{source}' is a directory but must be a file")
+
+
+def __check_rcs_path(path: str):
+    rcs_path = path.replace("rcs://", "/")
+    if rcs_path.startswith("/."):
+        ErrorPrinter.print_hint("Invalid path")
+        exit(1)
+    return rcs_path

cli/cloud/rest_helper.py

@@ -10,6 +10,7 @@ from pathlib import Path
 from typing import Dict, List, Union
 
 import requests
+from requests.exceptions import JSONDecodeError
 from rich.console import Console
 from rich.progress import Progress, SpinnerColumn, TextColumn, wrap_file
 
@@ -118,7 +119,11 @@ def check_api_result(response, allow_status_codes: List[int] = None):
 def print_api_result(response):
     if response.status_code >= 200 and response.status_code < 300:
         if len(response.content) > 4:
-            print(json.dumps(response.json()))
+            try:
+                print(json.dumps(response.json()))
+            except JSONDecodeError:
+                err_console.print(":boom: [bold red]Json parse error[/bold red]: Please try again and report if the error persists")
+
         exit(0)
     else:
         err_console.print(f":boom: [bold red]Got status code[/bold red]: {response.status_code}")
@@ -205,7 +210,9 @@ def download_file(save_file_name: str, url: str):
     if download_resp.status_code == 200:
         content_length = int(download_resp.headers["Content-Length"])
         with open(save_file_name, "wb") as out_file:
-            with wrap_file(download_resp.raw, content_length, refresh_per_second=100) as stream_with_progress:
+            with wrap_file(
+                download_resp.raw, content_length, refresh_per_second=100, description=f"Downloading to {save_file_name}"
+            ) as stream_with_progress:
                 shutil.copyfileobj(stream_with_progress, out_file)
     else:
         check_api_result(download_resp)

cli/cloud/resumable_upload.py

@@ -0,0 +1,61 @@
+import os
+
+import requests
+from rich.progress import wrap_file
+
+from ..errors import ErrorPrinter
+
+
+def __get_uploaded_bytes(upload_url):
+    headers = {"Content-Range": "bytes */*"}
+    response = requests.put(upload_url, headers=headers)
+    if response.status_code != 308:
+        raise Exception(f"Failed to retrieve upload status: {response.status_code} {response.text}")
+
+    # Parse the Range header to get the last byte uploaded
+    range_header = response.headers.get("Range")
+    if range_header:
+        last_byte = int(range_header.split("-")[1])
+        return last_byte + 1
+    return 0
+
+
+def upload_signed_url(signed_url, source_file_name, content_type):
+    """
+    Upload file to file storage with signed url and resumable upload.
+    Resumable upload will only work with the same URL and not if a new signed URL is requested with the
+    same object id.
+    :param signed_url:
+    :param source_file_name:
+    :return:
+    """
+
+    file_size = os.path.getsize(source_file_name)
+    headers = {"x-goog-resumable": "start", "content-type": content_type}
+    response = requests.post(signed_url, headers=headers)
+    if response.status_code not in (200, 201, 308):
+        ErrorPrinter.print_generic_error(f"Failed to upload file: {response.status_code} - {response.text}")
+        exit(1)
+
+    upload_url = response.headers["Location"]
+
+    # Check how many bytes have already been uploaded
+    uploaded_bytes = __get_uploaded_bytes(upload_url)
+
+    # Upload the remaining file in chunks
+    # Not sure what a good chunk size is or if we even should have resumable uploads here, probably not..
+    chunk_size = 256 * 1024 * 10
+    # Upload the file in chunks
+    with open(source_file_name, "rb") as f:
+        with wrap_file(f, os.stat(source_file_name).st_size, description=f"Uploading {source_file_name}...") as file:
+            file.seek(uploaded_bytes)  # Seek to the position of the last uploaded byte
+            for chunk_start in range(uploaded_bytes, file_size, chunk_size):
+                chunk_end = min(chunk_start + chunk_size, file_size) - 1
+                chunk = file.read(chunk_end - chunk_start + 1)
+                headers = {"Content-Range": f"bytes {chunk_start}-{chunk_end}/{file_size}"}
+                response = requests.put(upload_url, headers=headers, data=chunk)
+                if response.status_code not in (200, 201, 308):
+                    ErrorPrinter.print_generic_error(f"Failed to upload file: {response.status_code} - {response.text}")
+                    exit(1)
+
+    print(f"File {source_file_name} uploaded successfully.")

{remotivelabs_cli-0.0.24.dist-info → remotivelabs_cli-0.0.25.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: remotivelabs-cli
-Version: 0.0.24
+Version: 0.0.25
 Summary: CLI for operating RemotiveCloud and RemotiveBroker
 Author: Johan Rask
 Author-email: johan.rask@remotivelabs.com
@@ -15,7 +15,7 @@ Requires-Dist: plotext (>=5.2,<6.0)
 Requires-Dist: pyjwt (>=2.6,<3.0)
 Requires-Dist: python-can (>=4.3.1)
 Requires-Dist: python-socketio (>=4.6.1)
-Requires-Dist: remotivelabs-broker (>=0.1.17)
+Requires-Dist: remotivelabs-broker (>=0.1.17,<0.2.0)
 Requires-Dist: rich (>=13.7.0,<13.8.0)
 Requires-Dist: trogon (>=0.5.0)
 Requires-Dist: typer (>=0.9.0,<0.10.0)

{remotivelabs_cli-0.0.24.dist-info → remotivelabs_cli-0.0.25.dist-info}/RECORD

@@ -15,12 +15,14 @@ cli/cloud/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 cli/cloud/auth.py,sha256=xi47cQWjs6q7Kje4XUbTqEk_BgG3NJ1gMOYPU-_4tgw,3470
 cli/cloud/auth_tokens.py,sha256=5ox0Pxfij8aTgEcDwRBasaSQvDuLIGCJc-INjmIQN2M,3408
 cli/cloud/brokers.py,sha256=wa3uMg91IZdrP0tMpTdO9cBIkZHtMHxQ-zEXwFiye_I,4127
-cli/cloud/cloud_cli.py,sha256=nRRXFF_IgUMayerxS-h7oqsNe6tt34Q5VeThq8gatEg,1443
+cli/cloud/cloud_cli.py,sha256=oAk8Fw_A0XNjuFztHcMhWKrvGYE-WqYobPXIHBq5ZS4,1503
 cli/cloud/configs.py,sha256=2p1mCHf5BwYNtwbY0Cbed5t6-79WHGKWU4Fv6LuJ21o,4069
+cli/cloud/filestorage.py,sha256=2921FbflnGA89L7ARv8UXo8S5cOZpNPTGo2xySpo5Ho,4648
 cli/cloud/projects.py,sha256=-uqltAOficwprOKaPd2R0Itm4sqTz3VJNs9Sc8jtO5k,1369
 cli/cloud/recordings.py,sha256=QpNb9HWK_LC7Trex8E7RevEy7GTOBZHw0RsgOt_RmUA,23286
 cli/cloud/recordings_playback.py,sha256=crrwQ3kl8LzCVBan9B1a15t-vhDCNYqLKSVQbLf58Ng,10907
-cli/cloud/rest_helper.py,sha256=dUVmykp0_M4pxzI3WiGoURjlaeCe_V_2h7JwnteW69w,7785
+cli/cloud/rest_helper.py,sha256=4wuE7YxFNAaU4BdDIhipfazzuzulmxGGUsvOOn3xv3U,8102
+cli/cloud/resumable_upload.py,sha256=7M7Nm-FEJ7u2c7ezWZfJUOmDbG-Dk1St7xv9I7VDRZY,2586
 cli/cloud/sample_recordings.py,sha256=g1X6JTxvzWInSP9R1BJsDmL4WqvpEKqjdJR_xT4bo1U,639
 cli/cloud/service_account_tokens.py,sha256=7vjoMd6Xq7orWCUP7TVUVa86JA0OiX8O10NZcHUE6rM,2294
 cli/cloud/service_accounts.py,sha256=GCYdYPnP5uWVsg1bTIS67CmoPWDng5dupJHmlThrJ80,1606
@@ -35,8 +37,8 @@ cli/tools/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 cli/tools/can/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 cli/tools/can/can.py,sha256=kSd1c-nxxXyeKkm19oDILiDBZsKOcpjsUT0T3xox5Qs,2172
 cli/tools/tools.py,sha256=LwQdWMcJ19pCyKUsVfSB2B3R6ui61NxxFWP0Nrnd5Jk,198
-remotivelabs_cli-0.0.24.dist-info/LICENSE,sha256=qDPP_yfuv1fF-u7EfexN-cN3M8aFgGVndGhGLovLKz0,608
-remotivelabs_cli-0.0.24.dist-info/METADATA,sha256=376RvVFRj7s-wIYVbeK8XUi0Ogj8rOIf4jtgwSh-Zms,1224
-remotivelabs_cli-0.0.24.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-remotivelabs_cli-0.0.24.dist-info/entry_points.txt,sha256=lvDhPgagLqW_KTnLPCwKSqfYlEp-1uYVosRiPjsVj10,45
-remotivelabs_cli-0.0.24.dist-info/RECORD,,
+remotivelabs_cli-0.0.25.dist-info/LICENSE,sha256=qDPP_yfuv1fF-u7EfexN-cN3M8aFgGVndGhGLovLKz0,608
+remotivelabs_cli-0.0.25.dist-info/METADATA,sha256=aH4B6GkhSub8xR2cGdJbfjKcm_8BUhkwzgTa6DNCSlI,1231
+remotivelabs_cli-0.0.25.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+remotivelabs_cli-0.0.25.dist-info/entry_points.txt,sha256=lvDhPgagLqW_KTnLPCwKSqfYlEp-1uYVosRiPjsVj10,45
+remotivelabs_cli-0.0.25.dist-info/RECORD,,