seqslab-cli 3.3.3__tar.gz → 3.3.5__tar.gz

{seqslab-cli-3.3.3/python/seqslab_cli.egg-info → seqslab-cli-3.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: seqslab-cli
-Version: 3.3.3
+Version: 3.3.5
 Summary: Atgenomix SeqsLab Command Line Tool
 Home-page: https://github.com/AnomeGAP/seqslab-cli
 Author: Allen Chang

{seqslab-cli-3.3.3 → seqslab-cli-3.3.5}/python/seqslab/__init__.py

@@ -21,6 +21,6 @@ name = "seqslab"
 __all__ = []
 
 
-__version__ = "3.3.3"
+__version__ = "3.3.5"
 
 LOGGING = {"DIR_PATH": "/var/log/seqslab"}

{seqslab-cli-3.3.3 → seqslab-cli-3.3.5}/python/seqslab/auth/azuread.py

@@ -72,14 +72,14 @@ class ClientApp(AuthBaseModel):
     public_app: msal.ClientApplication = None
     confidential_app: msal.ClientApplication = None
 
-    def load_client(self, credential: str = None) -> Client:
+    def load_client(self, credential: str = None, tenant: str = None) -> Client:
         proxies = context.get_context().args.proxy
         if credential:
             if not self.confidential_app:
                 self.confidential_app = msal.ConfidentialClientApplication(
                     SOCIAL_AUTH_AZURE_KEY,
                     client_credential=credential,
-                    authority=f"https://login.microsoftonline.com/{SOCIAL_AUTH_AZURE_TENANT_ID}",
+                    authority=f"https://login.microsoftonline.com/{tenant}",
                     app_name=seqslab.name,
                     app_version=seqslab.__version__,
                     proxies=proxies,

{seqslab-cli-3.3.3 → seqslab-cli-3.3.5}/python/seqslab/auth/commands.py

@@ -60,7 +60,9 @@ class BaseAuth:
         proxy: str = None,
     ) -> dict:
         scopes = azuread.SOCIAL_AUTH_AZURE_SCOPE_APP.get(scope)
-        client = aad_app.load_client(credential=credential)
+        client = aad_app.load_client(
+            credential=credential, tenant=self._decode(assertion)["tid"]
+        )
         result = client.acquire_token_silent(scopes, account=None)
         if not result:
             logging.info(
@@ -92,7 +94,7 @@ class BaseAuth:
         result = None
         token_uri = self.ACCESS_TOKEN_URL.format(backend=backend)
         scopes = azuread.SOCIAL_AUTH_AZURE_SCOPE
-        client = aad_app.load_client()
+        client = aad_app.load_client(tenant=azuread.SOCIAL_AUTH_AZURE_TENANT_ID)
         accounts = client.get_accounts()
         if accounts:
             logging.info(

seqslab-cli-3.3.5/python/seqslab/color.py

@@ -0,0 +1,8 @@
+def color_handler(func):
+    ColorDict = {"yellow": "\033[01;33m", "green": "\033[01;32m", "red": "\033[01;31m"}
+
+    def inner_function(*args, **kwargs):
+        kwargs.update(ColorDict)
+        return func(*args, **kwargs)
+
+    return inner_function

{seqslab-cli-3.3.3 → seqslab-cli-3.3.5}/python/seqslab/drs/api/base.py

@@ -333,9 +333,9 @@ class DRSregister:
             folder_stdin["access_methods"].clear()
             for i, dsts in enumerate(dsts_list):
                 folder_stdin["access_methods"].append(access_method_infos[i])
-                folder_stdin["access_methods"][i]["access_url"][
-                    "url"
-                ] = self.common_root(dsts=dsts, workspace=kwargs.get("workspace"))
+                folder_stdin["access_methods"][i]["access_url"]["url"] = (
+                    self.common_root(dsts=dsts, workspace=kwargs.get("workspace"))
+                )
 
         folder_stdin["name"] = kwargs.get(
             "name",

{seqslab-cli-3.3.3 → seqslab-cli-3.3.5}/python/seqslab/drs/commands.py

@@ -15,10 +15,12 @@ from urllib.parse import unquote
 
 from jsonpath_ng.ext import parse
 from nubia import argument, command, context
+from seqslab.color import color_handler
 from seqslab.drs.utils.atgxmetadata import AtgxMetaData
 from seqslab.drs.utils.progressbar import ProgressBarObject
 from seqslab.exceptions import async_exception_handler, exception_handler
 from seqslab.runsheet.runsheet import RunSheet
+from seqslab.workspace.internal.common import get_factory as get_workspace_factory
 from tabulate import tabulate
 from termcolor import cprint
 from yarl import URL
@@ -57,6 +59,7 @@ class BaseDatahub:
         return context.get_context().args.proxy
 
     @staticmethod
+    @color_handler
     @exception_handler
     def _upload(src: URL, dst: URL, recursive: bool, **kwargs) -> list:
         # copy from local to cloud
@@ -67,33 +70,40 @@ class BaseDatahub:
             if os.path.isdir(paths[0]):
                 if not recursive:
                     raise OSError("--recursive (-r) is Required.")
-                if not str(dst).endswith("/") and not kwargs.get("non_interactive"):
-                    sys.stderr.write(
-                        f'f"Do you mean to rename the `{os.path.basename(str(src))}` to '
-                        f'`{os.path.basename(str(dst))}`？(Y/N): "'
-                    )
-                    user_input = input()
-                    if user_input.lower() != "y":
-                        raise InterruptedError("Interrupt all the process.")
+                sys.stderr.write(
+                    f"{kwargs['green']}"
+                    + "Destination Path: "
+                    + f"{kwargs['red']}"
+                    + f"{str(dst)}\n"
+                    + f"{kwargs['yellow']}\0"
+                )
                 coro = aiocopy.dir_to_blob(URL(*paths), dst, **kwargs)
             else:
-                if not str(dst).endswith("/") and not kwargs.get("non_interactive"):
+                if str(dst).endswith("/"):
+                    sys.stderr.write(
+                        f"{kwargs['green']}"
+                        + "Destination Path: "
+                        + f"{kwargs['red']}"
+                        + f"{os.path.join(str(dst), os.path.basename(str(src)))} \n"
+                        + f"{kwargs['yellow']}\0"
+                    )
+                else:
                     sys.stderr.write(
-                        f'f"Do you mean to rename the `{os.path.basename(str(src))}` to '
-                        f'`{os.path.basename(str(dst))}`？(Y/N): "'
+                        f"{kwargs['green']}"
+                        + "Destination Path: "
+                        + f"{kwargs['red']}"
+                        + f"{str(dst)} \n"
+                        + f"{kwargs['yellow']}\0"
                     )
-                    user_input = input()
-                    if user_input.lower() != "y":
-                        raise InterruptedError("Interrupt all the process.")
                 coro = aiocopy.file_to_blob([URL(*paths)], dst, **kwargs)
         else:
-            if not str(dst).endswith("/") and not kwargs.get("non_interactive"):
-                sys.stderr.write(
-                    f"Do you mean to upload all the files into {str(dst)}/？(Y/N): "
-                )
-                user_input = input()
-                if user_input.lower() != "y":
-                    raise InterruptedError("Interrupt all the process.")
+            sys.stderr.write(
+                f"{kwargs['green']}"
+                + "Destination Path: "
+                + f"{kwargs['red']}"
+                + f"cloud://{str(dst)}. \n"
+                + f"{kwargs['yellow']}\0"
+            )
             files = []
             dirs = []
             for p in paths:
@@ -156,6 +166,13 @@ class BaseDatahub:
                 "Stdin format only supports json. Please read the document and make sure the format is valid."
             )
 
+    @staticmethod
+    def _valide_workspace(workspace: str) -> bool:
+        ctx = context.get_context()
+        backend = ctx.args.backend
+        resource = get_workspace_factory().load_resource()
+        return resource.validate_workspace(workspace, backend)
+
     @command(aliases=["copy"])
     @argument(
         "workspace",
@@ -173,7 +190,9 @@ class BaseDatahub:
         "dst",
         type=str,
         positional=False,
-        description="Specify the destination directory, file path, or DRS URL (optional, default ='MyDir/'). ",
+        description="Specify the destination directory, file path, or DRS URL. Paths with leading slashes, leading "
+        "backslashes, or dot-only segments are invalid. Additionally, paths should not contain characters other than "
+        "[0-9a-zA-Z-._:/]. If not provided, the default is the current Linux epoch timestamp (optional).",
     )
     @argument(
         "recursive",
@@ -194,15 +213,13 @@ class BaseDatahub:
         "multiprocessing",
         type=int,
         positional=False,
-        description="Specify the number of files that you want to upload at any given time. "
-        "Changing the default value is not recommended (default = 1).",
+        description="Specify the number of files that you want to upload at any given time. (default = 1).",
     )
     @argument(
         "chunk_size",
         type=str,
         positional=False,
-        description="Specify the size of each file that you want to upload in megabytes (MB). "
-        "Changing the default value is not recommended (default = 8MB).",
+        description="Specify the size of each file that you want to upload in mebibyte (MiB). (default = 8MiB).",
         choices=["8", "16"],
     )
     @argument(
@@ -212,28 +229,19 @@ class BaseDatahub:
         description="Specify the stdout format (default = json).",
         choices=["json", "table", "tsv"],
     )
-    @argument(
-        "non_interactive",
-        type=bool,
-        positional=False,
-        description="Whether to run in non-interactive mode, i.e. without prompt (default = False).",
-        aliases=["ni"],
-    )
     def upload(
         self,
         workspace: str,
         src: str,
-        dst: str = "MyDir/",
+        dst: str = f"{str(int(datetime.datetime.now().timestamp()))}/",
         recursive: bool = False,
         output: str = "json",
         concurrency: int = 0,
         multiprocessing: int = 1,
         chunk_size: str = "8",
-        expiry_time: str = None,
-        non_interactive: bool = False,
     ) -> int:
         """
-        Upload data from the local file system to a SeqsLab Data Hub cloud service.
+        Upload data from the local file system to a SeqsLab Data Hub cloud service.  This command behaves similarly to gsutil cp command.
         """
 
         def __log(results: List[dict], output: str):
@@ -249,12 +257,26 @@ class BaseDatahub:
                 else:
                     logging.info(msg)
 
-        if not workspace:
-            logging.error("Invalid workspace name")
-            cprint("Enter a valid workspace name.", "red")
+        if not self._valide_workspace(workspace):
+            logging.error("Workspace not found")
+            cprint(f"Workspace {workspace} not found.", "red")
             return errno.EINVAL
 
-        if dst.isspace() or len(dst) == 0:
+        if dst:
+            # Standard Library
+            import re
+
+            invalid_patterns = [
+                r"(^|/|\\)\.{1,}($|/|\\|$)|^\s*$|^\/$|^\.$|^/|//",  # ex: ., .., ..., ./, ../, .../, ././. etc
+                r"[^0-9a-zA-Z\-\._:/]+",
+                # ex: Only alphanumeric characters, hyphen, period, colon, and underscore are allowed.
+            ]
+            for pattern in invalid_patterns:
+                if re.search(pattern, dst):
+                    logging.error(f"Invalid dst path {dst}")
+                    cprint(f"Invalid dst path {dst}.", "red")
+                    return errno.EINVAL
+        else:
             logging.error("Invalid dst path")
             cprint("Enter a valid dst path.", "red")
             return errno.EINVAL
@@ -262,19 +284,7 @@ class BaseDatahub:
         if os.path.isdir(src):
             if src.endswith("/"):
                 src = URL(f'{str(src).rstrip("/")}', encoded=True)
-        if expiry_time:
-            # Standard Library
-            import re
-
-            pattern = r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}"
-            if not re.fullmatch(pattern, expiry_time):
-                cprint(
-                    "Make sure expiry_time format is correct. e.g. 2023-12-31 00:00:00",
-                    "red",
-                )
-                return errno.EINVAL
 
-        dst = "/" if dst == "." else dst
         result = self._upload(
             src=URL(src, encoded=True),
             dst=URL(dst, encoded=True),
@@ -284,8 +294,6 @@ class BaseDatahub:
             concurrency=concurrency,
             chunk_size=self.size[chunk_size],
             proxy=self.proxy,
-            non_interactive=non_interactive,
-            expiry_time=expiry_time,
         )
         if isinstance(result, int):
             return result
@@ -322,23 +330,20 @@ class BaseDatahub:
         type=int,
         positional=False,
         description="Specify the numbers of chunks that can be transferred concurrently. The download bandwidth "
-        "can be estimated using the formula concurrency * chunk_size. "
-        "Changing the default value is not recommended (optional, default = 120).",
+        "can be estimated using the formula concurrency * chunk_size. (optional, default = 120).",
     )
     @argument(
         "multiprocessing",
         type=int,
         positional=False,
-        description="Specify the number of files that you want to upload at any given time. "
-        "Changing the default value is not recommended (default = 1).",
+        description="Specify the number of files that you want to upload at any given time. (default = 1).",
     )
     @argument(
         "chunk_size",
         type=str,
         positional=False,
         description="Specify the size of chunk in the chunked downloading process. "
-        "If the specified size is greater than 4MB, the md5 hashes are not checked. "
-        "Changing the default value is not recommended (optional, default = 4MB).",
+        "If the specified size is greater than 4MiB, the md5 hashes are not checked. (optional, default = 4MiB).",
         choices=["4", "8", "16", "32"],
     )
     @argument(
@@ -384,9 +389,9 @@ class BaseDatahub:
                     else:
                         logging.info(msg)
 
-        if not workspace:
-            logging.error("Invalid workspace name")
-            cprint("Invalid workspace name", "red")
+        if not self._valide_workspace(workspace):
+            logging.error("Workspace not found")
+            cprint(f"Workspace {workspace} not found.", "red")
             return errno.EINVAL
 
         tasks = []
@@ -461,9 +466,9 @@ class BaseDatahub:
         browsers. The share link expires either after the first successful download or after a period of three months.
         """
 
-        if not workspace:
-            logging.error("Invalid workspace name")
-            cprint("Invalid workspace name", "red")
+        if not self._valide_workspace(workspace):
+            logging.error("Workspace not found")
+            cprint("Workspace not found.", "red")
             return errno.EINVAL
 
         api_backend = drs_register().load_register(workspace)
@@ -663,6 +668,10 @@ class BaseDatahub:
         """
         Register a DRS object as a blob.
         """
+        if not self._valide_workspace(workspace):
+            logging.error("Workspace not found")
+            cprint("Workspace not found.", "red")
+            return errno.EINVAL
 
         def __log(results: List[dict], output: str):
             self._stdout(results=results, output=output)
@@ -725,17 +734,21 @@ class BaseDatahub:
                     "mime_type": kwargs.get("mime_type"),
                     "file_type": kwargs.get("file_type"),
                     "created_time": kwargs.get("created_time"),
-                    "updated_time": kwargs.get("updated_time")
-                    if kwargs.get("updated_time")
-                    else kwargs.get("created_time"),
+                    "updated_time": (
+                        kwargs.get("updated_time")
+                        if kwargs.get("updated_time")
+                        else kwargs.get("created_time")
+                    ),
                     "size": kwargs.get("size"),
                     "access_methods": json.loads(kwargs.get("access_methods")),
                     "checksums": checksums,
                     "description": kwargs.get("description"),
                     "aliases": kwargs.get("aliases"),
-                    "metadata": json.loads(kwargs.get("metadata"))
-                    if kwargs.get("metadata")
-                    else None,
+                    "metadata": (
+                        json.loads(kwargs.get("metadata"))
+                        if kwargs.get("metadata")
+                        else None
+                    ),
                     "tags": kwargs.get("tags"),
                     "id": kwargs.get("id"),
                     "deleted_time": kwargs.get("deleted_time"),
@@ -828,8 +841,9 @@ class BaseDatahub:
                 msg = f"{r['id']}  has been registered."
                 logging.info(msg)
 
-        if not workspace:
-            cprint("Enter a valid workspace", "red")
+        if not self._valide_workspace(workspace):
+            logging.error("Workspace not found")
+            cprint("Workspace not found.", "red")
             return errno.EINVAL
 
         results = self._register_bundle(
@@ -856,9 +870,11 @@ class BaseDatahub:
                     "file_type": "bundle",
                     "description": kwargs.get("description"),
                     "aliases": kwargs.get("aliases"),
-                    "metadata": json.loads(kwargs.get("metadata"))
-                    if kwargs.get("metadata")
-                    else None,
+                    "metadata": (
+                        json.loads(kwargs.get("metadata"))
+                        if kwargs.get("metadata")
+                        else None
+                    ),
                     "tags": kwargs.get("tags"),
                     "id": kwargs.get("id"),
                     "contents": drs_ids,
@@ -872,7 +888,7 @@ class BaseDatahub:
 
     @command(aliases=["deregister"])
     @argument(
-        "ids",
+        "id",
         type=List[str],
         positional=False,
         description="Specify the IDs of the DRS objects that you want to delete "
@@ -894,14 +910,14 @@ class BaseDatahub:
     )
     def deregister(
         self,
-        ids: List[str] = [],
+        id: List[str] = [],
         tags: List[str] = [],
         names: List[str] = [],
     ) -> int:
         """
         Deregister DRS objects by DRS ID, DRS name, or DRS tag.
         """
-        if not ids and not names and not tags:
+        if not id and not names and not tags:
             cprint(
                 "Must specify one of the IDs, names or tags to identify DRS objects for deletion",
                 "red",
@@ -909,7 +925,7 @@ class BaseDatahub:
             return errno.ENOENT
 
         resps = asyncio.run(
-            utils.drs_delete(ids, names, tags, query_opts="?backend_content=false")
+            utils.drs_delete(id, names, tags, query_opts="?backend_content=false")
         )
         for r in resps:
             cprint(r, "yellow")
@@ -918,7 +934,7 @@ class BaseDatahub:
 
     @command(aliases=["clean"])
     @argument(
-        "ids",
+        "id",
         type=List[str],
         positional=False,
         description="Specify the IDs of the DRS objects that you want to delete "
@@ -940,14 +956,14 @@ class BaseDatahub:
     )
     def delete(
         self,
-        ids: List[str] = [],
+        id: List[str] = [],
         tags: List[str] = [],
         names: List[str] = [],
     ) -> int:
         """
         Delete DRS objects and the associated files in cloud storage by DRS ID, DRS name, or DRS tag.
         """
-        if not ids and not names and not tags:
+        if not id and not names and not tags:
             cprint(
                 "Must specify one of the IDs, names or tags to identify "
                 "the associated files in cloud storage of DRS objects for deletion",
@@ -956,7 +972,7 @@ class BaseDatahub:
             return errno.ENOENT
 
         resps = asyncio.run(
-            utils.drs_delete(ids, names, tags, query_opts="?backend_content=true")
+            utils.drs_delete(id, names, tags, query_opts="?backend_content=true")
         )
         for r in resps:
             cprint(r, "yellow")
@@ -1202,24 +1218,15 @@ class BaseDatahub:
         "multiprocessing",
         type=int,
         positional=False,
-        description="Specify the number of files that you want to upload at any given time. "
-        "Changing the default value is not recommended (optional, default = 1).",
+        description="Specify the number of files that you want to upload at any given time. (optional, default = 1).",
     )
     @argument(
         "chunk_size",
         type=str,
         positional=False,
-        description="Specify the size of each file that you want to upload in megabytes (MB). "
-        "Changing the default value is not recommended (default = 8MB).",
+        description="Specify the size of each file that you want to upload in mebibyte (MiB). (default = 8MiB).",
         choices=["8", "16"],
     )
-    @argument(
-        "non_interactive",
-        type=bool,
-        positional=False,
-        description="Whether to run in non-interactive mode, i.e. without prompt (default = False).",
-        aliases=["ni"],
-    )
     @argument(
         "seq_run_id",
         type=str,
@@ -1237,17 +1244,16 @@ class BaseDatahub:
         concurrency: int = 0,
         multiprocessing: int = 1,
         chunk_size: str = "8",
-        expiry_time: str = None,
-        non_interactive: bool = False,
         seq_run_id: str = "",
     ) -> int:
         """
         Upload samples based on the Run Sheet and FASTQ path information.
         """
-        if not workspace:
-            logging.error("Invalid workspace name")
-            cprint("Invalid workspace name", "red")
+        if not self._valide_workspace(workspace):
+            logging.error("Workspace not found")
+            cprint("Workspace not found.", "red")
             return errno.EINVAL
+
         failed = False
         if not upload_dst:
             upload_dst = int(datetime.datetime.timestamp(datetime.datetime.now()))
@@ -1282,8 +1288,7 @@ class BaseDatahub:
                 concurrency=concurrency,
                 chunk_size=self.size[chunk_size],
                 proxy=self.proxy,
-                non_interactive=non_interactive,
-                expiry_time=expiry_time,
+                non_interactive=False,
             )
             if isinstance(result, list):
                 r = result[0]
@@ -1375,8 +1380,9 @@ class BaseDatahub:
         """
         Update a DRS object.
         """
-        if not workspace:
-            cprint("Enter a valid workspace", "red")
+        if not self._valide_workspace(workspace):
+            logging.error("Workspace not found")
+            cprint("Workspace not found.", "red")
             return errno.EINVAL
 
         if stdin:
@@ -1427,8 +1433,9 @@ class BaseDatahub:
         """
         Get a DRS object.
         """
-        if not workspace:
-            cprint("Enter a valid workspace", "red")
+        if not self._valide_workspace(workspace):
+            logging.error("Workspace not found")
+            cprint("Workspace not found.", "red")
             return errno.EINVAL
 
         def __log(results: dict):

{seqslab-cli-3.3.3 → seqslab-cli-3.3.5}/python/seqslab/drs/internal/aiocopy.py

@@ -193,9 +193,11 @@ async def result_setting(status: list, files: List[URL], resp_list: list) -> Lis
         name = os.path.basename(resp["dst"][0]).replace(f".{file_extension}", "")
         access_methods = [
             CopyResult.access_method(
-                access_methods_type=resp["access_methods_type"][i]
-                if resp.get("access_methods_type")
-                else None,
+                access_methods_type=(
+                    resp["access_methods_type"][i]
+                    if resp.get("access_methods_type")
+                    else None
+                ),
                 access_tier="hot",
                 dst=dst,
                 region=resp["region"],
@@ -208,7 +210,7 @@ async def result_setting(status: list, files: List[URL], resp_list: list) -> Lis
             file_type=file_extension,
             created_time=resp["created_time"],
             size=sent,
-            aliases=[os.path.basename(resp["dst"][0])],
+            aliases=[],
             access_methods=access_methods if status == "complete" else None,
             checksums=checksums if status == "complete" else None,
             status=status,
@@ -381,14 +383,17 @@ async def dir_to_blob(dir: URL, dst: URL, **kwargs) -> List[dict]:
     """
     files = []
     relpath = []
-    root_path = os.path.basename(dir.human_repr())
     for root, dirlist, filelist in os.walk(dir.human_repr()):
         if filelist:
             for file in filelist:
+                if file.startswith("."):
+                    # skip file start with '.'
+                    print(f"skip {file}")
+                    continue
                 absolute_path = os.path.join(root, file)
                 relative_path = os.path.relpath(absolute_path, dir.human_repr())
                 files.append(URL(absolute_path))
-                relpath.append(os.path.join(root_path, relative_path))
+                relpath.append(relative_path)
 
     async with get_factory().load_storage(kwargs.get("workspace")) as store:
         progress: int = 0

{seqslab-cli-3.3.3 → seqslab-cli-3.3.5}/python/seqslab/drs/storage/azure.py

@@ -517,11 +517,13 @@ class BlobStorage(BaseBlob):
                     ),
                     "region": workspace["location"],
                     "access_methods_type": workspace["resources"][0]["type"],
-                    "exception": ValueError(
-                        "Make sure the the file needs to be overwritten or rename the dst name."
-                    )
-                    if exist_size != file_size
-                    else None,
+                    "exception": (
+                        ValueError(
+                            "Make sure the the file needs to be overwritten or rename the dst name."
+                        )
+                        if exist_size != file_size
+                        else None
+                    ),
                 }
             elif os.path.exists(temp_file):
                 # start with temp

{seqslab-cli-3.3.3 → seqslab-cli-3.3.5}/python/seqslab/drs/utils/atgxmetadata.py

@@ -10,6 +10,7 @@ preparation of derivative works based upon the program or distribution of
 copies by sale, rental, lease or lending are violations of federal copyright
 laws and state trade secret laws, punishable by civil and criminal penalties.
 """
+
 # Standard Library
 import re