NGPIris 5.4.3.dev1__tar.gz → 5.5.0__tar.gz

{ngpiris-5.4.3.dev1 → ngpiris-5.5.0}/NGPIris/cli/__init__.py

@@ -6,37 +6,73 @@ from pathlib import Path
 from boto3 import set_stream_logger
 from typing import Any, Generator
 from bitmath import Byte, TiB
+import lazy_table as lt
 import sys
+import os
 
 from NGPIris.hcp import HCPHandler
 
-def get_HCPHandler(context : Context) -> HCPHandler:
-    return context.obj["hcph"]
-
-def format_list(list_of_things : list) -> str:
-    list_of_buckets = list(map(lambda s : s + "\n", list_of_things))
-    return "".join(list_of_buckets).strip("\n")
-
-def _list_objects_generator(hcph : HCPHandler, path : str, name_only : bool, files_only : bool) -> Generator[str, Any, None]:
-    """
-    Handle object list as a paginator that `click` can handle. It works slightly 
-    different from `list_objects` in `hcp.py` in order to make the output 
-    printable in a terminal
-    """
-    objects = hcph.list_objects(path, name_only, files_only)
-    for obj in objects:
-        yield str(obj) + "\n"
-
-def object_is_folder(object_path : str, hcph : HCPHandler) -> bool:
-    return (object_path[-1] == "/") and (hcph.get_object(object_path)["ContentLength"] == 0)
-
 def add_trailing_slash(path : str) -> str:
     if not path[-1] == "/":
         path += "/"
     return path
 
+def create_HCPHandler(context : Context) -> HCPHandler:
+    # hcp_credentials : str | dict[str, str], debug : bool, transfer_config : str
+
+    if context.parent:
+        parent_context = context.parent
+    else:
+        # Should never happen
+        click.echo("Something went wrong with the subcommand and parent command relation", err = True)
+        sys.exit(1)
+    
+    credentials : str | None = parent_context.params.get("credentials")
+
+    if credentials:
+        hcp_credentials = credentials
+    elif os.environ.get("NGPIRIS_CREDENTIALS_PATH", None):
+        hcp_credentials = os.environ["NGPIRIS_CREDENTIALS_PATH"]
+    else: 
+        endpoint : str = click.prompt(
+            "Please enter your tenant endpoint"
+        )
+
+        aws_access_key_id : str = click.prompt(
+            "Please enter your base64 hashed aws_access_key_id"
+        )
+
+        aws_secret_access_key : str = click.prompt(
+            "Please enter your md5 hashed aws_secret_access_key",
+            hide_input = True,
+            confirmation_prompt = True
+        )
+
+        hcp_credentials = {
+            "endpoint" : endpoint,
+            "aws_access_key_id" : aws_access_key_id,
+            "aws_secret_access_key" : aws_secret_access_key
+        }
+    
+    debug : bool | None = parent_context.params.get("debug")
+    transfer_config : str | None = parent_context.params.get("transfer_config")
+    if transfer_config:
+        hcp_h = HCPHandler(hcp_credentials, custom_config_path = transfer_config)
+    else:    
+        hcp_h = HCPHandler(hcp_credentials)
+    
+    if debug:
+        set_stream_logger(name="")
+        click.echo(hcp_h.transfer_config.__dict__)
+        
+    return hcp_h
+
 @click.group()
-@click.argument("credentials")
+@click.option(
+    "-c", 
+    "--credentials", 
+    help = "Path to a JSON file with credentials", 
+)
 @click.option(
     "--debug",
     help = "Get the debug log for running a command",
@@ -45,7 +81,7 @@ def add_trailing_slash(path : str) -> str:
 @click.option(
     "-tc", 
     "--transfer_config", 
-    help = "Use a custom transfer config for uploads or downloads", 
+    help = "Path for using a custom transfer config for uploads or downloads", 
 )
 @click.version_option(package_name = "NGPIris")
 @click.pass_context
@@ -56,18 +92,47 @@ def cli(context : Context, credentials : str, debug : bool, transfer_config : st
     CREDENTIALS refers to the path to the JSON credentials file.
     """
 
-    if transfer_config:
-        context.ensure_object(dict)
-        context.obj["hcph"] = HCPHandler(credentials, custom_config_path = transfer_config)
-    else:    
-        context.ensure_object(dict)
-        context.obj["hcph"] = HCPHandler(credentials)
+@cli.command(short_help = "This command returns a shell command that sets the `NGPIRIS_CREDENTIALS_PATH` enviroment variable depending on your shell")
+@click.argument(
+    "credentials_path", 
+    required = False
+)
+@click.option(
+    "-s",
+    "--shell", 
+    type = click.Choice(
+        ["bash", "fish", "zsh"],
+        case_sensitive = False
+    ),
+    help = "Allows for selection of shell that the produced command should support",
+    default = "bash"
+)
+@click.pass_context
+def shell_env(context : Context, credentials_path : str, shell : str):
+    """
+    NGP IRIS will look for an enviroment variable called 
+    `NGPIRIS_CREDENTIALS_PATH` when authenicating. This command returns a 
+    shell command that sets the `NGPIRIS_CREDENTIALS_PATH` enviroment variable 
+    depending on your shell. 
+    
+    NOTE: The enviroment variable will only last for this 
+    shell session, but you could set the value of `NGPIRIS_CREDENTIALS_PATH`
+    permanently via other commands if you wish to do so. 
 
-    if debug:
-        set_stream_logger(name="")
-        click.echo(
-            context.obj["hcph"].transfer_config.__dict__
-        )
+    CREDENTIALS PATH is the either absolute or relative path to your credentials 
+    JSON file
+    """
+    if not credentials_path:
+        click.prompt("Please enter the path to your credentials file")
+    
+    click.echo("Copy and paste the following command in order to set your environment variable:")
+    match shell:
+        case "bash":
+            click.echo("export NGPIRIS_CREDENTIALS_PATH=" + credentials_path)
+        case "fish":
+            click.echo("set -x NGPIRIS_CREDENTIALS_PATH " + credentials_path)
+        case "zsh":
+            click.echo("export NGPIRIS_CREDENTIALS_PATH=" + credentials_path)
 
 @cli.command()
 @click.argument("bucket")
@@ -114,7 +179,7 @@ def upload(context : Context, bucket : str, source : str, destination : str, dry
     
     upload_mode_choice = HCPHandler.UploadMode(upload_mode.lower())
 
-    hcph : HCPHandler = get_HCPHandler(context)
+    hcph : HCPHandler = create_HCPHandler(context)
     hcph.mount_bucket(bucket)
     destination = add_trailing_slash(destination)
     if Path(source).is_dir():
@@ -131,7 +196,7 @@ def upload(context : Context, bucket : str, source : str, destination : str, dry
         else:
             hcph.upload_file(source, destination, upload_mode = upload_mode_choice, equal_parts = equal_parts)
 
-@cli.command()
+@cli.command(short_help = "Download a file or folder from an HCP bucket/namespace.")
 @click.argument("bucket")
 @click.argument("source")
 @click.argument("destination")
@@ -164,7 +229,10 @@ def download(context : Context, bucket : str, source : str, destination : str, f
 
     DESTINATION is the folder where the downloaded object or object folder is to be stored locally. 
     """
-    hcph : HCPHandler = get_HCPHandler(context)
+    def object_is_folder(object_path : str, hcph : HCPHandler) -> bool:
+        return (object_path[-1] == "/") and (hcph.get_object(object_path)["ContentLength"] == 0)
+        
+    hcph : HCPHandler = create_HCPHandler(context)
     hcph.mount_bucket(bucket)
     if not Path(destination).exists():
         Path(destination).mkdir()
@@ -227,15 +295,14 @@ def delete_object(context : Context, bucket : str, object : str, dry_run : bool)
 
     OBJECT is the name of the object to be deleted.
     """
-    hcph : HCPHandler = get_HCPHandler(context)
+    hcph : HCPHandler = create_HCPHandler(context)
     hcph.mount_bucket(bucket)
     if not dry_run:
-        hcph.delete_object(object)
+        click.echo(hcph.delete_object(object))
     else: 
         click.echo("This command would delete:")
         click.echo(list(hcph.list_objects(object))[0])
 
-
 @cli.command()
 @click.argument("bucket")
 @click.argument("folder")
@@ -254,34 +321,47 @@ def delete_folder(context : Context, bucket : str, folder : str, dry_run : bool)
 
     FOLDER is the name of the folder to be deleted.
     """
-    hcph : HCPHandler = get_HCPHandler(context)
+    hcph : HCPHandler = create_HCPHandler(context)
     hcph.mount_bucket(bucket)
     if not dry_run:
-        hcph.delete_folder(folder)
+        click.echo(hcph.delete_folder(folder))
     else:
         click.echo("By deleting \"" + folder + "\", the following objects would have been deleted (not including objects in sub-folders):")
         for obj in hcph.list_objects(folder):
             click.echo(obj)
 
+@cli.command()
+@click.argument("bucket")
+@click.option(
+    "-dr", 
+    "--dry_run", 
+    help = "Simulate the command execution without making actual changes. Useful for testing and verification", 
+    is_flag = True
+)
+@click.pass_context
+def delete_bucket(context : Context, bucket : str, dry_run : bool):
+    hcph : HCPHandler = create_HCPHandler(context)
+    if not dry_run:
+        click.echo(hcph.delete_bucket(bucket))
+    else:
+        click.echo("This command would have deleted the bucket called \"" + bucket + "\"")
+
 @cli.command()
 @click.pass_context
 def list_buckets(context : Context):
     """
     List the available buckets/namespaces on the HCP.
     """
-    hcph : HCPHandler = get_HCPHandler(context)
-    click.echo(format_list(hcph.list_buckets()))
+    hcph : HCPHandler = create_HCPHandler(context)
+    click.echo(
+        "".join(
+            list(map(lambda s : s + "\n", hcph.list_buckets()))
+        ).strip("\n")
+    )
 
-@cli.command()
+@cli.command(short_help = "List the objects in a certain bucket/namespace on the HCP.")
 @click.argument("bucket")
 @click.argument("path", required = False)
-@click.option(
-    "-no", 
-    "--name-only", 
-    help = "Output only the name of the objects instead of all the associated metadata", 
-    default = False,
-    is_flag = True
-)
 @click.option(
     "-p",
     "--pagination",
@@ -296,8 +376,15 @@ def list_buckets(context : Context):
     default = False,
     is_flag = True
 )
+@click.option(
+    "-e", 
+    "--extended-information", 
+    help = "Output the fully exteded information for each object", 
+    default = False,
+    is_flag = True
+)
 @click.pass_context
-def list_objects(context : Context, bucket : str, path : str, name_only : bool, pagination : bool, files_only : bool):
+def list_objects(context : Context, bucket : str, path : str, pagination : bool, files_only : bool, extended_information : bool):
     """
     List the objects in a certain bucket/namespace on the HCP.
 
@@ -305,8 +392,23 @@ def list_objects(context : Context, bucket : str, path : str, name_only : bool,
 
     PATH is an optional argument for where to list the objects
     """
-    hcph : HCPHandler = get_HCPHandler(context)
+    def list_objects_generator(hcph : HCPHandler, path : str, files_only : bool, output_mode : HCPHandler.ListObjectsOutputMode) -> Generator[str, Any, None]:
+        """
+        Handle object list as a paginator that `click` can handle. It works slightly 
+        different from `list_objects` in `hcp.py` in order to make the output 
+        printable in a terminal
+        """
+        objects = hcph.list_objects(path, output_mode = output_mode, files_only = files_only)
+        for obj in objects:
+            yield str(obj) + "\n"
+
+    hcph : HCPHandler = create_HCPHandler(context)
     hcph.mount_bucket(bucket)
+    output_mode = (
+        HCPHandler.ListObjectsOutputMode.EXTENDED if extended_information 
+        else HCPHandler.ListObjectsOutputMode.SIMPLE
+    )
+
     if path:
         path_with_slash = add_trailing_slash(path)
 
@@ -316,12 +418,25 @@ def list_objects(context : Context, bucket : str, path : str, name_only : bool,
         path_with_slash = ""
 
     if pagination:
-        click.echo_via_pager(_list_objects_generator(hcph, path_with_slash, name_only, files_only))
+        click.echo_via_pager(
+            list_objects_generator(
+                hcph, 
+                path_with_slash, 
+                files_only, 
+                output_mode
+            )
+        )
     else:
-        for obj in hcph.list_objects(path_with_slash, name_only, files_only):
-            click.echo(obj)
+        lt.stream(
+            hcph.list_objects(
+                path_with_slash, 
+                output_mode = output_mode,
+                files_only = files_only
+            ),
+            headers = "keys"
+        )
 
-@cli.command()
+@cli.command(short_help = "Make simple search using substrings in a bucket/namespace on the HCP.")
 @click.argument("bucket")
 @click.argument("search_string")
 @click.option(
@@ -331,15 +446,8 @@ def list_objects(context : Context, bucket : str, path : str, name_only : bool,
     default = False,
     is_flag = True
 )
-@click.option(
-    "-v", 
-    "--verbose", 
-    help = "Get a verbose output of files. Default value is False, since it might be slower", 
-    default = False,
-    is_flag = True
-)
 @click.pass_context
-def simple_search(context : Context, bucket : str, search_string : str, case_sensitive : bool, verbose : bool):
+def simple_search(context : Context, bucket : str, search_string : str, case_sensitive : bool):
     """
     Make simple search using substrings in a bucket/namespace on the HCP.
 
@@ -350,18 +458,19 @@ def simple_search(context : Context, bucket : str, search_string : str, case_sen
 
     SEARCH_STRING is any string that is to be used for the search.
     """
-    hcph : HCPHandler = get_HCPHandler(context)
+    hcph : HCPHandler = create_HCPHandler(context)
     hcph.mount_bucket(bucket)
     list_of_results = hcph.search_in_bucket(
-        search_string, 
-        name_only = (not verbose), 
+        search_string,  
         case_sensitive = case_sensitive
     )
     click.echo("Search results:")
-    for result in list_of_results:
-        click.echo(result)
+    lt.stream(
+        list_of_results,
+        headers = "keys"
+    )
 
-@cli.command()
+@cli.command(short_help = "Make a fuzzy search using a search string in a bucket/namespace on the HCP.")
 @click.argument("bucket")
 @click.argument("search_string")
 @click.option(
@@ -371,13 +480,6 @@ def simple_search(context : Context, bucket : str, search_string : str, case_sen
     default = False,
     is_flag = True
 )
-@click.option(
-    "-v", 
-    "--verbose", 
-    help = "Get a verbose output of files. Default value is False, since it might be slower", 
-    default = False,
-    is_flag = True
-)
 @click.option(
     "-t", 
     "--threshold", 
@@ -385,29 +487,30 @@ def simple_search(context : Context, bucket : str, search_string : str, case_sen
     default = 80
 )
 @click.pass_context
-def fuzzy_search(context : Context, bucket : str, search_string : str, case_sensitive : bool, verbose : bool, threshold : int):
+def fuzzy_search(context : Context, bucket : str, search_string : str, case_sensitive : bool, threshold : int):
     """
     Make a fuzzy search using a search string in a bucket/namespace on the HCP.
 
     NOTE: This command does not use the HCI. Instead, it uses the RapidFuzz 
-    library in order to find objects in the HCP. As such, this search might 
-    be slow.
+    library in order to find objects in the HCP. As such, this search might be 
+    slow.
 
     BUCKET is the name of the bucket in which to make the search.
 
     SEARCH_STRING is any string that is to be used for the search.
     """
-    hcph : HCPHandler = get_HCPHandler(context)
+    hcph : HCPHandler = create_HCPHandler(context)
     hcph.mount_bucket(bucket)
     list_of_results = hcph.fuzzy_search_in_bucket(
         search_string, 
-        name_only = (not verbose), 
         case_sensitive = case_sensitive,
         threshold = threshold
     ) 
     click.echo("Search results:")
-    for result in list_of_results:
-        click.echo(result) 
+    lt.stream(
+        list_of_results,
+        headers = "keys"
+    )
 
 @cli.command()
 @click.argument("bucket")
@@ -418,7 +521,7 @@ def test_connection(context : Context, bucket : str):
 
     BUCKET is the name of the bucket for which a connection test should be made.
     """
-    hcph : HCPHandler = get_HCPHandler(context)
+    hcph : HCPHandler = create_HCPHandler(context)
     click.echo(hcph.test_connection(bucket))
 
 @click.command()
@@ -470,5 +573,3 @@ def iris_generate_credentials_file(path : str, name : str):
         
     with open(file_path, "w") as f:
         dump(credentials_dict, f, indent = 4)
-
-    

{ngpiris-5.4.3.dev1 → ngpiris-5.5.0}/NGPIris/hcp/hcp.py

@@ -20,8 +20,9 @@ from botocore.paginate import PageIterator, Paginator
 from botocore.exceptions import EndpointConnectionError, ClientError
 from boto3.s3.transfer import TransferConfig
 from configparser import ConfigParser
-
 from pathlib import Path
+from itertools import islice
+from more_itertools import peekable
 
 from os import (
     stat,
@@ -43,18 +44,18 @@ from tqdm import tqdm
 from bitmath import TiB, Byte
 
 from enum import Enum
-from typing import Generator
+from typing import Any, Generator
 
 _KB = 1024
 _MB = _KB * _KB
 
 class HCPHandler:
-    def __init__(self, credentials_path : str, use_ssl : bool = False, proxy_path : str = "", custom_config_path : str = "") -> None:
+    def __init__(self, credentials : str | dict[str, str], use_ssl : bool = False, proxy_path : str = "", custom_config_path : str = "") -> None:       
         """
         Class for handling HCP requests.
 
-        :param credentials_path: Path to the JSON credentials file
-        :type credentials_path: str
+        :param credentials: If `credentials` is a `str`, then it will be interpreted as a path to the JSON credentials file. If `credentials` is a `dict`, then a dictionary with the appropriate HCP credentials is expected: ```{"endpoint" : "", "aws_access_key_id" : "", "aws_secret_access_key" : "" }```
+        :type credentials: str | dict
         
         :param use_ssl: Boolean choice between using SSL, defaults to False
         :type use_ssl: bool, optional
@@ -62,9 +63,18 @@ class HCPHandler:
         :param custom_config_path: Path to a .ini file for customs settings regarding download and upload
         :type custom_config_path: str, optional
         """
-        credentials_handler = CredentialsHandler(credentials_path)
-        self.hcp = credentials_handler.hcp
-        self.endpoint = "https://" + self.hcp["endpoint"]
+        if type(credentials) is str:
+            credentials_handler = CredentialsHandler(credentials)
+            self.hcp = credentials_handler.hcp
+            
+            self.endpoint = "https://" + self.hcp["endpoint"]
+            self.aws_access_key_id = self.hcp["aws_access_key_id"]
+            self.aws_secret_access_key = self.hcp["aws_secret_access_key"]
+        elif type(credentials) is dict:
+            self.endpoint = "https://" + credentials["endpoint"]
+            self.aws_access_key_id = credentials["aws_access_key_id"]
+            self.aws_secret_access_key = credentials["aws_secret_access_key"]
+
 
         # A lookup table for GMC names to HCP tenant names
         gmc_tenant_map = {
@@ -98,8 +108,6 @@ class HCPHandler:
         if not self.tenant:
             raise RuntimeError("Unable to parse endpoint, \"" + self.endpoint + "\". Make sure that you have entered the correct endpoint in your credentials JSON file. Hints:\n - The endpoint should *not* contain \"https://\" or port numbers\n - Is the endpoint spelled correctly?")
         self.base_request_url = self.endpoint + ":9090/mapi/tenants/" + self.tenant
-        self.aws_access_key_id = self.hcp["aws_access_key_id"]
-        self.aws_secret_access_key = self.hcp["aws_secret_access_key"]
         self.token = self.aws_access_key_id + ":" + self.aws_secret_access_key
         self.bucket_name = None
         self.use_ssl = use_ssl
@@ -204,8 +212,7 @@ class HCPHandler:
         try:
             response = dict(self.s3_client.head_bucket(Bucket = bucket_name))
         except EndpointConnectionError as e: # pragma: no cover
-            print(e)
-            raise VPNConnectionError("Please check your connection and that you have your VPN enabled")
+            raise e
         except ClientError as e:
             status_code = e.response["ResponseMetadata"].get("HTTPStatusCode", -1)
             match status_code:
@@ -256,26 +263,36 @@ class HCPHandler:
         list_of_buckets : list[str] = response["name"]
         return list_of_buckets
 
+    class ListObjectsOutputMode(Enum):
+        SIMPLE = "simple"
+        EXTENDED = "extended"
+        NAME_ONLY = "name_only"
+
     @check_mounted
     def list_objects(
         self, 
         path_key : str = "", 
-        list_all_bucket_objects : bool = False,
-        name_only : bool = False, 
-        files_only : bool = False
-    ) -> Generator:
+        output_mode : ListObjectsOutputMode = ListObjectsOutputMode.EXTENDED,
+        files_only : bool = False,
+        list_all_bucket_objects : bool = False
+    ) -> Generator[dict[str, Any], Any, None]:
         """
         List all objects in the mounted bucket as a generator. If one wishes to 
         get the result as a list, use :py:function:`list` to type cast the generator
 
         :param path_key: Filter string for which keys to list, specifically for finding objects in certain folders. Defaults to \"the root\" of the bucket
         :type path_key: str, optional
-        :param list_all_bucket_objects: If True, the value of `path_key` will be ignored and instead will list all objects in the bucket. Defaults to False
-        :type list_all_bucket_objects: bool, optional
-        :param name_only: If True, yield only a the object names. If False, yield the full metadata about each object. Defaults to False.
-        :type name_only: bool, optional
+        :param output_mode: 
+            The upload mode of the transfer is any of the following:\n
+                    HCPHandler.ListObjectsOutputMode.SIMPLE,\n
+                    HCPHandler.ListObjectsOutputMode.EXTENDED,\n
+                    HCPHandler.ListObjectsOutputMode.NAME_ONLY\n
+            Default is EXTENDED
+        :type output_mode: ListObjectsOutputMode, optional
         :param files_only: If True, only yield file objects. Defaults to False
         :type files_only: bool, optional
+        :param list_all_bucket_objects: If True, the value of `path_key` will be ignored and instead will list all objects in the bucket. Defaults to False
+        :type list_all_bucket_objects: bool, optional
         :yield: A generator of all objects in specified folder in a bucket
         :rtype: Generator
         """
@@ -291,39 +308,45 @@ class HCPHandler:
             if not page:
                 break
 
-            if files_only:
-                for file_object in page.get("Contents", []):
-                    file_object : dict
-                    if file_object["Key"] != path_key:
-                        file_object["IsFile"] = True
-                        if name_only:
-                            yield file_object["Key"]
-                        else:
-                            yield file_object
-            else:
+            if not files_only: # Hide folder objects when flag `files_only` is True
+                # Handle folder objects before file objects
                 for folder_object in page.get("CommonPrefixes", []):
                     folder_object : dict
                     folder_object_metadata = self.get_object(folder_object["Prefix"])
-                    
-                    if name_only: 
-                        yield folder_object["Prefix"]
-                    else:
-                        yield {
-                            "Key" : folder_object["Prefix"],
-                            "LastModified" : folder_object_metadata["LastModified"],
-                            "ETag" : folder_object_metadata["ETag"],
-                            "IsFile" : False,
-                        }
-
-                for file_object in page.get("Contents", []):
-                    file_object : dict
-                    if file_object["Key"] != path_key:
-                        file_object["IsFile"] = True
-                        if name_only:
-                            yield file_object["Key"]
-                        else:
+                    match output_mode:
+                        case HCPHandler.ListObjectsOutputMode.EXTENDED:
+                            yield {
+                                "Key" : folder_object["Prefix"],
+                                "LastModified" : folder_object_metadata["LastModified"],
+                                "ETag" : folder_object_metadata["ETag"],
+                                "IsFile" : False,
+                            }
+                        case HCPHandler.ListObjectsOutputMode.SIMPLE:
+                            yield {
+                                "Key" : folder_object["Prefix"],
+                                "LastModified" : folder_object_metadata["LastModified"],
+                                "IsFile" : False,
+                            }
+                        case HCPHandler.ListObjectsOutputMode.NAME_ONLY:
+                            yield {"Key" : folder_object["Prefix"]}   
+            
+            # Handle file objects
+            for file_object in page.get("Contents", []):
+                file_object : dict
+                if file_object["Key"] != path_key:
+                    file_object["IsFile"] = True
+                    match output_mode:
+                        case HCPHandler.ListObjectsOutputMode.EXTENDED:
                             yield file_object
-        
+                        case HCPHandler.ListObjectsOutputMode.SIMPLE:
+                            yield {
+                                "Key" : file_object["Key"],
+                                "LastModified" : file_object["LastModified"],
+                                "Size" : file_object["Size"],
+                                "IsFile" : file_object["IsFile"]
+                            }
+                        case HCPHandler.ListObjectsOutputMode.NAME_ONLY:
+                            yield {"Key" : file_object["Key"]}
                     
     @check_mounted
     def get_object(self, key : str) -> dict:
@@ -406,7 +429,6 @@ class HCPHandler:
                     Config = self.transfer_config,
                 )
         except ClientError as e0: 
-            print(str(e0))
             raise e0
         except Exception as e: # pragma: no cover
             raise Exception(e)
@@ -490,10 +512,11 @@ class HCPHandler:
         :type show_progress_bar: bool, optional
 
         :param upload_mode: 
-            The upload mode of the transfer is any of the following:
-                HCPHandler.UploadMode.STANDARD,
-                HCPHandler.UploadMode.SIMPLE,
+            The upload mode of the transfer is any of the following:\n
+                HCPHandler.UploadMode.STANDARD,\n
+                HCPHandler.UploadMode.SIMPLE,\n
                 HCPHandler.UploadMode.EQUAL_PARTS\n
+            Default is STANDARD
         :type upload_mode: UploadMode, optional
 
         :param equal_parts: The number of equal parts that each file should be divided into when using the HCPHandler.UploadMode.EQUAL_PARTS mode. Default is 5
@@ -587,14 +610,15 @@ class HCPHandler:
             )
 
     @check_mounted
-    def delete_objects(self, keys : list[str], verbose : bool = True) -> None:
-        """Delete a list of objects on the mounted bucket 
+    def delete_objects(self, keys : list[str]) -> str:
+        """
+        Delete a list of objects on the mounted bucket 
 
         :param keys: List of object names to be deleted
         :type keys: list[str]
 
-        :param verbose: Print the result of the deletion. Defaults to True
-        :type verbose: bool, optional
+        :return: The result of the deletion 
+        :rtype: str 
         """
         object_list = []
         does_not_exist = []
@@ -604,45 +628,58 @@ class HCPHandler:
             else:
                 does_not_exist.append(key)
 
+        result = ""
         if object_list:
             deletion_dict = {"Objects": object_list}
             response : dict = self.s3_client.delete_objects(
                 Bucket = self.bucket_name,
                 Delete = deletion_dict
             )
-            if verbose:
-                print(dumps(response, indent=4))
             
-        if verbose and does_not_exist:
-            print("The following could not be deleted because they didn't exist: \n" + "\n".join(does_not_exist))
+            deleted_files = list(d["Key"] for d in response["Deleted"])
+            result += "The following was successfully deleted: \n" + "\n".join(deleted_files)
+        
+        if does_not_exist:
+            result += "The following could not be deleted because they didn't exist: \n" + "\n".join(does_not_exist)
+        
+        return result
     
     @check_mounted
-    def delete_object(self, key : str, verbose : bool = True) -> None:
+    def delete_object(self, key : str) -> str:
         """
         Delete a single object in the mounted bucket
 
         :param key: The object to be deleted
         :type key: str
-        :param verbose: Print the result of the deletion. Defaults to True
-        :type verbose: bool, optional
+
+        :return: The result of the deletion 
+        :rtype: str 
         """
-        self.delete_objects([key], verbose = verbose)
+        return self.delete_objects([key])
 
     @check_mounted
-    def delete_folder(self, key : str, verbose : bool = True) -> None:
+    def delete_folder(self, key : str) -> str:
         """
         Delete a folder of objects in the mounted bucket. If there are subfolders, a RuntimeError is raised
 
         :param key: The folder of objects to be deleted
         :type key: str
-        :param verbose: Print the result of the deletion. defaults to True
-        :type verbose: bool, optional
+
         :raises RuntimeError: If there are subfolders, a RuntimeError is raised
+
+        :return: The result of the deletion 
+        :rtype: str
         """
         if key[-1] != "/":
             key += "/"
 
-        objects : list[str] = list(self.list_objects(key, name_only = True))
+        objects : list[str] = list(
+            obj["Key"] for obj in 
+            self.list_objects(
+                key, 
+                output_mode = HCPHandler.ListObjectsOutputMode.NAME_ONLY
+            )
+        )
         objects.append(key) # Include the object "folder" path to be deleted
 
         if not objects:
@@ -651,13 +688,28 @@ class HCPHandler:
         for object_path in objects:
             if (object_path[-1] == "/") and (not object_path == key): # `objects` might contain key, in which case everything is fine
                 raise RuntimeError("There are subfolders in this folder. Please remove these first, before deleting this one")
-        self.delete_objects(objects, verbose = verbose)
+        
+        return self.delete_objects(objects)
+
+    def delete_bucket(self, bucket : str) -> str:
+        """
+        Delete a specified bucket
+
+        :param bucket: The bucket to be deleted
+        :type bucket: str
+        :return: The result of the deletion 
+        :rtype: str 
+        """
+        self.s3_client.delete_bucket(
+            Bucket = bucket
+        )
+        # If the deletion was not successful, `self.s3_client.delete_bucket` would have thrown an error 
+        return bucket + " was successfully deleted"
 
     @check_mounted
     def search_in_bucket(
         self, 
         search_string : str, 
-        name_only : bool = True, 
         case_sensitive : bool = False
     ) -> Generator:
         """
@@ -667,22 +719,18 @@ class HCPHandler:
         :param search_string: Substring to be used in the search
         :type search_string: str
 
-        :param name_only: If True, yield only a the object names. If False, yield the full metadata about each object. Defaults to False.
-        :type name_only: bool, optional
-
         :param case_sensitive: Case sensitivity. Defaults to False
         :type case_sensitive: bool, optional
 
         :return: A generator of objects based on the search string
         :rtype: Generator
         """
-        return self.fuzzy_search_in_bucket(search_string, name_only, case_sensitive, 100)
+        return self.fuzzy_search_in_bucket(search_string, case_sensitive, 100)
         
     @check_mounted
     def fuzzy_search_in_bucket(
         self, 
         search_string : str, 
-        name_only : bool = True, 
         case_sensitive : bool = False, 
         threshold : int = 80
     ) -> Generator:
@@ -710,20 +758,24 @@ class HCPHandler:
         else:
             processor = utils.default_process 
 
-        if not name_only:
-            full_list = list(self.list_objects(list_all_bucket_objects = True))
+        full_list = peekable(self.list_objects(list_all_bucket_objects = True))
 
-        for item, score, index in process.extract_iter(
+        full_list_names_only = peekable(
+            obj["Key"] for obj in 
+            self.list_objects(
+                output_mode = HCPHandler.ListObjectsOutputMode.NAME_ONLY, 
+                list_all_bucket_objects = True
+            )
+        )
+
+        for _, score, index in process.extract_iter(
                 search_string, 
-                self.list_objects(list_all_bucket_objects = True, name_only = True), 
+                full_list_names_only,
                 scorer = fuzz.partial_ratio,
                 processor = processor
             ):
             if score >= threshold:
-                if name_only:
-                    yield item
-                else:
-                    yield full_list[index] # type: ignore
+                yield full_list[index]
 
     @check_mounted
     def get_object_acl(self, key : str) -> dict:

{ngpiris-5.4.3.dev1 → ngpiris-5.5.0/NGPIris.egg-info}/PKG-INFO

@@ -1,18 +1,19 @@
 Metadata-Version: 2.4
 Name: NGPIris
-Version: 5.4.3.dev1
+Version: 5.5.0
 Author-email: Erik Brink <erik.brink@gu.se>, Isak Sylvin <isak.sylvin@gu.se>
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: requests>=2.31.0
-Requires-Dist: urllib3==1.26.19
+Requires-Dist: urllib3==2.5.0
 Requires-Dist: boto3==1.35.81
 Requires-Dist: parse>=1.19.1
 Requires-Dist: RapidFuzz>=3.10.1
 Requires-Dist: tqdm>=4.66.2
 Requires-Dist: click>=8.1.7
 Requires-Dist: bitmath==1.3.3.1
-Requires-Dist: tabulate==0.9.0
+Requires-Dist: lazy_table==0.2.1
+Requires-Dist: more-itertools==10.7.0
 Dynamic: license-file
 
 # NGP IRIS 👀

{ngpiris-5.4.3.dev1 → ngpiris-5.5.0}/NGPIris.egg-info/requires.txt

@@ -1,9 +1,10 @@
 requests>=2.31.0
-urllib3==1.26.19
+urllib3==2.5.0
 boto3==1.35.81
 parse>=1.19.1
 RapidFuzz>=3.10.1
 tqdm>=4.66.2
 click>=8.1.7
 bitmath==1.3.3.1
-tabulate==0.9.0
+lazy_table==0.2.1
+more-itertools==10.7.0

{ngpiris-5.4.3.dev1/NGPIris.egg-info → ngpiris-5.5.0}/PKG-INFO

@@ -1,18 +1,19 @@
 Metadata-Version: 2.4
 Name: NGPIris
-Version: 5.4.3.dev1
+Version: 5.5.0
 Author-email: Erik Brink <erik.brink@gu.se>, Isak Sylvin <isak.sylvin@gu.se>
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: requests>=2.31.0
-Requires-Dist: urllib3==1.26.19
+Requires-Dist: urllib3==2.5.0
 Requires-Dist: boto3==1.35.81
 Requires-Dist: parse>=1.19.1
 Requires-Dist: RapidFuzz>=3.10.1
 Requires-Dist: tqdm>=4.66.2
 Requires-Dist: click>=8.1.7
 Requires-Dist: bitmath==1.3.3.1
-Requires-Dist: tabulate==0.9.0
+Requires-Dist: lazy_table==0.2.1
+Requires-Dist: more-itertools==10.7.0
 Dynamic: license-file
 
 # NGP IRIS 👀

{ngpiris-5.4.3.dev1 → ngpiris-5.5.0}/pyproject.toml

@@ -1,17 +1,18 @@
 [project]
 name = "NGPIris"
-version = "5.4.3.dev1"
+version = "5.5.0"
 readme = "README.md"
 dependencies = [
     "requests >= 2.31.0",
-    "urllib3 == 1.26.19",
+    "urllib3 == 2.5.0",
     "boto3 == 1.35.81",
     "parse >= 1.19.1",
     "RapidFuzz >= 3.10.1",
     "tqdm >= 4.66.2",
     "click >= 8.1.7",
     "bitmath == 1.3.3.1",
-    "tabulate == 0.9.0"
+    "lazy_table == 0.2.1",
+    "more-itertools == 10.7.0"
 ]
 authors = [
     {name = "Erik Brink", email = "erik.brink@gu.se"},

ngpiris-5.5.0/tests/test_hci.py

@@ -0,0 +1,80 @@
+
+from conftest import CustomConfig
+from random import randint
+
+def test_list_index_names_type(custom_config : CustomConfig) -> None:
+    custom_config.hci_h.request_token()
+    list_of_indexes = custom_config.hci_h.list_index_names()
+    assert type(list_of_indexes) is list
+
+def test_look_up_all_indexes(custom_config : CustomConfig) -> None:
+    custom_config.hci_h.request_token()
+    list_of_indexes = custom_config.hci_h.list_index_names()
+    for index in list_of_indexes:
+        assert custom_config.hci_h.look_up_index(index)
+
+def test_fail_index_look_up(custom_config : CustomConfig) -> None:
+    custom_config.hci_h.request_token()    
+    assert not custom_config.hci_h.look_up_index("anIndexThatDoesNotExist")
+    
+def test_make_simple_raw_query(custom_config : CustomConfig) -> None:
+    custom_config.hci_h.request_token()
+    list_of_indexes = custom_config.hci_h.list_index_names()
+    arbitrary_index = list_of_indexes[randint(0, len(list_of_indexes) - 1)]
+    result = custom_config.hci_h.raw_query(
+        {
+            "indexName" : arbitrary_index
+        }
+    )
+    assert result["indexName"] == arbitrary_index
+
+def test_fail_raw_query(custom_config : CustomConfig) -> None:
+    custom_config.hci_h.request_token()
+    query = {}
+    try:
+        custom_config.hci_h.raw_query(query)
+    except:
+        assert True
+    else: # pragma: no cover
+        assert False
+
+def test_make_query(custom_config : CustomConfig) -> None:
+    custom_config.hci_h.request_token()
+    custom_config.hci_h.query(
+        custom_config.test_index
+    )
+
+def test_make_query_with_query_string(custom_config : CustomConfig) -> None:
+    custom_config.hci_h.request_token()
+    custom_config.hci_h.query(
+        custom_config.test_index,
+        custom_config.query_string
+    )
+
+def test_make_query_with_query_string_and_facet(custom_config : CustomConfig) -> None:
+    custom_config.hci_h.request_token()
+    if custom_config.test_facet:
+        custom_config.hci_h.query(
+            custom_config.test_index,
+            custom_config.query_string,
+            [custom_config.test_facet]
+        )
+    else: 
+        custom_config.hci_h.query(
+            custom_config.test_index,
+            custom_config.query_string,
+            []
+        )
+
+def test_make_query_with_facet(custom_config : CustomConfig) -> None:
+    custom_config.hci_h.request_token()
+    if custom_config.test_facet:
+        custom_config.hci_h.query(
+            custom_config.test_index,
+            facets = [custom_config.test_facet]
+        )
+    else:
+        custom_config.hci_h.query(
+            custom_config.test_index,
+            facets = []
+        )

ngpiris-5.4.3.dev1/tests/test_hci.py

@@ -1,73 +0,0 @@
-
-from configparser import ConfigParser
-from NGPIris.hci import HCIHandler
-from random import randint
-from json import dump
-from os import remove
-
-ini_config = ConfigParser()
-ini_config.read("tests/test_conf.ini")
-
-hci_h = HCIHandler(ini_config.get("General", "credentials_path"))
-hci_h.request_token()
-
-def test_list_index_names_type() -> None:
-    list_of_indexes = hci_h.list_index_names()
-    assert type(list_of_indexes) is list
-
-def test_look_up_all_indexes() -> None:
-    list_of_indexes = hci_h.list_index_names()
-    for index in list_of_indexes:
-        assert hci_h.look_up_index(index)
-
-def test_fail_index_look_up() -> None:
-    assert not hci_h.look_up_index("anIndexThatDoesNotExist")
-    
-def test_make_simple_raw_query() -> None:
-    list_of_indexes = hci_h.list_index_names()
-    arbitrary_index = list_of_indexes[randint(0, len(list_of_indexes) - 1)]
-    result = hci_h.raw_query(
-        {
-            "indexName" : arbitrary_index
-        }
-    )
-    assert result["indexName"] == arbitrary_index
-
-def test_raw_query_with_results() -> None:
-    query = {
-        "indexName" : "Parsed_VCF_Index",
-        "facetRequests" : [
-            {
-                "fieldName" : "ref"
-            },
-            {
-                "fieldName" : "alt"
-            }
-        ]
-    }
-    parsed_VCF_index_result : list = hci_h.raw_query(query)["facets"]
-    for i in range(len(parsed_VCF_index_result)):
-        assert parsed_VCF_index_result[i]["termCounts"] # Assert if the result is not empty
-
-def test_fail_raw_query() -> None:
-    query = {}
-    try:
-        hci_h.raw_query(query)
-    except:
-        assert True
-    else: # pragma: no cover
-        assert False
-
-def test_make_simple_raw_query_from_JSON() -> None:
-    list_of_indexes = hci_h.list_index_names()
-    arbitrary_index = list_of_indexes[randint(0, len(list_of_indexes) - 1)]
-    path = "tests/data/json_test_query.json"
-    with open(path, "w") as f:
-        query = {
-            "indexName": arbitrary_index
-        }
-        dump(query, f, indent = 4)
-    result = hci_h.raw_query_from_JSON(path)
-    assert result["indexName"] == arbitrary_index
-    remove(path)    
-