NGPIris 5.1.0__tar.gz → 5.1.1.dev3__tar.gz

{ngpiris-5.1.0 → ngpiris-5.1.1.dev3}/NGPIris/cli/__init__.py

@@ -222,23 +222,86 @@ def list_objects(context : Context, bucket : str, name_only : bool):
     "-cs", 
     "--case_sensitive", 
     help = "Use case sensitivity? Default value is False", 
-    default = False
+    default = False,
+    is_flag = True
+)
+@click.option(
+    "-v", 
+    "--verbose", 
+    help = "Get a verbose output of files. Default value is False, since it might be slower", 
+    default = False,
+    is_flag = True
 )
 @click.pass_context
-def simple_search(context : Context, bucket : str, search_string : str, case_sensitive : bool):
+def simple_search(context : Context, bucket : str, search_string : str, case_sensitive : bool, verbose : bool):
     """
     Make simple search using substrings in a bucket/namespace on the HCP.
 
+    NOTE: This command does not use the HCI. Instead, it uses a linear search of 
+    all the objects in the HCP. As such, this search might be slow.
+
+    BUCKET is the name of the bucket in which to make the search.
+
+    SEARCH_STRING is any string that is to be used for the search.
+    """
+    hcph : HCPHandler = get_HCPHandler(context)
+    hcph.mount_bucket(bucket)
+    list_of_results = hcph.search_in_bucket(
+        search_string, 
+        name_only = (not verbose), 
+        case_sensitive = case_sensitive
+    )
+    click.echo("Search results:")
+    for result in list_of_results:
+        click.echo(result)
+
+@cli.command()
+@click.argument("bucket")
+@click.argument("search_string")
+@click.option(
+    "-cs", 
+    "--case_sensitive", 
+    help = "Use case sensitivity? Default value is False", 
+    default = False,
+    is_flag = True
+)
+@click.option(
+    "-v", 
+    "--verbose", 
+    help = "Get a verbose output of files. Default value is False, since it might be slower", 
+    default = False,
+    is_flag = True
+)
+@click.option(
+    "-t", 
+    "--threshold", 
+    help = "Set the threshold for the fuzzy search score. Default value is 80", 
+    default = 80
+)
+@click.pass_context
+def fuzzy_search(context : Context, bucket : str, search_string : str, case_sensitive : bool, verbose : bool, threshold : int):
+    """
+    Make a fuzzy search using a search string in a bucket/namespace on the HCP.
+
+    NOTE: This command does not use the HCI. Instead, it uses the RapidFuzz 
+    library in order to find objects in the HCP. As such, this search might 
+    be slow.
+
     BUCKET is the name of the bucket in which to make the search.
 
     SEARCH_STRING is any string that is to be used for the search.
     """
     hcph : HCPHandler = get_HCPHandler(context)
     hcph.mount_bucket(bucket)
-    list_of_results = hcph.search_objects_in_bucket(search_string, case_sensitive)
+    list_of_results = hcph.fuzzy_search_in_bucket(
+        search_string, 
+        name_only = (not verbose), 
+        case_sensitive = case_sensitive,
+        threshold = threshold
+    ) 
     click.echo("Search results:")
     for result in list_of_results:
-        click.echo("- " + result)
+        click.echo(result) 
 
 @cli.command()
 @click.argument("bucket")

{ngpiris-5.1.0 → ngpiris-5.1.1.dev3}/NGPIris/hcp/hcp.py

@@ -31,9 +31,13 @@ from os import (
 from json import dumps
 from parse import (
     parse,
-    search,
     Result
 )
+from rapidfuzz import (
+    fuzz,
+    process, 
+    utils
+)
 from requests import get
 from urllib3 import disable_warnings
 from tqdm import tqdm
@@ -62,10 +66,15 @@ class HCPHandler:
         credentials_handler = CredentialsHandler(credentials_path)
         self.hcp = credentials_handler.hcp
         self.endpoint = "https://" + self.hcp["endpoint"]
-        tenant_parse = parse("https://{}.hcp1.vgregion.se", self.endpoint)
-        if type(tenant_parse) is Result:
-            self.tenant = str(tenant_parse[0])
-        else: # pragma: no cover
+
+        self.tenant = None
+        for endpoint_format_string in ["https://{}.ngp-fs1000.vgregion.se", "https://{}.ngp-fs2000.vgregion.se", "https://{}.ngp-fs3000.vgregion.se", "https://{}.vgregion.sjunet.org"]:
+            tenant_parse = parse(endpoint_format_string, self.endpoint) 
+            if type(tenant_parse) is Result:
+                self.tenant = str(tenant_parse[0])
+                break
+        
+        if not self.tenant:
             raise RuntimeError("Unable to parse endpoint. Make sure that you have entered the correct endpoint in your credentials JSON file. Hint: The endpoint should *not* contain \"https://\" or port numbers")
         self.base_request_url = self.endpoint + ":9090/mapi/tenants/" + self.tenant
         self.aws_access_key_id = self.hcp["aws_access_key_id"]
@@ -168,7 +177,7 @@ class HCPHandler:
         elif bucket_name:
             pass
         else:
-            raise RuntimeError("No bucket selected. Either use `mount_bucket` first or supply the optional `bucket_name` paramter for `test_connection`")
+            raise RuntimeError("No bucket selected. Either use `mount_bucket` first or supply the optional `bucket_name` parameter for `test_connection`")
         try:
             response = dict(self.s3_client.head_bucket(Bucket = bucket_name))
         except EndpointConnectionError as e: # pragma: no cover
@@ -421,25 +430,24 @@ class HCPHandler:
         :type verbose: bool, optional
         """
         object_list = []
+        does_not_exist = []
         for key in keys:
-            object_list.append({"Key" : key})
-
-        deletion_dict = {"Objects": object_list}
+            if self.object_exists(key):
+                object_list.append({"Key" : key})
+            else:
+                does_not_exist.append(key)
 
-        response : dict = self.s3_client.delete_objects(
-            Bucket = self.bucket_name,
-            Delete = deletion_dict
-        )
-        if verbose:
-            print(dumps(response, indent=4))
-        
-        deleted_dict_list : list[dict] = response["Deleted"]
-        does_not_exist = []
-        for deleted_dict in deleted_dict_list:
-            if not "VersionId" in deleted_dict:
-                does_not_exist.append("- " + key + "\n")
-        if does_not_exist:
-            print("The following could not be deleted because they didn't exist: \n" + "".join(does_not_exist))
+        if object_list:
+            deletion_dict = {"Objects": object_list}
+            response : dict = self.s3_client.delete_objects(
+                Bucket = self.bucket_name,
+                Delete = deletion_dict
+            )
+            if verbose:
+                print(dumps(response, indent=4))
+            
+        if verbose and does_not_exist:
+            print("The following could not be deleted because they didn't exist: \n" + "\n".join(does_not_exist))
     
     @check_mounted
     def delete_object(self, key : str, verbose : bool = True) -> None:
@@ -456,51 +464,99 @@ class HCPHandler:
     @check_mounted
     def delete_folder(self, key : str, verbose : bool = True) -> None:
         """
-        Delete a folder of objects in the mounted bucket. If there are subfolders, a RuntimeError is raisesd
+        Delete a folder of objects in the mounted bucket. If there are subfolders, a RuntimeError is raised
 
         :param key: The folder of objects to be deleted
         :type key: str
         :param verbose: Print the result of the deletion. defaults to True
         :type verbose: bool, optional
-        :raises RuntimeError: If there are subfolders, a RuntimeError is raisesd
+        :raises RuntimeError: If there are subfolders, a RuntimeError is raised
         """
         if key[-1] != "/":
             key += "/"
-        object_path_in_folder = []
-        for s in self.search_objects_in_bucket(key):
-            parse_object = parse(key + "{}", s)
-            if type(parse_object) is Result:
-                object_path_in_folder.append(s)
-
-        for object_path in object_path_in_folder:
-            if object_path[-1] == "/":
+
+        objects : list[str] = list(self.list_objects(key, name_only = True))
+
+        if not objects:
+            raise RuntimeError("\"" + key + "\"" + " is not a valid path") #TODO: change this error
+
+        for object_path in objects:
+            if (object_path[-1] == "/") and (not object_path == key): # `objects` might contain key, in which case everything is fine
                 raise RuntimeError("There are subfolders in this folder. Please remove these first, before deleting this one")
-        self.delete_objects(object_path_in_folder + [key], verbose = verbose)
+        self.delete_objects(objects, verbose = verbose)
 
     @check_mounted
-    def search_objects_in_bucket(self, search_string : str, case_sensitive : bool = False) -> list[str]:
+    def search_in_bucket(
+        self, 
+        search_string : str, 
+        name_only : bool = True, 
+        case_sensitive : bool = False
+    ) -> Generator:
         """
-        Simple search method using substrings in order to find certain objects. Case insensitive by default. Does not utilise the HCI
+        Simple search method using exact substrings in order to find certain 
+        objects. Case insensitive by default. Does not utilise the HCI
 
         :param search_string: Substring to be used in the search
         :type search_string: str
 
+        :param name_only: If True, yield only a the object names. If False, yield the full metadata about each object. Defaults to False.
+        :type name_only: bool, optional
+
         :param case_sensitive: Case sensitivity. Defaults to False
         :type case_sensitive: bool, optional
 
-        :return: List of object names that match the in some way to the object names
-        :rtype: list[str]
+        :return: A generator of objects based on the search string
+        :rtype: Generator
         """
-        search_result : list[str] = []
-        for key in self.list_objects(name_only = True):
-            parse_object = search(
+        return self.fuzzy_search_in_bucket(search_string, name_only, case_sensitive, 100)
+        
+
+    @check_mounted
+    def fuzzy_search_in_bucket(
+        self, 
+        search_string : str, 
+        name_only : bool = True, 
+        case_sensitive : bool = False, 
+        threshold : int = 80
+    ) -> Generator:
+        """
+        Fuzzy search implementation based on the `RapidFuzz` library.
+
+        :param search_string: Substring to be used in the search
+        :type search_string: str
+
+        :param name_only: If True, yield only a the object names. If False, yield the full metadata about each object. Defaults to False.
+        :type name_only: bool, optional
+
+        :param case_sensitive: Case sensitivity. Defaults to False
+        :type case_sensitive: bool, optional
+
+        :param threshold: The fuzzy search similarity score. Defaults to 80
+        :type threshold: int, optional
+        
+        :return: A generator of objects based on the search string
+        :rtype: Generator
+        """
+        
+        if case_sensitive:
+            processor = None
+        else:
+            processor = utils.default_process 
+
+        if not name_only:
+            full_list = list(self.list_objects())
+
+        for item, score, index in process.extract_iter(
                 search_string, 
-                key, 
-                case_sensitive = case_sensitive
-            )
-            if type(parse_object) is Result:
-                search_result.append(key)
-        return search_result
+                self.list_objects(name_only = True), 
+                scorer = fuzz.partial_ratio,
+                processor = processor
+            ):
+            if score >= threshold:
+                if name_only:
+                    yield item
+                else:
+                    yield full_list[index]
 
     @check_mounted
     def get_object_acl(self, key : str) -> dict:

{ngpiris-5.1.0 → ngpiris-5.1.1.dev3}/NGPIris.egg-info/PKG-INFO

@@ -1,14 +1,14 @@
 Metadata-Version: 2.1
 Name: NGPIris
-Version: 5.1.0
+Version: 5.1.1.dev3
 Author-email: Erik Brink <erik.brink@gu.se>, Isak Sylvin <isak.sylvin@gu.se>
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: requests>=2.31.0
 Requires-Dist: urllib3==1.26.19
-Requires-Dist: requests>=2.31.0
 Requires-Dist: boto3>=1.26.76
 Requires-Dist: parse>=1.19.1
+Requires-Dist: RapidFuzz>=3.10.1
 Requires-Dist: tqdm>=4.66.2
 Requires-Dist: click>=8.1.7
 Requires-Dist: bitmath==1.3.3.1
@@ -165,116 +165,8 @@ Commands:
   test-connection  Test the connection to a bucket/namespace.
   upload           Upload files to an HCP bucket/namespace.
 ```
+Each sub-command has its own help message and is displayed by `iris path/to/your/credentials.json sub-command --help`
 
-#### The `delete-folder` command
-```
-Usage: iris CREDENTIALS delete-folder [OPTIONS] FOLDER BUCKET
-
-  Delete a folder from an HCP bucket/namespace.
-
-  FOLDER is the name of the folder to be deleted.
-
-  BUCKET is the name of the bucket where the folder to be deleted exist.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `delete-object` command
-```
-Usage: iris CREDENTIALS delete-object [OPTIONS] OBJECT BUCKET
-
-  Delete an object from an HCP bucket/namespace.
-
-  OBJECT is the name of the object to be deleted.
-
-  BUCKET is the name of the bucket where the object to be deleted exist.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `download` command
-```
-Usage: iris CREDENTIALS download [OPTIONS] OBJECT BUCKET LOCAL_PATH
-
-  Download files from an HCP bucket/namespace.
-
-  OBJECT is the name of the object to be downloaded.
-
-  BUCKET is the name of the upload destination bucket.
-
-  LOCAL_PATH is the path to where the downloaded objects are to be stored
-  locally.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `list-buckets` command
-```
-Usage: iris CREDENTIALS list-buckets [OPTIONS]
-
-  List the available buckets/namespaces on the HCP.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `list-objects` command
-```
-Usage: iris CREDENTIALS list-objects [OPTIONS] BUCKET
-
-  List the objects in a certain bucket/namespace on the HCP.
-
-  BUCKET is the name of the bucket in which to list its objects.
-
-Options:
-  -no, --name-only BOOLEAN  Output only the name of the objects instead of all
-                            the associated metadata
-  --help                    Show this message and exit.
-```
-
-#### The `simple-search` command
-```
-Usage: iris CREDENTIALS simple-search [OPTIONS] BUCKET SEARCH_STRING
-
-  Make simple search using substrings in a bucket/namespace on the HCP.
-
-  BUCKET is the name of the bucket in which to make the search.
-
-  SEARCH_STRING is any string that is to be used for the search.
-
-Options:
-  -cs, --case_sensitive BOOLEAN  Use case sensitivity?
-  --help                         Show this message and exit.
-```
-
-#### The `test-connection` command
-```
-Usage: iris CREDENTIALS test-connection [OPTIONS] BUCKET
-
-  Test the connection to a bucket/namespace.
-
-  BUCKET is the name of the bucket for which a connection test should be made.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `upload` command
-```
-Usage: iris CREDENTIALS upload [OPTIONS] FILE_OR_FOLDER BUCKET
-
-  Upload files to an HCP bucket/namespace.
-
-  FILE-OR-FOLDER is the path to the file or folder of files to be uploaded.
-
-  BUCKET is the name of the upload destination bucket.
-
-Options:
-  --help  Show this message and exit.
-```
 ## Testing
 Assuming that the repository has been cloned, run the following tests:
 ```shell

{ngpiris-5.1.0 → ngpiris-5.1.1.dev3}/NGPIris.egg-info/requires.txt

@@ -1,8 +1,8 @@
 requests>=2.31.0
 urllib3==1.26.19
-requests>=2.31.0
 boto3>=1.26.76
 parse>=1.19.1
+RapidFuzz>=3.10.1
 tqdm>=4.66.2
 click>=8.1.7
 bitmath==1.3.3.1

{ngpiris-5.1.0 → ngpiris-5.1.1.dev3}/PKG-INFO

@@ -1,14 +1,14 @@
 Metadata-Version: 2.1
 Name: NGPIris
-Version: 5.1.0
+Version: 5.1.1.dev3
 Author-email: Erik Brink <erik.brink@gu.se>, Isak Sylvin <isak.sylvin@gu.se>
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: requests>=2.31.0
 Requires-Dist: urllib3==1.26.19
-Requires-Dist: requests>=2.31.0
 Requires-Dist: boto3>=1.26.76
 Requires-Dist: parse>=1.19.1
+Requires-Dist: RapidFuzz>=3.10.1
 Requires-Dist: tqdm>=4.66.2
 Requires-Dist: click>=8.1.7
 Requires-Dist: bitmath==1.3.3.1
@@ -165,116 +165,8 @@ Commands:
   test-connection  Test the connection to a bucket/namespace.
   upload           Upload files to an HCP bucket/namespace.
 ```
+Each sub-command has its own help message and is displayed by `iris path/to/your/credentials.json sub-command --help`
 
-#### The `delete-folder` command
-```
-Usage: iris CREDENTIALS delete-folder [OPTIONS] FOLDER BUCKET
-
-  Delete a folder from an HCP bucket/namespace.
-
-  FOLDER is the name of the folder to be deleted.
-
-  BUCKET is the name of the bucket where the folder to be deleted exist.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `delete-object` command
-```
-Usage: iris CREDENTIALS delete-object [OPTIONS] OBJECT BUCKET
-
-  Delete an object from an HCP bucket/namespace.
-
-  OBJECT is the name of the object to be deleted.
-
-  BUCKET is the name of the bucket where the object to be deleted exist.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `download` command
-```
-Usage: iris CREDENTIALS download [OPTIONS] OBJECT BUCKET LOCAL_PATH
-
-  Download files from an HCP bucket/namespace.
-
-  OBJECT is the name of the object to be downloaded.
-
-  BUCKET is the name of the upload destination bucket.
-
-  LOCAL_PATH is the path to where the downloaded objects are to be stored
-  locally.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `list-buckets` command
-```
-Usage: iris CREDENTIALS list-buckets [OPTIONS]
-
-  List the available buckets/namespaces on the HCP.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `list-objects` command
-```
-Usage: iris CREDENTIALS list-objects [OPTIONS] BUCKET
-
-  List the objects in a certain bucket/namespace on the HCP.
-
-  BUCKET is the name of the bucket in which to list its objects.
-
-Options:
-  -no, --name-only BOOLEAN  Output only the name of the objects instead of all
-                            the associated metadata
-  --help                    Show this message and exit.
-```
-
-#### The `simple-search` command
-```
-Usage: iris CREDENTIALS simple-search [OPTIONS] BUCKET SEARCH_STRING
-
-  Make simple search using substrings in a bucket/namespace on the HCP.
-
-  BUCKET is the name of the bucket in which to make the search.
-
-  SEARCH_STRING is any string that is to be used for the search.
-
-Options:
-  -cs, --case_sensitive BOOLEAN  Use case sensitivity?
-  --help                         Show this message and exit.
-```
-
-#### The `test-connection` command
-```
-Usage: iris CREDENTIALS test-connection [OPTIONS] BUCKET
-
-  Test the connection to a bucket/namespace.
-
-  BUCKET is the name of the bucket for which a connection test should be made.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `upload` command
-```
-Usage: iris CREDENTIALS upload [OPTIONS] FILE_OR_FOLDER BUCKET
-
-  Upload files to an HCP bucket/namespace.
-
-  FILE-OR-FOLDER is the path to the file or folder of files to be uploaded.
-
-  BUCKET is the name of the upload destination bucket.
-
-Options:
-  --help  Show this message and exit.
-```
 ## Testing
 Assuming that the repository has been cloned, run the following tests:
 ```shell

{ngpiris-5.1.0 → ngpiris-5.1.1.dev3}/README.md

@@ -149,116 +149,8 @@ Commands:
   test-connection  Test the connection to a bucket/namespace.
   upload           Upload files to an HCP bucket/namespace.
 ```
+Each sub-command has its own help message and is displayed by `iris path/to/your/credentials.json sub-command --help`
 
-#### The `delete-folder` command
-```
-Usage: iris CREDENTIALS delete-folder [OPTIONS] FOLDER BUCKET
-
-  Delete a folder from an HCP bucket/namespace.
-
-  FOLDER is the name of the folder to be deleted.
-
-  BUCKET is the name of the bucket where the folder to be deleted exist.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `delete-object` command
-```
-Usage: iris CREDENTIALS delete-object [OPTIONS] OBJECT BUCKET
-
-  Delete an object from an HCP bucket/namespace.
-
-  OBJECT is the name of the object to be deleted.
-
-  BUCKET is the name of the bucket where the object to be deleted exist.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `download` command
-```
-Usage: iris CREDENTIALS download [OPTIONS] OBJECT BUCKET LOCAL_PATH
-
-  Download files from an HCP bucket/namespace.
-
-  OBJECT is the name of the object to be downloaded.
-
-  BUCKET is the name of the upload destination bucket.
-
-  LOCAL_PATH is the path to where the downloaded objects are to be stored
-  locally.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `list-buckets` command
-```
-Usage: iris CREDENTIALS list-buckets [OPTIONS]
-
-  List the available buckets/namespaces on the HCP.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `list-objects` command
-```
-Usage: iris CREDENTIALS list-objects [OPTIONS] BUCKET
-
-  List the objects in a certain bucket/namespace on the HCP.
-
-  BUCKET is the name of the bucket in which to list its objects.
-
-Options:
-  -no, --name-only BOOLEAN  Output only the name of the objects instead of all
-                            the associated metadata
-  --help                    Show this message and exit.
-```
-
-#### The `simple-search` command
-```
-Usage: iris CREDENTIALS simple-search [OPTIONS] BUCKET SEARCH_STRING
-
-  Make simple search using substrings in a bucket/namespace on the HCP.
-
-  BUCKET is the name of the bucket in which to make the search.
-
-  SEARCH_STRING is any string that is to be used for the search.
-
-Options:
-  -cs, --case_sensitive BOOLEAN  Use case sensitivity?
-  --help                         Show this message and exit.
-```
-
-#### The `test-connection` command
-```
-Usage: iris CREDENTIALS test-connection [OPTIONS] BUCKET
-
-  Test the connection to a bucket/namespace.
-
-  BUCKET is the name of the bucket for which a connection test should be made.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `upload` command
-```
-Usage: iris CREDENTIALS upload [OPTIONS] FILE_OR_FOLDER BUCKET
-
-  Upload files to an HCP bucket/namespace.
-
-  FILE-OR-FOLDER is the path to the file or folder of files to be uploaded.
-
-  BUCKET is the name of the upload destination bucket.
-
-Options:
-  --help  Show this message and exit.
-```
 ## Testing
 Assuming that the repository has been cloned, run the following tests:
 ```shell