NGPIris 5.1.1.dev2__tar.gz → 5.2.0__tar.gz

{ngpiris-5.1.1.dev2 → ngpiris-5.2.0}/NGPIris/cli/__init__.py

@@ -19,14 +19,14 @@ def format_list(list_of_things : list) -> str:
     list_of_buckets = list(map(lambda s : s + "\n", list_of_things))
     return "".join(list_of_buckets).strip("\n")
 
-def _list_objects_generator(hcph : HCPHandler, name_only : bool) -> Generator[str, Any, None]:
+def _list_objects_generator(hcph : HCPHandler, path : str, name_only : bool, files_only : bool) -> Generator[str, Any, None]:
     """
     Handle object list as a paginator that `click` can handle. It works slightly 
     different from `list_objects` in `hcp.py` in order to make the output 
     printable in a terminal
     """
     paginator : Paginator = hcph.s3_client.get_paginator("list_objects_v2")
-    pages : PageIterator = paginator.paginate(Bucket = hcph.bucket_name)
+    pages : PageIterator = paginator.paginate(Bucket = hcph.bucket_name, Prefix = path)
     (nb_of_cols, _) = get_terminal_size()
     max_width = floor(nb_of_cols / 5)
     if (not name_only):
@@ -36,7 +36,11 @@ def _list_objects_generator(hcph : HCPHandler, name_only : bool) -> Generator[st
             tablefmt = "plain",
             stralign = "center"
         ) + "\n" + "-"*nb_of_cols + "\n"
-    for object in pages.search("Contents[?!ends_with(Key, '/')][]"): # filter objects that does not end with "/"
+    if files_only:
+        filter_string = "Contents[?!ends_with(Key, '/')][]" # filter objects that does not end with "/"
+    else:
+        filter_string = "Contents[*][]"
+    for object in pages.search(filter_string): 
         if name_only:
             yield str(object["Key"]) + "\n"
         else:
@@ -55,6 +59,11 @@ def _list_objects_generator(hcph : HCPHandler, name_only : bool) -> Generator[st
 def object_is_folder(object_path : str, hcph : HCPHandler) -> bool:
     return (object_path[-1] == "/") and (hcph.get_object(object_path)["ContentLength"] == 0)
 
+def add_trailing_slash(path : str) -> str:
+    if not path[-1] == "/":
+        path += "/"
+    return path
+
 @click.group()
 @click.argument("credentials")
 @click.version_option(package_name = "NGPIris")
@@ -85,9 +94,13 @@ def upload(context : Context, bucket : str, source : str, destination : str):
     """
     hcph : HCPHandler = get_HCPHandler(context)
     hcph.mount_bucket(bucket)
+    destination = add_trailing_slash(destination)
     if Path(source).is_dir():
+        source = add_trailing_slash(source)
         hcph.upload_folder(source, destination)
     else:
+        file_name = Path(source).name
+        destination += file_name
         hcph.upload_file(source, destination)
 
 @cli.command()
@@ -198,22 +211,52 @@ def list_buckets(context : Context):
 
 @cli.command()
 @click.argument("bucket")
+@click.argument("path", required = False)
 @click.option(
     "-no", 
     "--name-only", 
     help = "Output only the name of the objects instead of all the associated metadata", 
-    default = False
+    default = False,
+    is_flag = True
+)
+@click.option(
+    "-p",
+    "--pagination",
+    help = "Output as a paginator",
+    default = False,
+    is_flag = True
+)
+@click.option(
+    "-fo", 
+    "--files-only", 
+    help = "Output only file objects", 
+    default = False,
+    is_flag = True
 )
 @click.pass_context
-def list_objects(context : Context, bucket : str, name_only : bool):
+def list_objects(context : Context, bucket : str, path : str, name_only : bool, pagination : bool, files_only : bool):
     """
     List the objects in a certain bucket/namespace on the HCP.
 
     BUCKET is the name of the bucket in which to list its objects.
+
+    PATH is an optional argument for where to list the objects
     """
     hcph : HCPHandler = get_HCPHandler(context)
     hcph.mount_bucket(bucket)
-    click.echo_via_pager(_list_objects_generator(hcph, name_only))
+    if path:
+        path_with_slash = add_trailing_slash(path)
+
+        if not hcph.object_exists(path_with_slash):
+            raise RuntimeError("Path does not exist")
+    else:
+        path_with_slash = ""
+
+    if pagination:
+        click.echo_via_pager(_list_objects_generator(hcph, path_with_slash, name_only, files_only))
+    else:
+        for obj in hcph.list_objects(path_with_slash, name_only, files_only):
+            click.echo(obj)
 
 @cli.command()
 @click.argument("bucket")
@@ -222,23 +265,86 @@ def list_objects(context : Context, bucket : str, name_only : bool):
     "-cs", 
     "--case_sensitive", 
     help = "Use case sensitivity? Default value is False", 
-    default = False
+    default = False,
+    is_flag = True
+)
+@click.option(
+    "-v", 
+    "--verbose", 
+    help = "Get a verbose output of files. Default value is False, since it might be slower", 
+    default = False,
+    is_flag = True
 )
 @click.pass_context
-def simple_search(context : Context, bucket : str, search_string : str, case_sensitive : bool):
+def simple_search(context : Context, bucket : str, search_string : str, case_sensitive : bool, verbose : bool):
     """
     Make simple search using substrings in a bucket/namespace on the HCP.
 
+    NOTE: This command does not use the HCI. Instead, it uses a linear search of 
+    all the objects in the HCP. As such, this search might be slow.
+
+    BUCKET is the name of the bucket in which to make the search.
+
+    SEARCH_STRING is any string that is to be used for the search.
+    """
+    hcph : HCPHandler = get_HCPHandler(context)
+    hcph.mount_bucket(bucket)
+    list_of_results = hcph.search_in_bucket(
+        search_string, 
+        name_only = (not verbose), 
+        case_sensitive = case_sensitive
+    )
+    click.echo("Search results:")
+    for result in list_of_results:
+        click.echo(result)
+
+@cli.command()
+@click.argument("bucket")
+@click.argument("search_string")
+@click.option(
+    "-cs", 
+    "--case_sensitive", 
+    help = "Use case sensitivity? Default value is False", 
+    default = False,
+    is_flag = True
+)
+@click.option(
+    "-v", 
+    "--verbose", 
+    help = "Get a verbose output of files. Default value is False, since it might be slower", 
+    default = False,
+    is_flag = True
+)
+@click.option(
+    "-t", 
+    "--threshold", 
+    help = "Set the threshold for the fuzzy search score. Default value is 80", 
+    default = 80
+)
+@click.pass_context
+def fuzzy_search(context : Context, bucket : str, search_string : str, case_sensitive : bool, verbose : bool, threshold : int):
+    """
+    Make a fuzzy search using a search string in a bucket/namespace on the HCP.
+
+    NOTE: This command does not use the HCI. Instead, it uses the RapidFuzz 
+    library in order to find objects in the HCP. As such, this search might 
+    be slow.
+
     BUCKET is the name of the bucket in which to make the search.
 
     SEARCH_STRING is any string that is to be used for the search.
     """
     hcph : HCPHandler = get_HCPHandler(context)
     hcph.mount_bucket(bucket)
-    list_of_results = hcph.search_objects_in_bucket(search_string, case_sensitive)
+    list_of_results = hcph.fuzzy_search_in_bucket(
+        search_string, 
+        name_only = (not verbose), 
+        case_sensitive = case_sensitive,
+        threshold = threshold
+    ) 
     click.echo("Search results:")
     for result in list_of_results:
-        click.echo("- " + result)
+        click.echo(result) 
 
 @cli.command()
 @click.argument("bucket")

{ngpiris-5.1.1.dev2 → ngpiris-5.2.0}/NGPIris/hcp/hcp.py

@@ -31,9 +31,13 @@ from os import (
 from json import dumps
 from parse import (
     parse,
-    search,
     Result
 )
+from rapidfuzz import (
+    fuzz,
+    process, 
+    utils
+)
 from requests import get
 from urllib3 import disable_warnings
 from tqdm import tqdm
@@ -173,7 +177,9 @@ class HCPHandler:
         elif bucket_name:
             pass
         else:
-            raise RuntimeError("No bucket selected. Either use `mount_bucket` first or supply the optional `bucket_name` paramter for `test_connection`")
+            raise RuntimeError("No bucket selected. Either use `mount_bucket` first or supply the optional `bucket_name` parameter for `test_connection`")
+        
+        response = {}
         try:
             response = dict(self.s3_client.head_bucket(Bucket = bucket_name))
         except EndpointConnectionError as e: # pragma: no cover
@@ -230,7 +236,7 @@ class HCPHandler:
         return list_of_buckets
     
     @check_mounted
-    def list_objects(self, path_key : str = "", name_only : bool = False) -> Generator:
+    def list_objects(self, path_key : str = "", name_only : bool = False, files_only : bool = False) -> Generator:
         """
         List all objects in the mounted bucket as a generator. If one wishes to 
         get the result as a list, use :py:function:`list` to type cast the generator
@@ -239,16 +245,35 @@ class HCPHandler:
         :type path_key: str, optional
         :param name_only: If True, yield only a the object names. If False, yield the full metadata about each object. Defaults to False.
         :type name_only: bool, optional
+        :param files_only: If true, only yield file objects. Defaults to False
+        :type files_only: bool, optional
         :yield: A generator of all objects in a bucket
         :rtype: Generator
         """
         paginator : Paginator = self.s3_client.get_paginator("list_objects_v2")
-        pages : PageIterator = paginator.paginate(Bucket = self.bucket_name)
-        for object in pages.search("Contents[?starts_with(Key, '" + path_key + "')][]"):
-            if name_only:
-                yield str(object["Key"])
-            else:
-                yield object
+        pages : PageIterator = paginator.paginate(Bucket = self.bucket_name, Prefix = path_key)
+
+        if files_only:
+            filter_string = "Contents[?!ends_with(Key, '/')][]"
+        else:
+            filter_string = "Contents[*][]"
+
+        split_path_key = len(path_key.split("/")) + 1
+
+        pages_filtered = pages.search(filter_string)
+        for object in pages_filtered:
+            # Split the object key by "/"
+            split_object = object["Key"].split("/")
+            # Check if the object is within the specified path_key depth
+            if len(split_object) <= split_path_key:
+                # Skip objects that are not at the desired depth
+                if (len(split_object) == split_path_key) and split_object[-1]:
+                    continue
+                
+                if name_only:
+                    yield str(object["Key"])
+                else:
+                    yield object
                     
     @check_mounted
     def get_object(self, key : str) -> dict:
@@ -378,6 +403,9 @@ class HCPHandler:
             file_name = Path(local_file_path).name
             key = file_name
 
+        if "\\" in local_file_path:
+            raise RuntimeError("The \"\\\" character is not allowed in the file path")
+
         if self.object_exists(key):
             raise ObjectAlreadyExist("The object \"" + key + "\" already exist in the mounted bucket")
         else:
@@ -426,25 +454,24 @@ class HCPHandler:
         :type verbose: bool, optional
         """
         object_list = []
+        does_not_exist = []
         for key in keys:
-            object_list.append({"Key" : key})
-
-        deletion_dict = {"Objects": object_list}
+            if self.object_exists(key):
+                object_list.append({"Key" : key})
+            else:
+                does_not_exist.append(key)
 
-        response : dict = self.s3_client.delete_objects(
-            Bucket = self.bucket_name,
-            Delete = deletion_dict
-        )
-        if verbose:
-            print(dumps(response, indent=4))
-        
-        deleted_dict_list : list[dict] = response["Deleted"]
-        does_not_exist = []
-        for deleted_dict in deleted_dict_list:
-            if not "VersionId" in deleted_dict:
-                does_not_exist.append("- " + key + "\n")
-        if does_not_exist:
-            print("The following could not be deleted because they didn't exist: \n" + "".join(does_not_exist))
+        if object_list:
+            deletion_dict = {"Objects": object_list}
+            response : dict = self.s3_client.delete_objects(
+                Bucket = self.bucket_name,
+                Delete = deletion_dict
+            )
+            if verbose:
+                print(dumps(response, indent=4))
+            
+        if verbose and does_not_exist:
+            print("The following could not be deleted because they didn't exist: \n" + "\n".join(does_not_exist))
     
     @check_mounted
     def delete_object(self, key : str, verbose : bool = True) -> None:
@@ -461,51 +488,98 @@ class HCPHandler:
     @check_mounted
     def delete_folder(self, key : str, verbose : bool = True) -> None:
         """
-        Delete a folder of objects in the mounted bucket. If there are subfolders, a RuntimeError is raisesd
+        Delete a folder of objects in the mounted bucket. If there are subfolders, a RuntimeError is raised
 
         :param key: The folder of objects to be deleted
         :type key: str
         :param verbose: Print the result of the deletion. defaults to True
         :type verbose: bool, optional
-        :raises RuntimeError: If there are subfolders, a RuntimeError is raisesd
+        :raises RuntimeError: If there are subfolders, a RuntimeError is raised
         """
         if key[-1] != "/":
             key += "/"
-        object_path_in_folder = []
-        for s in self.search_objects_in_bucket(key):
-            parse_object = parse(key + "{}", s)
-            if type(parse_object) is Result:
-                object_path_in_folder.append(s)
-
-        for object_path in object_path_in_folder:
-            if object_path[-1] == "/":
+
+        objects : list[str] = list(self.list_objects(key, name_only = True))
+
+        if not objects:
+            raise RuntimeError("\"" + key + "\"" + " is not a valid path") #TODO: change this error
+
+        for object_path in objects:
+            if (object_path[-1] == "/") and (not object_path == key): # `objects` might contain key, in which case everything is fine
                 raise RuntimeError("There are subfolders in this folder. Please remove these first, before deleting this one")
-        self.delete_objects(object_path_in_folder + [key], verbose = verbose)
+        self.delete_objects(objects, verbose = verbose)
 
     @check_mounted
-    def search_objects_in_bucket(self, search_string : str, case_sensitive : bool = False) -> list[str]:
+    def search_in_bucket(
+        self, 
+        search_string : str, 
+        name_only : bool = True, 
+        case_sensitive : bool = False
+    ) -> Generator:
         """
-        Simple search method using substrings in order to find certain objects. Case insensitive by default. Does not utilise the HCI
+        Simple search method using exact substrings in order to find certain 
+        objects. Case insensitive by default. Does not utilise the HCI
 
         :param search_string: Substring to be used in the search
         :type search_string: str
 
+        :param name_only: If True, yield only a the object names. If False, yield the full metadata about each object. Defaults to False.
+        :type name_only: bool, optional
+
         :param case_sensitive: Case sensitivity. Defaults to False
         :type case_sensitive: bool, optional
 
-        :return: List of object names that match the in some way to the object names
-        :rtype: list[str]
+        :return: A generator of objects based on the search string
+        :rtype: Generator
+        """
+        return self.fuzzy_search_in_bucket(search_string, name_only, case_sensitive, 100)
+        
+    @check_mounted
+    def fuzzy_search_in_bucket(
+        self, 
+        search_string : str, 
+        name_only : bool = True, 
+        case_sensitive : bool = False, 
+        threshold : int = 80
+    ) -> Generator:
+        """
+        Fuzzy search implementation based on the `RapidFuzz` library.
+
+        :param search_string: Substring to be used in the search
+        :type search_string: str
+
+        :param name_only: If True, yield only a the object names. If False, yield the full metadata about each object. Defaults to False.
+        :type name_only: bool, optional
+
+        :param case_sensitive: Case sensitivity. Defaults to False
+        :type case_sensitive: bool, optional
+
+        :param threshold: The fuzzy search similarity score. Defaults to 80
+        :type threshold: int, optional
+        
+        :return: A generator of objects based on the search string
+        :rtype: Generator
         """
-        search_result : list[str] = []
-        for key in self.list_objects(name_only = True):
-            parse_object = search(
+        
+        if case_sensitive:
+            processor = None
+        else:
+            processor = utils.default_process 
+
+        if not name_only:
+            full_list = list(self.list_objects())
+
+        for item, score, index in process.extract_iter(
                 search_string, 
-                key, 
-                case_sensitive = case_sensitive
-            )
-            if type(parse_object) is Result:
-                search_result.append(key)
-        return search_result
+                self.list_objects(name_only = True), 
+                scorer = fuzz.partial_ratio,
+                processor = processor
+            ):
+            if score >= threshold:
+                if name_only:
+                    yield item
+                else:
+                    yield full_list[index] # type: ignore
 
     @check_mounted
     def get_object_acl(self, key : str) -> dict:

{ngpiris-5.1.1.dev2 → ngpiris-5.2.0}/NGPIris.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: NGPIris
-Version: 5.1.1.dev2
+Version: 5.2.0
 Author-email: Erik Brink <erik.brink@gu.se>, Isak Sylvin <isak.sylvin@gu.se>
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -8,6 +8,7 @@ Requires-Dist: requests>=2.31.0
 Requires-Dist: urllib3==1.26.19
 Requires-Dist: boto3>=1.26.76
 Requires-Dist: parse>=1.19.1
+Requires-Dist: RapidFuzz>=3.10.1
 Requires-Dist: tqdm>=4.66.2
 Requires-Dist: click>=8.1.7
 Requires-Dist: bitmath==1.3.3.1
@@ -164,116 +165,8 @@ Commands:
   test-connection  Test the connection to a bucket/namespace.
   upload           Upload files to an HCP bucket/namespace.
 ```
+Each sub-command has its own help message and is displayed by `iris path/to/your/credentials.json sub-command --help`
 
-#### The `delete-folder` command
-```
-Usage: iris CREDENTIALS delete-folder [OPTIONS] FOLDER BUCKET
-
-  Delete a folder from an HCP bucket/namespace.
-
-  FOLDER is the name of the folder to be deleted.
-
-  BUCKET is the name of the bucket where the folder to be deleted exist.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `delete-object` command
-```
-Usage: iris CREDENTIALS delete-object [OPTIONS] OBJECT BUCKET
-
-  Delete an object from an HCP bucket/namespace.
-
-  OBJECT is the name of the object to be deleted.
-
-  BUCKET is the name of the bucket where the object to be deleted exist.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `download` command
-```
-Usage: iris CREDENTIALS download [OPTIONS] OBJECT BUCKET LOCAL_PATH
-
-  Download files from an HCP bucket/namespace.
-
-  OBJECT is the name of the object to be downloaded.
-
-  BUCKET is the name of the upload destination bucket.
-
-  LOCAL_PATH is the path to where the downloaded objects are to be stored
-  locally.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `list-buckets` command
-```
-Usage: iris CREDENTIALS list-buckets [OPTIONS]
-
-  List the available buckets/namespaces on the HCP.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `list-objects` command
-```
-Usage: iris CREDENTIALS list-objects [OPTIONS] BUCKET
-
-  List the objects in a certain bucket/namespace on the HCP.
-
-  BUCKET is the name of the bucket in which to list its objects.
-
-Options:
-  -no, --name-only BOOLEAN  Output only the name of the objects instead of all
-                            the associated metadata
-  --help                    Show this message and exit.
-```
-
-#### The `simple-search` command
-```
-Usage: iris CREDENTIALS simple-search [OPTIONS] BUCKET SEARCH_STRING
-
-  Make simple search using substrings in a bucket/namespace on the HCP.
-
-  BUCKET is the name of the bucket in which to make the search.
-
-  SEARCH_STRING is any string that is to be used for the search.
-
-Options:
-  -cs, --case_sensitive BOOLEAN  Use case sensitivity?
-  --help                         Show this message and exit.
-```
-
-#### The `test-connection` command
-```
-Usage: iris CREDENTIALS test-connection [OPTIONS] BUCKET
-
-  Test the connection to a bucket/namespace.
-
-  BUCKET is the name of the bucket for which a connection test should be made.
-
-Options:
-  --help  Show this message and exit.
-```
-
-#### The `upload` command
-```
-Usage: iris CREDENTIALS upload [OPTIONS] FILE_OR_FOLDER BUCKET
-
-  Upload files to an HCP bucket/namespace.
-
-  FILE-OR-FOLDER is the path to the file or folder of files to be uploaded.
-
-  BUCKET is the name of the upload destination bucket.
-
-Options:
-  --help  Show this message and exit.
-```
 ## Testing
 Assuming that the repository has been cloned, run the following tests:
 ```shell

ngpiris-5.2.0/NGPIris.egg-info/SOURCES.txt

@@ -0,0 +1,52 @@
+LICENSE
+MANIFEST.in
+README.md
+pyproject.toml
+NGPIris/__init__.py
+NGPIris.egg-info/PKG-INFO
+NGPIris.egg-info/SOURCES.txt
+NGPIris.egg-info/dependency_links.txt
+NGPIris.egg-info/entry_points.txt
+NGPIris.egg-info/requires.txt
+NGPIris.egg-info/top_level.txt
+NGPIris/cli/__init__.py
+NGPIris/hci/__init__.py
+NGPIris/hci/exceptions.py
+NGPIris/hci/hci.py
+NGPIris/hci/helpers.py
+NGPIris/hcp/__init__.py
+NGPIris/hcp/exceptions.py
+NGPIris/hcp/hcp.py
+NGPIris/hcp/helpers.py
+NGPIris/hcp/statistics.py
+NGPIris/parse_credentials/__init__.py
+NGPIris/parse_credentials/parse_credentials.py
+NGPIris/utils/__init__.py
+NGPIris/utils/utils.py
+docs/IRIS_5_changelog.md
+docs/Makefile
+docs/NGPIris.cli.rst
+docs/NGPIris.hci.rst
+docs/NGPIris.hcp.rst
+docs/NGPIris.parse_credentials.rst
+docs/NGPIris.rst
+docs/NGPIris.utils.rst
+docs/Tutorial.md
+docs/conf.py
+docs/index.rst
+docs/make.bat
+docs/modules.rst
+tests/test_hci.py
+tests/test_hcp.py
+tests/test_parse_credentials.py
+tests/data/80MB_test_file
+tests/data/complete_creds_0.json
+tests/data/complete_creds_1.json
+tests/data/complete_creds_2.json
+tests/data/incomplete_creds_0.json
+tests/data/incomplete_creds_1.json
+tests/data/incomplete_creds_2.json
+tests/data/incomplete_creds_3.json
+tests/data/incomplete_creds_4.json
+tests/data/a folder of data/80MB_test_file_0
+tests/data/a folder of data/80MB_test_file_1

{ngpiris-5.1.1.dev2 → ngpiris-5.2.0}/NGPIris.egg-info/requires.txt

@@ -2,6 +2,7 @@ requests>=2.31.0
 urllib3==1.26.19
 boto3>=1.26.76
 parse>=1.19.1
+RapidFuzz>=3.10.1
 tqdm>=4.66.2
 click>=8.1.7
 bitmath==1.3.3.1