NGPIris 5.0.1__tar.gz → 5.1.1.dev2__tar.gz

{ngpiris-5.0.1 → ngpiris-5.1.1.dev2}/NGPIris/cli/__init__.py

@@ -1,18 +1,60 @@
 
 import click
 from click.core import Context
-from json import dumps, dump
+from json import dump
 from pathlib import Path
+from botocore.paginate import PageIterator, Paginator
+from typing import Any, Generator
+from os import get_terminal_size
+from math import floor
+from tabulate import tabulate
+from bitmath import Byte, TiB
 
 from NGPIris.hcp import HCPHandler
 
-def get_HCPHandler(context : Context)-> HCPHandler:
+def get_HCPHandler(context : Context) -> HCPHandler:
     return context.obj["hcph"]
 
 def format_list(list_of_things : list) -> str:
     list_of_buckets = list(map(lambda s : s + "\n", list_of_things))
     return "".join(list_of_buckets).strip("\n")
 
+def _list_objects_generator(hcph : HCPHandler, name_only : bool) -> Generator[str, Any, None]:
+    """
+    Handle object list as a paginator that `click` can handle. It works slightly 
+    different from `list_objects` in `hcp.py` in order to make the output 
+    printable in a terminal
+    """
+    paginator : Paginator = hcph.s3_client.get_paginator("list_objects_v2")
+    pages : PageIterator = paginator.paginate(Bucket = hcph.bucket_name)
+    (nb_of_cols, _) = get_terminal_size()
+    max_width = floor(nb_of_cols / 5)
+    if (not name_only):
+        yield tabulate(
+            [],
+            headers = ["Key", "LastModified", "ETag", "Size", "StorageClass"],
+            tablefmt = "plain",
+            stralign = "center"
+        ) + "\n" + "-"*nb_of_cols + "\n"
+    for object in pages.search("Contents[?!ends_with(Key, '/')][]"): # filter objects that does not end with "/"
+        if name_only:
+            yield str(object["Key"]) + "\n"
+        else:
+            yield tabulate(
+                [
+                    [str(object["Key"]), 
+                        str(object["LastModified"]), 
+                        str(object["ETag"]), 
+                        str(object["Size"]), 
+                        str(object["StorageClass"])]
+                ],
+                maxcolwidths = max_width,
+                tablefmt = "plain"
+            ) + "\n" + "-"*nb_of_cols + "\n"
+
+def object_is_folder(object_path : str, hcph : HCPHandler) -> bool:
+    return (object_path[-1] == "/") and (hcph.get_object(object_path)["ContentLength"] == 0)
+
 @click.group()
 @click.argument("credentials")
 @click.version_option(package_name = "NGPIris")
@@ -28,52 +70,90 @@ def cli(context : Context, credentials : str):
 
 @cli.command()
 @click.argument("bucket")
-@click.argument("file-or-folder")
+@click.argument("source")
+@click.argument("destination")
 @click.pass_context
-def upload(context : Context, bucket : str, file_or_folder : str):
+def upload(context : Context, bucket : str, source : str, destination : str):
     """
     Upload files to an HCP bucket/namespace. 
     
     BUCKET is the name of the upload destination bucket.
 
-    FILE-OR-FOLDER is the path to the file or folder of files to be uploaded.
+    SOURCE is the path to the file or folder of files to be uploaded.
+    
+    DESTINATION is the destination path on the HCP. 
     """
     hcph : HCPHandler = get_HCPHandler(context)
     hcph.mount_bucket(bucket)
-    if Path(file_or_folder).is_dir():
-        hcph.upload_folder(file_or_folder)
+    if Path(source).is_dir():
+        hcph.upload_folder(source, destination)
     else:
-        hcph.upload_file(file_or_folder)
+        hcph.upload_file(source, destination)
 
 @cli.command()
 @click.argument("bucket")
-@click.argument("object_path")
-@click.argument("local_path")
+@click.argument("source")
+@click.argument("destination")
 @click.option(
     "-f", 
     "--force", 
-    help = "Overwrite existing file with the same name", 
+    help = "Overwrite existing file with the same name (single file download only)", 
+    is_flag = True
+)
+@click.option(
+    "-iw", 
+    "--ignore_warning", 
+    help = "Ignore the download limit", 
     is_flag = True
 )
 @click.pass_context
-def download(context : Context, bucket : str, object_path : str, local_path : str, force : bool):
+def download(context : Context, bucket : str, source : str, destination : str, force : bool, ignore_warning : bool):
     """
-    Download a file from an HCP bucket/namespace.
+    Download a file or folder from an HCP bucket/namespace.
 
-    BUCKET is the name of the upload destination bucket.
+    BUCKET is the name of the download source bucket.
 
-    OBJECT_PATH is the path to the object to be downloaded.
+    SOURCE is the path to the object or object folder to be downloaded.
 
-    LOCAL_PATH is the folder where the downloaded object is to be stored locally.
+    DESTINATION is the folder where the downloaded object or object folder is to be stored locally. 
     """
-    if not Path(local_path).exists():
-        Path(local_path).mkdir()
-    downloaded_object_path = Path(local_path) / Path(object_path).name
-    if downloaded_object_path.exists() and not force:
-        exit("Object already exists. If you wish to overwrite the existing file, use the -f, --force option")
     hcph : HCPHandler = get_HCPHandler(context)
     hcph.mount_bucket(bucket)
-    hcph.download_file(object_path, downloaded_object_path.as_posix())
+    if not Path(destination).exists():
+        Path(destination).mkdir()
+    
+    if object_is_folder(source, hcph):
+        if source == "/":
+            source = ""
+
+        cumulative_download_size = Byte(0)
+        if not ignore_warning:
+            click.echo("Computing download size...")
+            for object in hcph.list_objects(source):
+                object : dict
+                cumulative_download_size += Byte(object["Size"])
+                if cumulative_download_size >= TiB(1):
+                    click.echo("WARNING: You are about to download more than 1 TB of data. Is this your intention? [y/N]: ", nl = False)
+                    inp = click.getchar(True)
+                    if inp == "y" or inp == "Y":
+                        break
+                    else: # inp == "n" or inp == "N" or something else
+                        exit("\nAborting download")
+    
+        hcph.download_folder(source, Path(destination).as_posix())
+    else: 
+        if Byte(hcph.get_object(source)["ContentLength"]) >= TiB(1):
+            click.echo("WARNING: You are about to download more than 1 TB of data. Is this your intention? [y/N]: ", nl = False)
+            inp = click.getchar(True)
+            if inp == "y" or inp == "Y":
+                pass
+            else: # inp == "n" or inp == "N" or something else
+                exit("\nAborting download")
+
+        downloaded_source = Path(destination) / Path(source).name
+        if downloaded_source.exists() and not force:
+            exit("Object already exists. If you wish to overwrite the existing file, use the -f, --force option")
+        hcph.download_file(source, downloaded_source.as_posix())
 
 @cli.command()
 @click.argument("bucket")
@@ -133,14 +213,7 @@ def list_objects(context : Context, bucket : str, name_only : bool):
     """
     hcph : HCPHandler = get_HCPHandler(context)
     hcph.mount_bucket(bucket)
-    objects_list = hcph.list_objects(name_only)
-    if name_only:
-        click.echo(format_list(objects_list))
-    else: 
-        out = []
-        for d in objects_list:
-            out.append(dumps(d, indent = 4, default = str) + "\n")
-        click.echo("".join(out))
+    click.echo_via_pager(_list_objects_generator(hcph, name_only))
 
 @cli.command()
 @click.argument("bucket")

ngpiris-5.1.1.dev2/NGPIris/hcp/exceptions.py

@@ -0,0 +1,23 @@
+class VPNConnectionError(Exception):
+    pass
+
+class NoBucketMounted(Exception):
+    pass
+
+class BucketNotFound(Exception):
+    pass
+
+class BucketForbidden(Exception):
+    pass
+
+class ObjectAlreadyExist(Exception):
+    pass
+
+class ObjectDoesNotExist(Exception):
+    pass
+
+class DownloadLimitReached(Exception):
+    pass
+
+class NotADirectory(Exception):
+    pass

{ngpiris-5.0.1 → ngpiris-5.1.1.dev2}/NGPIris/hcp/hcp.py

@@ -5,16 +5,26 @@ from NGPIris.hcp.helpers import (
     create_access_control_policy,
     check_mounted
 )
-from NGPIris.hcp.exceptions import *
+from NGPIris.hcp.exceptions import (
+    VPNConnectionError,
+    BucketNotFound,
+    BucketForbidden,
+    ObjectAlreadyExist,
+    ObjectDoesNotExist,
+    DownloadLimitReached,
+    NotADirectory
+)
 
 from boto3 import client
 from botocore.client import Config
+from botocore.paginate import PageIterator, Paginator
 from botocore.exceptions import EndpointConnectionError, ClientError
 from boto3.s3.transfer import TransferConfig
 from configparser import ConfigParser
 
+from pathlib import Path
+
 from os import (
-    path,
     stat,
     listdir
 )
@@ -28,6 +38,10 @@ from requests import get
 from urllib3 import disable_warnings
 from tqdm import tqdm
 
+from bitmath import TiB, Byte
+
+from typing import Generator
+
 _KB = 1024
 _MB = _KB * _KB
 
@@ -48,10 +62,15 @@ class HCPHandler:
         credentials_handler = CredentialsHandler(credentials_path)
         self.hcp = credentials_handler.hcp
         self.endpoint = "https://" + self.hcp["endpoint"]
-        tenant_parse = parse("https://{}.hcp1.vgregion.se", self.endpoint)
-        if type(tenant_parse) is Result:
-            self.tenant = str(tenant_parse[0])
-        else: # pragma: no cover
+
+        self.tenant = None
+        for endpoint_format_string in ["https://{}.ngp-fs1000.vgregion.se", "https://{}.ngp-fs2000.vgregion.se", "https://{}.ngp-fs3000.vgregion.se", "https://{}.vgregion.sjunet.org"]:
+            tenant_parse = parse(endpoint_format_string, self.endpoint) 
+            if type(tenant_parse) is Result:
+                self.tenant = str(tenant_parse[0])
+                break
+        
+        if not self.tenant:
             raise RuntimeError("Unable to parse endpoint. Make sure that you have entered the correct endpoint in your credentials JSON file. Hint: The endpoint should *not* contain \"https://\" or port numbers")
         self.base_request_url = self.endpoint + ":9090/mapi/tenants/" + self.tenant
         self.aws_access_key_id = self.hcp["aws_access_key_id"]
@@ -156,19 +175,20 @@ class HCPHandler:
         else:
             raise RuntimeError("No bucket selected. Either use `mount_bucket` first or supply the optional `bucket_name` paramter for `test_connection`")
         try:
-            response =  dict(self.s3_client.head_bucket(Bucket = bucket_name))
+            response = dict(self.s3_client.head_bucket(Bucket = bucket_name))
         except EndpointConnectionError as e: # pragma: no cover
             print(e)
             raise VPNConnectionError("Please check your connection and that you have your VPN enabled")
         except ClientError as e:
-            print(e)
-            raise BucketNotFound("Bucket \"" + bucket_name + "\" was not found")
+            status_code = e.response["ResponseMetadata"].get("HTTPStatusCode", -1)
+            match status_code:
+                case 404:
+                    raise BucketNotFound("Bucket \"" + bucket_name + "\" was not found")
+                case 403:
+                    raise BucketForbidden("Bucket \"" + bucket_name + "\" could not be accessed due to lack of permissions")
         except Exception as e: # pragma: no cover
             raise Exception(e)
             
-        if response["ResponseMetadata"].get("HTTPStatusCode", -1) != 200: # pragma: no cover
-            error_msg = "The response code from the request made at " + self.endpoint + " returned status code " + response["ResponseMetadata"]["HTTPStatusCode"]
-            raise Exception(error_msg)
         return response
         
     def mount_bucket(self, bucket_name : str) -> None:
@@ -185,6 +205,18 @@ class HCPHandler:
         self.test_connection(bucket_name = bucket_name)
         self.bucket_name = bucket_name
 
+    def create_bucket(self, bucket_name : str) -> None:
+        """
+        Create a bucket. The user in the given credentials will be the owner 
+        of the bucket
+
+        :param bucket_name: Name of the new bucket
+        :type bucket_name: str
+        """
+        self.s3_client.create_bucket(
+            Bucket = bucket_name
+        )
+
     def list_buckets(self) -> list[str]:
         """
         List all available buckets at endpoint.
@@ -198,27 +230,26 @@ class HCPHandler:
         return list_of_buckets
     
     @check_mounted
-    def list_objects(self, name_only : bool = False) -> list:
+    def list_objects(self, path_key : str = "", name_only : bool = False) -> Generator:
         """
-        List all objects in the mounted bucket
+        List all objects in the mounted bucket as a generator. If one wishes to 
+        get the result as a list, use :py:function:`list` to type cast the generator
 
-        :param name_only: If True, return only a list of the object names. If False, return the full metadata about each object. Defaults to False.
+        :param path_key: Filter string for which keys to list, specifically for finding objects in certain folders.
+        :type path_key: str, optional
+        :param name_only: If True, yield only a the object names. If False, yield the full metadata about each object. Defaults to False.
         :type name_only: bool, optional
-
-        :return: A list of of either strings or a list of object metadata (the form of a dictionary)
-        :rtype: list
-        """
-        response_list_objects = dict(self.s3_client.list_objects_v2(
-            Bucket = self.bucket_name
-        ))
-        if "Contents" not in response_list_objects.keys(): # pragma: no cover
-            return []
-        list_of_objects : list[dict] = response_list_objects["Contents"]
-        if name_only:
-            return [object["Key"] for object in list_of_objects]
-        else:
-            return list_of_objects
-    
+        :yield: A generator of all objects in a bucket
+        :rtype: Generator
+        """
+        paginator : Paginator = self.s3_client.get_paginator("list_objects_v2")
+        pages : PageIterator = paginator.paginate(Bucket = self.bucket_name)
+        for object in pages.search("Contents[?starts_with(Key, '" + path_key + "')][]"):
+            if name_only:
+                yield str(object["Key"])
+            else:
+                yield object
+                    
     @check_mounted
     def get_object(self, key : str) -> dict:
         """
@@ -264,9 +295,13 @@ class HCPHandler:
         :param key: Name of the object
         :type key: str
 
-        :param local_file_path: Path to a file on your local system where the contents of the object file can be put.
+        :param local_file_path: Path to a file on your local system where the contents of the object file can be put
         :type local_file_path: str
         """
+        try:
+            self.get_object(key)
+        except:
+            raise ObjectDoesNotExist("Could not find object", "\"" + key + "\"", "in bucket", "\"" + str(self.bucket_name) + "\"")
         try:
             file_size : int = self.s3_client.head_object(Bucket = self.bucket_name, Key = key)["ContentLength"]
             with tqdm(
@@ -284,10 +319,48 @@ class HCPHandler:
                 )
         except ClientError as e0: 
             print(str(e0))
-            raise Exception("Could not find object", "\"" + key + "\"", "in bucket", "\"" + str(self.bucket_name) + "\"")
+            raise e0
         except Exception as e: # pragma: no cover
             raise Exception(e)
 
+    @check_mounted
+    def download_folder(self, folder_key : str, local_folder_path : str, use_download_limit : bool = False, download_limit_in_bytes : Byte = TiB(1).to_Byte()) -> None:
+        """
+        Download multiple objects from a folder in the mounted bucket
+
+        :param folder_key: Name of the folder
+        :type folder_key: str
+
+        :param local_folder_path: Path to a folder on your local system where the contents of the objects can be put
+        :type local_folder_path: str
+
+        :param use_download_limit: Boolean choice for using a download limit. Defaults to False
+        :type use_download_limit: bool, optional
+
+        :param download_limit_in_bytes: The optional download limit in Byte (from the package `bitmath`). Defaults to 1 TB (`TiB(1).to_Byte()`)
+        :type download_limit_in_bytes: Byte, optional
+
+        :raises DownloadLimitReached: If download limit was reached while downloading files
+        :raises NotADirectory: If local_folder_path is not a directory
+        """
+        try:
+            self.get_object(folder_key)
+        except:
+            raise ObjectDoesNotExist("Could not find object", "\"" + folder_key + "\"", "in bucket", "\"" + str(self.bucket_name) + "\"")
+        if Path(local_folder_path).is_dir():
+            current_download_size_in_bytes = Byte(0) # For tracking download limit
+            for object in self.list_objects(folder_key): # Build the tree with directories or add files
+                p = Path(local_folder_path) / Path(object["Key"])
+                if object["Key"][-1] == "/": # If the object is a "folder"
+                    p.mkdir()
+                else: # If the object is a file
+                    current_download_size_in_bytes += Byte(object["Size"])
+                    if current_download_size_in_bytes >= download_limit_in_bytes and use_download_limit:
+                        raise DownloadLimitReached("The download limit was reached when downloading files")
+                    self.download_file(object["Key"], p.as_posix())
+        else:
+            raise NotADirectory(local_folder_path + " is not a directory")
+
     @check_mounted
     def upload_file(self, local_file_path : str, key : str = "") -> None:
         """
@@ -302,23 +375,26 @@ class HCPHandler:
         raise_path_error(local_file_path)
 
         if not key:
-            file_name = path.basename(local_file_path)
+            file_name = Path(local_file_path).name
             key = file_name
 
-        file_size : int = stat(local_file_path).st_size
-        with tqdm(
-            total = file_size, 
-            unit = "B", 
-            unit_scale = True, 
-            desc = local_file_path
-        ) as pbar:
-            self.s3_client.upload_file(
-                Filename = local_file_path, 
-                Bucket = self.bucket_name, 
-                Key = key,
-                Config = self.transfer_config,
-                Callback = lambda bytes_transferred : pbar.update(bytes_transferred)
-            )
+        if self.object_exists(key):
+            raise ObjectAlreadyExist("The object \"" + key + "\" already exist in the mounted bucket")
+        else:
+            file_size : int = stat(local_file_path).st_size
+            with tqdm(
+                total = file_size, 
+                unit = "B", 
+                unit_scale = True, 
+                desc = local_file_path
+            ) as pbar:
+                self.s3_client.upload_file(
+                    Filename = local_file_path, 
+                    Bucket = self.bucket_name, 
+                    Key = key,
+                    Config = self.transfer_config,
+                    Callback = lambda bytes_transferred : pbar.update(bytes_transferred)
+                )
 
     @check_mounted
     def upload_folder(self, local_folder_path : str, key : str = "") -> None:
@@ -355,19 +431,19 @@ class HCPHandler:
 
         deletion_dict = {"Objects": object_list}
 
-        list_of_objects_before = self.list_objects(True)
-
         response : dict = self.s3_client.delete_objects(
             Bucket = self.bucket_name,
             Delete = deletion_dict
         )
         if verbose:
             print(dumps(response, indent=4))
-        diff : set[str] = set(keys) - set(list_of_objects_before)
-        if diff:
-            does_not_exist = []
-            for key in diff:
+        
+        deleted_dict_list : list[dict] = response["Deleted"]
+        does_not_exist = []
+        for deleted_dict in deleted_dict_list:
+            if not "VersionId" in deleted_dict:
                 does_not_exist.append("- " + key + "\n")
+        if does_not_exist:
             print("The following could not be deleted because they didn't exist: \n" + "".join(does_not_exist))
     
     @check_mounted
@@ -409,7 +485,7 @@ class HCPHandler:
     @check_mounted
     def search_objects_in_bucket(self, search_string : str, case_sensitive : bool = False) -> list[str]:
         """
-        Simple search method using substrings in order to find certain objects. Case insensitive by default.
+        Simple search method using substrings in order to find certain objects. Case insensitive by default. Does not utilise the HCI
 
         :param search_string: Substring to be used in the search
         :type search_string: str
@@ -421,7 +497,7 @@ class HCPHandler:
         :rtype: list[str]
         """
         search_result : list[str] = []
-        for key in self.list_objects(True):
+        for key in self.list_objects(name_only = True):
             parse_object = search(
                 search_string, 
                 key, 

{ngpiris-5.0.1 → ngpiris-5.1.1.dev2}/NGPIris/hcp/helpers.py

@@ -26,6 +26,8 @@ def raise_path_error(path : str):
     if not p.exists(path):
         raise FileNotFoundError("\"" + path + "\"" + " does not exist")
 
+# Decorator for checking if a bucket is mounted. This is meant to be used by 
+# class methods, hence its possibly odd typing.
 T = TypeVar("T")
 P = ParamSpec("P")
 

ngpiris-5.1.1.dev2/NGPIris/hcp/statistics.py

@@ -0,0 +1,20 @@
+
+from NGPIris.hcp import HCPHandler
+from NGPIris.hcp.helpers import check_mounted
+
+
+class HCPStatistics(HCPHandler):
+    def __init__(self, credentials_path: str, use_ssl: bool = False, proxy_path: str = "", custom_config_path: str = "") -> None:
+        super().__init__(credentials_path, use_ssl, proxy_path, custom_config_path)
+
+    @check_mounted
+    def get_namespace_settings(self) -> dict:
+        return self.get_response("/namespaces/" + self.bucket_name) #type: ignore
+
+    @check_mounted
+    def get_namespace_statistics(self) -> dict:
+        return self.get_response("/namespaces/" + self.bucket_name + "/statistics") #type: ignore
+
+    @check_mounted
+    def get_namespace_permissions(self) -> dict:
+            return self.get_response("/namespaces/" + self.bucket_name + "/permissions") #type: ignore

{ngpiris-5.0.1 → ngpiris-5.1.1.dev2/NGPIris.egg-info}/PKG-INFO

@@ -1,16 +1,17 @@
 Metadata-Version: 2.1
 Name: NGPIris
-Version: 5.0.1
-Author-email: Erik Brink <erik.brink@gu.se>
+Version: 5.1.1.dev2
+Author-email: Erik Brink <erik.brink@gu.se>, Isak Sylvin <isak.sylvin@gu.se>
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: requests>=2.31.0
-Requires-Dist: urllib3==1.26.18
-Requires-Dist: requests>=2.31.0
+Requires-Dist: urllib3==1.26.19
 Requires-Dist: boto3>=1.26.76
 Requires-Dist: parse>=1.19.1
 Requires-Dist: tqdm>=4.66.2
 Requires-Dist: click>=8.1.7
+Requires-Dist: bitmath==1.3.3.1
+Requires-Dist: tabulate==0.9.0
 
 # NGP IRIS 👀
 NGP IRIS, or just Iris, is a tool for interacting with a Hitachi Content Platform (HCP) using S3 in the `boto3` package. NGP Iris is designed with two use cases in mind:
@@ -281,6 +282,19 @@ pytype
 ```shell
 pytest
 ```
+## Generating new documentation 
+With force flag `-f`:
+```shell
+cd docs/
+sphinx-apidoc ../NGPIris/ -o . -F -f 
+```
+
+Without force flag
+```shell
+cd docs/
+sphinx-apidoc ../NGPIris/ -o . -F
+```
+
 ## Compiling the documentation
 ```shell
 cd docs/