NGPIris 4.3.0__tar.gz → 5.0.0__tar.gz

ngpiris-5.0.0/NGPIris/__init__.py

@@ -0,0 +1,2 @@
+from NGPIris.hci import HCIHandler
+from NGPIris.hcp import HCPHandler

ngpiris-5.0.0/NGPIris/cli/__init__.py

@@ -0,0 +1,230 @@
+
+import click
+from click.core import Context
+from json import dumps, dump
+from pathlib import Path
+
+from NGPIris.hcp import HCPHandler
+
+def get_HCPHandler(context : Context)-> HCPHandler:
+    return context.obj["hcph"]
+
+def format_list(list_of_things : list) -> str:
+    list_of_buckets = list(map(lambda s : s + "\n", list_of_things))
+    return "".join(list_of_buckets).strip("\n")
+
+@click.group()
+@click.argument("credentials")
+@click.version_option(package_name = "NGPIris")
+@click.pass_context
+def cli(context : Context, credentials : str):
+    """
+    NGP Intelligence and Repository Interface Software, IRIS. 
+    
+    CREDENTIALS refers to the path to the JSON credentials file.
+    """
+    context.ensure_object(dict)
+    context.obj["hcph"] = HCPHandler(credentials)
+
+@cli.command()
+@click.argument("bucket")
+@click.argument("file-or-folder")
+@click.pass_context
+def upload(context : Context, bucket : str, file_or_folder : str):
+    """
+    Upload files to an HCP bucket/namespace. 
+    
+    BUCKET is the name of the upload destination bucket.
+
+    FILE-OR-FOLDER is the path to the file or folder of files to be uploaded.
+    """
+    hcph : HCPHandler = get_HCPHandler(context)
+    hcph.mount_bucket(bucket)
+    if Path(file_or_folder).is_dir():
+        hcph.upload_folder(file_or_folder)
+    else:
+        hcph.upload_file(file_or_folder)
+
+@cli.command()
+@click.argument("bucket")
+@click.argument("object_path")
+@click.argument("local_path")
+@click.option(
+    "-f", 
+    "--force", 
+    help = "Overwrite existing file with the same name", 
+    is_flag = True
+)
+@click.pass_context
+def download(context : Context, bucket : str, object_path : str, local_path : str, force : bool):
+    """
+    Download a file from an HCP bucket/namespace.
+
+    BUCKET is the name of the upload destination bucket.
+
+    OBJECT_PATH is the path to the object to be downloaded.
+
+    LOCAL_PATH is the folder where the downloaded object is to be stored locally.
+    """
+    if not Path(local_path).exists():
+        Path(local_path).mkdir()
+    downloaded_object_path = Path(local_path) / Path(object_path).name
+    if downloaded_object_path.exists() and not force:
+        exit("Object already exists. If you wish to overwrite the existing file, use the -f, --force option")
+    hcph : HCPHandler = get_HCPHandler(context)
+    hcph.mount_bucket(bucket)
+    hcph.download_file(object_path, downloaded_object_path.as_posix())
+
+@cli.command()
+@click.argument("bucket")
+@click.argument("object")
+@click.pass_context
+def delete_object(context : Context, bucket : str, object : str):
+    """
+    Delete an object from an HCP bucket/namespace. 
+
+    BUCKET is the name of the bucket where the object to be deleted exist.
+
+    OBJECT is the name of the object to be deleted.
+    """
+    hcph : HCPHandler = get_HCPHandler(context)
+    hcph.mount_bucket(bucket)
+    hcph.delete_object(object)
+
+@cli.command()
+@click.argument("bucket")
+@click.argument("folder")
+@click.pass_context
+def delete_folder(context : Context, bucket : str, folder : str):
+    """
+    Delete a folder from an HCP bucket/namespace. 
+
+    BUCKET is the name of the bucket where the folder to be deleted exist.
+
+    FOLDER is the name of the folder to be deleted.
+    """
+    hcph : HCPHandler = get_HCPHandler(context)
+    hcph.mount_bucket(bucket)
+    hcph.delete_folder(folder)
+
+@cli.command()
+@click.pass_context
+def list_buckets(context : Context):
+    """
+    List the available buckets/namespaces on the HCP.
+    """
+    hcph : HCPHandler = get_HCPHandler(context)
+    click.echo(format_list(hcph.list_buckets()))
+
+@cli.command()
+@click.argument("bucket")
+@click.option(
+    "-no", 
+    "--name-only", 
+    help = "Output only the name of the objects instead of all the associated metadata", 
+    default = False
+)
+@click.pass_context
+def list_objects(context : Context, bucket : str, name_only : bool):
+    """
+    List the objects in a certain bucket/namespace on the HCP.
+
+    BUCKET is the name of the bucket in which to list its objects.
+    """
+    hcph : HCPHandler = get_HCPHandler(context)
+    hcph.mount_bucket(bucket)
+    objects_list = hcph.list_objects(name_only)
+    if name_only:
+        click.echo(format_list(objects_list))
+    else: 
+        out = []
+        for d in objects_list:
+            out.append(dumps(d, indent = 4, default = str) + "\n")
+        click.echo("".join(out))
+
+@cli.command()
+@click.argument("bucket")
+@click.argument("search_string")
+@click.option(
+    "-cs", 
+    "--case_sensitive", 
+    help = "Use case sensitivity? Default value is False", 
+    default = False
+)
+@click.pass_context
+def simple_search(context : Context, bucket : str, search_string : str, case_sensitive : bool):
+    """
+    Make simple search using substrings in a bucket/namespace on the HCP.
+
+    BUCKET is the name of the bucket in which to make the search.
+
+    SEARCH_STRING is any string that is to be used for the search.
+    """
+    hcph : HCPHandler = get_HCPHandler(context)
+    hcph.mount_bucket(bucket)
+    list_of_results = hcph.search_objects_in_bucket(search_string, case_sensitive)
+    click.echo("Search results:")
+    for result in list_of_results:
+        click.echo("- " + result)
+
+@cli.command()
+@click.argument("bucket")
+@click.pass_context
+def test_connection(context : Context, bucket : str):
+    """
+    Test the connection to a bucket/namespace.
+
+    BUCKET is the name of the bucket for which a connection test should be made.
+    """
+    hcph : HCPHandler = get_HCPHandler(context)
+    click.echo(hcph.test_connection(bucket))
+
+@click.command()
+@click.option(
+    "--path",
+    help = "Path for where to put the new credentials file.",
+    default = ""
+)
+@click.option(
+    "--name",
+    help = "Custom name for the credentials file. Will filter out everything after a \".\" character, if any exist.",
+    default = "credentials"
+)
+def iris_generate_credentials_file(path : str, name : str):
+    """
+    Generate blank credentials file for the HCI and HCP. 
+
+    WARNING: This file will store sensitive information (such as passwords) in plaintext.
+    """
+    credentials_dict = {
+        "hcp" : {
+            "endpoint" : "",
+            "aws_access_key_id" : "",
+            "aws_secret_access_key" : ""
+        },
+        "hci" : {
+            "username" : "",
+            "password" : "",
+            "address" : "",
+            "auth_port" : "",
+            "api_port" : ""
+        }
+    }
+
+    name = name.split(".")[0] + ".json"
+
+    if not path[-1] == "/":
+        path += "/"
+
+    if path == ".":
+        file_path = name    
+    else:
+        file_path = path + name
+    
+    if not Path(path).is_dir():
+        Path(path).mkdir()
+        
+    with open(file_path, "w") as f:
+        dump(credentials_dict, f, indent = 4)
+
+    

ngpiris-5.0.0/NGPIris/hci/__init__.py

@@ -0,0 +1 @@
+from NGPIris.hci.hci import HCIHandler

ngpiris-5.0.0/NGPIris/hci/exceptions.py

@@ -0,0 +1,3 @@
+class VPNConnectionError(Exception):
+    pass
+

ngpiris-5.0.0/NGPIris/hci/hci.py

@@ -0,0 +1,132 @@
+
+from NGPIris.parse_credentials import CredentialsHandler
+from NGPIris.hci.helpers import (
+    get_index_response,
+    get_query_response
+)
+from NGPIris.hci.exceptions import *
+
+from requests import (
+    Response,
+    post
+)
+from urllib3 import disable_warnings
+from json import load
+
+class HCIHandler:
+    def __init__(self, credentials_path : str, use_ssl : bool = False) -> None:
+        """
+        Class for handling HCI requests.
+
+        :param credentials_path: Path to the JSON credentials file
+        :type credentials_path: str
+
+        :param use_ssl: Boolean choice between using SSL, defaults to False
+        :type use_ssl: bool, optional
+        """
+        credentials_handler = CredentialsHandler(credentials_path)
+        self.hci = credentials_handler.hci
+        self.username = self.hci["username"]
+        self.password = self.hci["password"]
+        self.address = self.hci["address"]
+        self.auth_port = self.hci["auth_port"]
+        self.api_port = self.hci["api_port"]
+        self.token = ""
+
+        self.use_ssl = use_ssl
+
+        if not self.use_ssl:
+            disable_warnings()
+    
+    def request_token(self) -> None:
+        """
+        Request a token from the HCI, which is stored in the HCIHandler object. 
+        The token is used for every operation that needs to send a request to 
+        HCI.
+
+        :raises VPNConnectionError: If there was a problem when requesting a token, a runtime error will be raised 
+        """
+        url = "https://" + self.address + ":" + self.auth_port + "/auth/oauth/"
+        data = {
+            "grant_type": "password", 
+            "username": self.username, 
+            "password": self.password,
+            "scope": "*",  
+            "client_secret": "hci-client", 
+            "client_id": "hci-client", 
+            "realm": "LOCAL"
+        }
+        try:
+            response : Response = post(url, data = data, verify = self.use_ssl)
+        except: # pragma: no cover
+            error_msg : str = "The token request made at " + url + " failed. Please check your connection and that you have your VPN enabled"
+            raise VPNConnectionError(error_msg) from None
+
+        token : str = response.json()["access_token"]
+        self.token = token
+
+    def list_index_names(self) -> list[str]:
+        """
+        Retrieve a list of all index names.
+
+        :return: A list of index names
+        :rtype: list[str]
+        """
+        response : Response = get_index_response(self.address, self.api_port, self.token, self.use_ssl)
+        return [entry["name"]for entry in response.json()]
+    
+    def look_up_index(self, index_name : str) -> dict:
+        """
+        Look up index information in the form of a dictionary by submitting 
+        the index name. Will return an empty dictionary if no index was found.
+
+        :param index_name: The index name
+        :type index_name: str
+
+        :return: A dictionary containing information about an index
+        :rtype: dict
+        """
+        response : Response = get_index_response(self.address, self.api_port, self.token, self.use_ssl)
+
+        for entry in response.json():
+            if entry["name"] == index_name:
+                return dict(entry)
+        
+        return {}
+
+    def raw_query(self, query_dict : dict[str, str]) -> dict:
+        """
+        Make query to an HCI index, with a dictionary
+
+        :param query_dict: Dictionary consisting of the query
+        :type query_dict: dict[str, str]
+
+        :return: Dictionary containing the raw query
+        :rtype: dict
+        """
+        return dict(get_query_response(
+            query_dict, 
+            self.address, 
+            self.api_port, 
+            self.token, 
+            self.use_ssl
+        ).json())
+            
+    def raw_query_from_JSON(self, query_path : str) -> dict:
+        """
+        Make query to an HCI index, with prewritten query in a JSON file
+
+        :param query_path: Path to the JSON file
+        :type query_path: str
+
+        :return: Dictionary containing the raw query
+        :rtype: dict
+        """
+        with open(query_path, "r") as inp:
+            return dict(get_query_response(
+            dict(load(inp)), 
+            self.address, 
+            self.api_port, 
+            self.token, 
+            self.use_ssl
+        ).json())

ngpiris-5.0.0/NGPIris/hci/helpers.py

@@ -0,0 +1,98 @@
+
+from json import (
+    dumps
+)
+from requests import (
+    Response,
+    get,
+    post    
+)
+
+def get_index_response(address : str, api_port : str, token : str, use_ssl : bool) -> Response:
+    """
+    Retrieve the index response given the address, API port and token.
+
+    :param address: The address where request is to be made
+    :type address: str
+
+    :param api_port: The API port at the given address
+    :type api_port: str
+
+    :param token: The HCI token 
+    :type token: str
+
+    :param use_ssl: Boolean choice of using SSL
+    :type use_ssl: bool
+
+    :return: A response containing information about the index
+    :rtype: requests.Response
+    """
+    url     : str            = "https://" + address + ":" + api_port + "/api/search/indexes/"
+    headers : dict[str, str] = {
+        "Accept": "application/json",
+        "Authorization": "Bearer " + token
+    }
+
+    response : Response = get(
+        url,
+        headers = headers,
+        verify = use_ssl
+    )
+
+    response.raise_for_status()
+
+    return response
+
+def get_query_response(
+        query_dict : dict[str, str], 
+        address : str, 
+        api_port : str, 
+        token : str, 
+        use_ssl : bool, 
+        path_extension : str = ""
+    ) -> Response:
+    """
+    Retrieve the query response given the address, API port and token.
+
+    :param query_dict: The query dictionary
+    :type query_dict: dict[str, str]
+
+    :param address: The address where request is to be made
+    :type address: str
+
+    :param api_port: The API port at the given address
+    :type api_port: str
+
+    :param token: The HCI token 
+    :type token: str
+
+    :param use_ssl: Boolean choice of using SSL
+    :type use_ssl: bool
+
+    :param path_extension: possibly extend the request URL. Used for example when making SQL requests. Defaults to ""
+    :type path_extension: str, optional
+
+    :return: A response containing information about the query
+    :rtype: requests.Response
+    """
+    if not "indexName" in query_dict.keys():
+        raise RuntimeError("Field indexName is missing in the query dictionary")
+
+    url     : str            = "https://" + address + ":" + api_port + "/api/search/query/" + path_extension
+    query   : dict[str, str] = query_dict
+    headers : dict[str, str] = {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+        "Authorization": "Bearer " + token
+    }
+    response : Response = post(
+        url, 
+        dumps(query), 
+        headers = headers, 
+        verify = use_ssl
+    )
+
+    response.raise_for_status()
+    
+    return response
+    

ngpiris-5.0.0/NGPIris/hcp/__init__.py

@@ -0,0 +1 @@
+from NGPIris.hcp.hcp import HCPHandler

ngpiris-5.0.0/NGPIris/hcp/exceptions.py

@@ -0,0 +1,8 @@
+class VPNConnectionError(Exception):
+    pass
+
+class BucketNotFound(Exception):
+    pass
+
+class NoBucketMounted(Exception):
+    pass