fair-cli 0.7.3__tar.gz → 0.8.0__tar.gz

{fair_cli-0.7.3 → fair_cli-0.8.0}/CITATION.cff

@@ -36,4 +36,4 @@ license: BSD-2-Clause
 message: If you use this software, please cite it using these metadata.
 repository-code: https://github.com/FAIRDataPipeline/FAIR-CLI/
 title: "The FAIR Data Pipeline command line tool"
-version: 0.7.3
+version: 0.8.0

{fair_cli-0.7.3 → fair_cli-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fair-cli
-Version: 0.7.3
+Version: 0.8.0
 Summary: Synchronization interface for the SCRC FAIR Data Pipeline registry
 Home-page: https://www.fairdatapipeline.org/
 License: BSD-2-Clause
@@ -29,7 +29,7 @@ Requires-Dist: Jinja2 (>=3.0.1,<4.0.0)
 Requires-Dist: PyYAML (>=5.4.1,<7.0.0)
 Requires-Dist: click (>=8.0.0,<9.0.0)
 Requires-Dist: email-validator (>=1.1.3,<2.0.0)
-Requires-Dist: netCDF4 (>=1.5.8,<2.0.0)
+Requires-Dist: fake-useragent (>=1,<2)
 Requires-Dist: pre-commit (>=2.15.0,<3.0.0)
 Requires-Dist: pydantic[email] (>=1.9.0,<2.0.0)
 Requires-Dist: requests (>=2.23.0,<3.0.0)

{fair_cli-0.7.3 → fair_cli-0.8.0}/fair/cli.py

@@ -105,8 +105,7 @@ def status(verbose, debug) -> None:
     except fdp_exc.FAIRCLIException as e:
         e.err_print()
         if e.level.lower() == "error":
-            if e.level.lower() == "error":
-                sys.exit(e.exit_code)
+            sys.exit(e.exit_code)
 
 @cli.group(invoke_without_command=True)
 @click.option("--debug/--no-debug", help="Run in debug mode", default=False)
@@ -118,8 +117,10 @@ def list(ctx, debug, remote) -> None:
         ctx.obj = {}
     ctx.obj['DEBUG'] = debug
     ctx.obj['REMOTE'] = remote
-    ctx.invoke(data_products)
-    ctx.invoke(code_runs)
+    _current_args = " ".join(sys.argv)
+    if not ("data-products" in _current_args or "code-runs" in _current_args):
+        ctx.invoke(data_products)
+        ctx.invoke(code_runs)
 
 @list.command()
 @click.pass_context
@@ -161,7 +162,7 @@ def create(debug, output: str) -> None:
         if output
         else os.path.join(os.getcwd(), fdp_com.USER_CONFIG_FILE)
     )
-    click.echo(f"Generating new user configuration file" f" '{output}'")
+    click.echo(f"Generating new user configuration file '{output}'")
     with fdp_session.FAIR(os.getcwd(), debug=debug) as fair_session:
         fair_session.make_starter_config(output)
 
@@ -527,6 +528,7 @@ def run(
 ):
     """Initialises a job with the option to specify a bash command"""
     # Allow no config to be specified, if that is the case use default local
+    click.echo("Running run please wait")
     config = config[0] if config else fdp_com.local_user_config(os.getcwd())
     try:
         with fdp_session.FAIR(
@@ -636,6 +638,7 @@ def modify(ctx, label: str, url: str, debug: bool) -> None:
 )
 def push(remote: str, debug: bool, dirty: bool):
     """Push data between the local and remote registry"""
+    click.echo("Running push please wait")
     remote = remote[0] if remote else "origin"
     try:
         with fdp_session.FAIR(
@@ -679,6 +682,7 @@ def config_email(user_email: str) -> None:
 )
 def pull(config: str, debug: bool, local: bool):
     """Update local registry from remotes and sources"""
+    click.echo("Running pull please wait")
     config = config[0] if config else fdp_com.local_user_config(os.getcwd())
     try:
         with fdp_session.FAIR(

{fair_cli-0.7.3 → fair_cli-0.8.0}/fair/common.py

@@ -296,4 +296,11 @@ def set_file_permissions(path: str):
         for dir in [os.path.join(root,d) for d in dirs]:
             os.chmod(dir, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
         for file in [os.path.join(root, f) for f in files]:
-            os.chmod(file, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
+            os.chmod(file, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
+
+def remove_readonly(fn, path, excinfo):
+    try:
+        os.chmod(path, stat.S_IWRITE)
+        fn(path)
+    except Exception as exc:
+        print("Skipped:", path, "because:\n", exc)

{fair_cli-0.7.3 → fair_cli-0.8.0}/fair/configuration/__init__.py

@@ -262,7 +262,7 @@ def remote_git_repo(fair_repo_loc: str) -> str:
 
 def get_remote_token(
     repo_dir: str, remote: str = "origin", local: bool = False
-) -> str:
+) -> typing.Optional[str]:
     _local_config = read_local_fdpconfig(repo_dir)
     if remote not in _local_config["registries"]:
         raise fdp_exc.CLIConfigurationError(
@@ -378,6 +378,22 @@ def get_current_user_uri(repo_loc: str) -> str:
         raise fdp_exc.CLIConfigurationError("No user URI identifier defined.")
     return _uri
 
+def get_current_user_github(repo_loc: str) -> str:
+    """Retrieves the URI identifier for the current user
+
+    Returns
+    -------
+    str
+        github username
+    """
+    _local_conf = read_local_fdpconfig(repo_loc)
+    try:
+        _github = _local_conf["user"]["github"]
+    except KeyError:
+        _github = None
+    if not _github or _github == "None":
+        raise fdp_exc.CLIConfigurationError("No user GitHub username defined.")
+    return _github
 
 def check_registry_exists(registry: str = None) -> typing.Optional[str]:
     """Checks if fair registry is set up on users machine
@@ -436,10 +452,10 @@ def get_local_port(local_uri: str = None) -> int:
     return _port
 
 
-def update_local_port() -> str:
+def update_local_port(registry_dir: str = None) -> str:
     """Updates the local port in the global configuration from the session port file"""
-    _current_port = fdp_com.registry_session_port()
-    _current_address = fdp_com.registry_session_address()
+    _current_port = fdp_com.registry_session_port(registry_dir)
+    _current_address = fdp_com.registry_session_address(registry_dir)
 
     _new_url = f'http://{_current_address}:{_current_port}/api/'
 
@@ -474,7 +490,7 @@ def _handle_orcid(user_orcid: str) -> typing.Tuple[typing.Dict, str]:
         user_orcid = click.prompt("ORCID")
         _user_info = fdp_id.check_orcid(user_orcid.strip())
 
-    _user_info["orcid"] = user_orcid
+    _user_info["orcid"] = user_orcid.strip()
 
     click.echo(
         f"Found entry: {_user_info['given_names']} {_user_info['family_name']}"
@@ -509,7 +525,7 @@ def _handle_ror(user_ror: str) -> typing.Tuple[typing.Dict, str]:
         user_ror = click.prompt("ROR ID")
         _user_info = fdp_id.check_ror(user_ror.strip())
 
-    _user_info["ror"] = user_ror
+    _user_info["ror"] = user_ror.strip()
 
     click.echo(f"Found entry: {_user_info['family_name']} ")
 
@@ -540,7 +556,7 @@ def _handle_grid(user_grid: str) -> typing.Tuple[typing.Dict, str]:
         user_grid = click.prompt("GRID ID")
         _user_info = fdp_id.check_grid(user_grid.strip())
 
-    _user_info["grid"] = user_grid
+    _user_info["grid"] = user_grid.strip()
 
     click.echo(f"Found entry: {_user_info['family_name']} ")
 
@@ -548,6 +564,22 @@ def _handle_grid(user_grid: str) -> typing.Tuple[typing.Dict, str]:
 
     return _user_info, _def_ospace
 
+def _handle_github(user_github: str) -> typing.Tuple[typing.Dict, str]:
+
+    _user_info = fdp_id.check_github(user_github.strip())
+
+    while not _user_info:
+        time.sleep(3)
+        click.echo(f"Invalid GitHub Username '{user_github}' given.")
+        user_github = click.prompt("GitHub Username")
+        _user_info = fdp_id.check_github(user_github.strip())
+
+    _def_ospace = _user_info["github"].lower()
+    
+    click.echo(f"Found entry: {_user_info['github']} ")
+    return _user_info, _def_ospace
+
+
 
 def _handle_uuid() -> typing.Tuple[typing.Dict, str]:
     """Obtain metadata for user where no ID provided
@@ -575,17 +607,24 @@ def _handle_uuid() -> typing.Tuple[typing.Dict, str]:
     return _user_info, _def_ospace
 
 
-def _get_user_info_and_namespaces() -> typing.Dict[str, typing.Dict]:
-    _user_email = click.prompt("Email")
+def _get_user_info_and_namespaces(local: bool = False) -> typing.Dict[str, typing.Dict]:
+    _user_email = click.prompt("Email (optional)", default = "")
 
     _invalid_input = True
 
     while _invalid_input:
         _id_type = click.prompt(
-            "User ID system (ORCID/ROR/GRID/None)", default="None"
+            "User ID system (GITHUB/ORCID/ROR/GRID/None)", default="GITHUB"
         )
-
-        if _id_type.upper() == "ORCID":
+        if _id_type.upper() == "GITHUB":
+            _user_github = click.prompt("GitHub Username")
+            _user_info, _def_ospace = _handle_github(_user_github)
+            if not _user_info["name"]:
+                _user_info["given_names"] = click.prompt("Given Names")
+                _user_info["family_name"] = click.prompt("Family Name")
+                _user_info["name"] = " ".join([_user_info["given_names"], _user_info["family_name"]])
+            _invalid_input = False
+        elif _id_type.upper() == "ORCID":
             _user_orcid = click.prompt("ORCID")
             _user_info, _def_ospace = _handle_orcid(_user_orcid)
             _invalid_input = False
@@ -603,14 +642,25 @@ def _get_user_info_and_namespaces() -> typing.Dict[str, typing.Dict]:
             _user_info["uuid"] = _user_uuid
             _invalid_input = False
 
-    _user_info["email"] = _user_email
-
     _def_ospace = _def_ospace.lower().replace(" ", "").strip()
     _def_ospace = click.prompt("Default output namespace", default=_def_ospace)
     _def_ispace = click.prompt("Default input namespace", default=_def_ospace)
 
     _namespaces = {"input": _def_ispace, "output": _def_ospace}
 
+    if not "github" in _user_info:
+        if local:
+            _user_info["github"] = "FAIRDataPipeline"
+        else:
+            _user_github = click.prompt("GitHub Username")
+            _user_github_info = _handle_github(_user_github)[0]
+            _user_info["github"] = _user_github_info["github"]
+
+    if not _user_email:
+        _user_info["email"] = f'{_user_info["github"]}@users.noreply.github.com'
+    else:
+        _user_info["email"] = _user_email
+
     return {"user": _user_info, "namespaces": _namespaces}
 
 
@@ -668,21 +718,23 @@ def global_config_query(
             default=_remote_url.replace("/api/", "/data/"),
         )
 
-        _rem_key_file = click.prompt(
-            "Remote API Token File",
-        )
-        _rem_key_file = os.path.expandvars(_rem_key_file)
-
-        while (
-            not os.path.exists(_rem_key_file)
-            or not open(_rem_key_file).read().strip()
-        ):
-            click.echo(
-                f"Token file '{_rem_key_file}' does not exist or is empty, "
-                "please provide a valid token file."
+        _rem_key_valid = False
+        while not _rem_key_valid:
+            _rem_key = click.prompt(
+                f"Remote API Token",
             )
-            _rem_key_file = click.prompt("Remote API Token File")
-            _rem_key_file = os.path.expandvars(_rem_key_file)
+            if len(_rem_key) == 40:
+                _rem_key_valid = True
+            else:
+                click.echo("Remote token should be 40 characters long")
+
+        _rem_key_file = os.path.join(fdp_com.global_config_dir(), "remotetoken.txt")
+
+        with open(_rem_key_file, 'w') as f:
+            f.write(_rem_key)
+
+        if not os.path.exists(_rem_key_file):
+            raise fdp_exc.CLIConfigurationError(f'Token could not be written to {_rem_key_file}')
 
     if not fdp_serv.check_server_running():
         if _ := click.confirm(
@@ -696,7 +748,7 @@ def global_config_query(
             pathlib.Path(_cache_addr).touch()
 
         else:
-            click.echo("Temporarily launching server to retrieve API token.")
+            click.echo("Temporarily launching local registry to retrieve API token.")
             fdp_serv.launch_server(registry_dir=registry)
             fdp_serv.stop_server(registry_dir=registry, local_uri=_local_uri)
             try:
@@ -713,7 +765,7 @@ def global_config_query(
     if _loc_data_store[-1] != os.path.sep:
         _loc_data_store += os.path.sep
 
-    _glob_conf_dict = _get_user_info_and_namespaces()
+    _glob_conf_dict = _get_user_info_and_namespaces(local)
     _glob_conf_dict["registries"] = {
         "local": {
             "uri": _local_uri,

{fair_cli-0.7.3 → fair_cli-0.8.0}/fair/configuration/validation.py

@@ -108,6 +108,9 @@ class User(pydantic.BaseModel):
     name: typing.Optional[str] = pydantic.Field(
         None, title="Full Name", description="Full name for the user"
     )
+    github: typing.Optional[str] = pydantic.Field(
+        None, title="GitHub Username", description="GitHub Username for the user"
+    )
 
     class Config:
         extra = "forbid"

{fair_cli-0.7.3 → fair_cli-0.8.0}/fair/history.py

@@ -22,6 +22,8 @@ import os
 import click
 import rich
 
+from pathlib import Path
+
 import fair.common as fdp_com
 import fair.exceptions as fdp_exc
 import fair.run as fdp_run
@@ -61,8 +63,10 @@ def show_job_log(repo_loc: str, job_id: str) -> str:
     str
         log file location for the given job
     """
+    _job_dir  = Path(f"{fdp_com.default_jobs_dir()}")
+
     _sorted_time_dirs = sorted(
-        glob.glob(os.path.join(fdp_com.default_jobs_dir(), "*")), reverse=True
+        glob.glob(os.path.join(_job_dir, "*")), reverse=True
     )
 
     _log_files = [
@@ -107,8 +111,10 @@ def show_history(repo_loc: str, length: int = 10) -> None:
         max number of entries to display, by default 10
     """
 
+    _job_dir  = Path(f"{fdp_com.default_jobs_dir()}")
+
     _sorted_time_dirs = sorted(
-        glob.glob(os.path.join(fdp_com.default_jobs_dir(), "*")), reverse=True
+        glob.glob(os.path.join(_job_dir, "*")), reverse=True
     )
 
     _log_files = [

{fair_cli-0.7.3 → fair_cli-0.8.0}/fair/identifiers.py

@@ -24,15 +24,22 @@ import urllib.parse
 
 import requests
 import requests.exceptions
+import logging
+from urllib3.exceptions import InsecureRequestWarning
+from fake_useragent import UserAgent
+
+logger = logging.getLogger("FAIRDataPipeline.Identifiers")
 
 ID_URIS = {
     "orcid": "https://orcid.org/",
     "ror": "https://ror.org/",
+    "github" : "https://github.com/"
 }
 
 QUERY_URLS = {
     "orcid": "https://pub.orcid.org/v3.0/",
     "ror": "https://api.ror.org/organizations?query=",
+    "github": "https://api.github.com/users/"
 }
 
 
@@ -49,14 +56,16 @@ def check_orcid(orcid: str) -> typing.Dict:
     typing.Dict
         metadata from the given ID
     """
-
+    orcid = orcid.replace(ID_URIS["orcid"], "")
     _header = {"Accept": "application/json"}
     _url = urllib.parse.urljoin(QUERY_URLS["orcid"], orcid)
-    _response = requests.get(_url, headers=_header)
+    requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
+    _response = requests.get(_url, headers=_header, verify = False, allow_redirects = True)
 
     _result_dict: typing.Dict[str, typing.Any] = {}
 
     if _response.status_code != 200:
+        logger.debug(f"{_url} Responded with {_response.status_code}")
         return _result_dict
 
     _names = _response.json()["person"]["name"]
@@ -72,6 +81,42 @@ def check_orcid(orcid: str) -> typing.Dict:
 
     return _result_dict
 
+def check_github(github: str) -> typing.Dict:
+    """Checks if valid ORCID using ORCID public api
+
+    Parameters
+    ----------
+    github : str
+        github username to be checked
+
+    Returns
+    -------
+    typing.Dict
+        metadata from the given ID
+    """
+    _header = {"Accept": "application/json"}
+    _url = urllib.parse.urljoin(QUERY_URLS["github"], github)
+    requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
+    _response = requests.get(_url, headers=_header, verify = False, allow_redirects = True)
+
+    _result_dict: typing.Dict[str, typing.Any] = {}
+
+    if _response.status_code != 200:
+        logger.debug(f"{_url} Responded with {_response.status_code}")
+        return _result_dict
+
+    _login = _response.json()["login"]
+    _name = _response.json()["name"]
+    if _name:
+        _result_dict["family_name"] = _name.split()[-1]
+        _result_dict["given_names"] = " ".join(_name.split()[:-1])
+        
+    _result_dict["name"] = _name    
+    _result_dict["github"] = _login
+    _result_dict["uri"] = f'{ID_URIS["github"]}{_login}'
+
+    return _result_dict
+
 
 def check_ror(ror: str) -> typing.Dict:
     """Checks if valid ROR using ROR public api
@@ -86,7 +131,6 @@ def check_ror(ror: str) -> typing.Dict:
     typing.Dict
         metadata from the given ID
     """
-
     _result_dict = _check_generic_ror(ror)
     if _result_dict:
         _result_dict["ror"] = ror
@@ -124,13 +168,13 @@ def _check_generic_ror(id: str) -> typing.Dict:
     typing.Dict
         metadata from the given ID
     """
-
     _url = f"{QUERY_URLS['ror']}{id}"
     _response = requests.get(_url)
 
     _result_dict: typing.Dict[str, typing.Any] = {}
 
     if _response.status_code != 200:
+        logger.debug(f"{_url} Responded with {_response.status_code}")
         return _result_dict
 
     if _response.json()["number_of_results"] == 0:
@@ -164,18 +208,25 @@ def check_id_permitted(identifier: str, retries: int = 5) -> bool:
         if valid identifier
     """
     _n_attempts = 0
+    fake_agent = False
 
     while _n_attempts < retries:
         try:
-            requests.get(identifier).raise_for_status()
+            requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
+            headers = {}
+            if fake_agent:
+                headers = {'User-Agent':str(UserAgent().chrome)} 
+            requests.get(identifier, verify = False, allow_redirects = True, headers = headers).raise_for_status()
             return True
         except (
             requests.exceptions.MissingSchema,
             requests.exceptions.HTTPError,
             requests.exceptions.ConnectionError,
-        ):
+        ) as e:
             _n_attempts += 1
-            time.sleep(1)
+            time.sleep(3)
+            fake_agent = True
+            logger.warning(f"Error identifier: '{identifier}' caused '{e}'")
             continue
 
     return False

{fair_cli-0.7.3 → fair_cli-0.8.0}/fair/register.py

@@ -88,7 +88,7 @@ def fetch_registrations(
     typing.List[str]
         list of registered object URLs
     """
-    _expected_keys = [
+    _expected_keys_external_object = [
         "root",
         "path",
         "file_type",
@@ -97,9 +97,24 @@ def fetch_registrations(
         "public",
     ]
 
+    _expected_keys_data_product = [
+        "root",
+        "path",
+        "file_type",
+        "version",
+        "public"
+    ]
+
     _stored_objects: typing.List[str] = []
 
     for entry in user_config_register:
+        logger.debug(f"entry {entry}")
+
+        if "external_object" in entry:
+            _expected_keys = _expected_keys_external_object
+        else:
+            _expected_keys = _expected_keys_data_product
+
         for key in _expected_keys:
             if key not in entry and key not in entry["use"]:
                 raise fdp_exc.UserConfigError(
@@ -166,18 +181,22 @@ def fetch_registrations(
         _search_data["version"] = entry["use"]["version"]
         _namespace = entry["use"]["namespace"]
 
-        if not _identifier and not _unique_name:
-            raise fdp_exc.UserConfigError(
-                "Expected either 'unique_name' or 'identifier' in 'register' item"
-            )
-
-        elif _identifier and _unique_name:
-            raise fdp_exc.UserConfigError(
-                "Only one unique identifier may be provided (doi/unique_name)"
-            )
+        if _external_object:
+            if not _identifier and not _unique_name:
+                raise fdp_exc.UserConfigError(
+                    "Expected either 'unique_name' or 'identifier' in 'register' item"
+                )
 
+            elif _identifier and _unique_name:
+                raise fdp_exc.UserConfigError(
+                    "Only one unique identifier may be provided (doi/unique_name)"
+                )
+        # Set Remove to True by default so the tempory file gets deleted
+        _remove = True
         if "cache" in entry:
             _temp_data_file = entry["cache"]
+            # Don't delete the tempory file if it's from a cache
+            _remove = False
         else:
             _local_parsed = urllib.parse.urlparse(local_uri)
             _local_url = f"{_local_parsed.scheme}://{_local_parsed.netloc}"
@@ -207,7 +226,8 @@ def fetch_registrations(
                 " present with this name, deleting temporary data file",
                 _name,
             )
-            os.remove(_temp_data_file)
+            if _remove:
+                os.remove(_temp_data_file)
             continue
 
         # Item found but not hash matched retrieve a version number
@@ -239,7 +259,8 @@ def fetch_registrations(
         logger.debug("Saving data file to '%s'", _local_file)
         shutil.copy(_temp_data_file, _local_file)
 
-        os.remove(_temp_data_file)
+        if _remove:
+            os.remove(_temp_data_file)
 
         if "public" in entry:
             _public = entry["public"]

{fair_cli-0.7.3 → fair_cli-0.8.0}/fair/registry/requests.py

@@ -42,6 +42,9 @@ import fair.utilities as fdp_util
 import ssl
 from requests.adapters import HTTPAdapter
 from urllib3.poolmanager import PoolManager
+from urllib3.exceptions import InsecureRequestWarning
+
+from fake_useragent import UserAgent
 
 logger = logging.getLogger("FAIRDataPipeline.Requests")
 
@@ -528,8 +531,8 @@ def download_file(url: str, chunk_size: int = 8192) -> str:
     _file = tempfile.NamedTemporaryFile(delete=False)
     _fname = _file.name
 
-    # Copy File if local (Windows fix)
-    if "file://" in url and platform.system() == "Windows":
+    # Copy File if local
+    if "file://" in url:
         _local_fname = url.replace("file://", "")
         try:
             shutil.copy2(_local_fname, _fname)
@@ -538,17 +541,16 @@ def download_file(url: str, chunk_size: int = 8192) -> str:
                 f"Failed to download file '{url}'"
                 f" due to connection error: {traceback.format_exc()}"
             ) from e
-
     else:
         try:
-            with urllib.request.urlopen(url) as response, open(
-                _fname, "wb"
-            ) as out_file:
-                shutil.copyfileobj(response, out_file)
-        except urllib.error.URLError as e:
+            requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
+            headers = {'User-Agent':str(UserAgent().chrome)}
+            response = requests.get(url, allow_redirects = True, verify = False, headers = headers)
+            open(_fname, 'wb').write(response.content)
+        except Exception as e:
             raise fdp_exc.FAIRCLIException(
                 f"Failed to download file '{url}'"
-                f" due to connection error: {e.reason}"
+                f" due to connection error: {traceback.format_exc()}"
             ) from e
 
     return _fname
@@ -571,7 +573,7 @@ def get_dependency_listing(uri: str, token: str, read_only: bool = False) -> typ
     """
     try:
         _registry_objs = url_get(uri, token)
-    except:
+    except Exception:
         return {[]}
 
     _rtn =  {