materials-commons-cli 2.3.0b3__tar.gz → 2.3.0b4__tar.gz

{materials_commons_cli-2.3.0b3 → materials_commons_cli-2.3.0b4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: materials_commons_cli
-Version: 2.3.0b3
+Version: 2.3.0b4
 Summary: Materials Commons CLI
 Home-page: https://materials-commons.github.io/materials-commons-cli/html/index.html
 Author: Materials Commons development team

materials_commons_cli-2.3.0b4/materials_commons/cli/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.3.0b4'

materials_commons_cli-2.3.0b4/materials_commons/cli/old/cloned_project.py

@@ -0,0 +1,223 @@
+import os
+import pathlib
+import tempfile
+
+import materials_commons.cli.old.exceptions as cliexcept
+import materials_commons.cli.old.functions as clifuncs
+from materials_commons.cli.old.user_config import Config, \
+    get_remote_config_and_login_if_necessary
+from materials_commons.cli.subcommands.down import down_subcommand
+from materials_commons.cli.subcommands.up import up_subcommand
+
+# TODO: only call os.getcwd() from parser::main
+
+class ClonedProject(object):
+    """A cloned Materials Commons project instance
+
+    Attributes:
+        local_path (pathlib.Path): Location of the cloned Materials Commons
+            project
+        proj (materials_commons.api.models.Project): Materials Commons project
+            object
+        tmpdir (tempfile.TemporaryDirectory or None): Temporary directory
+            instance. If not None, the temporary directory is the parent of the
+            cloned Materials Commons project directory.
+
+    """
+
+    def __init__(self, email=None, mcurl=None, proj_id=None, path=None,
+                 parent_path=None, name=None):
+        """Construct a cloned Materials Commons project instance
+
+        Examples:
+
+            Open a project that has already been cloned:
+
+            .. code-block:: python
+
+                path = "/path/to/materials_commons_projects/ProjectName"
+                mc_proj = ClonedProject(path=path)
+
+            Clone project to a particular directory or open if already cloned:
+
+            .. code-block:: python
+
+                email = "username@domain.com"
+                mcurl = "https://materialscommons.org/api"
+                proj_id = 25
+                parent_path = "/path/to/materials_commons_projects"
+                name = None  # default uses remote project name
+                mc_proj = ClonedProject(email=email,
+                                        mcurl=mcurl,
+                                        proj_id=proj_id,
+                                        parent_path=parent_path,
+                                        name=name)
+
+            Clone project to a temporary directory:
+
+            .. code-block:: python
+
+                email = "username@domain.com"
+                mcurl = "https://materialscommons.org/api"
+                proj_id = 25
+                mc_proj = ClonedProject(email=email, mcurl=mcurl, proj_id=proj_id)
+
+        Args:
+            email (str): User account email
+            mcurl (str): URL for Materials Commons remote instance containing
+                the project. Example: "https://materialscommons.org/api".
+            proj_id (int): ID of project to clone.
+            path (str): Path where the project exists, if already cloned.
+            parent_path (str): Path to parent directory where the project should
+                be cloned if path is None. If neither path nor parent_path are
+                given, uses a tempfile.TemporaryDirectory for parent_path.
+            name (str): Name of created project directory. Default is remote
+                project name.
+        """
+        self.local_path = None
+        self.proj = None
+        self.tmpdir = None
+
+        if path is not None:
+            if clifuncs.project_exists(path):
+                self.proj = clifuncs.make_local_project(path)
+            else:
+                raise cliexcept.MCCLIException("No project found at " + path)
+        else:
+            if proj_id is None:
+                raise cliexcept.MCCLIException("`proj_id` is required if `path` is not provided")
+
+            if email is None or mcurl is None:
+                config = Config()
+                if not config.default_remote.mcurl or not config.default_remote.mcapikey:
+                    raise cliexcept.NoDefaultRemoteException("Default remote not set")
+                remote_config = config.default_remote
+            else:
+                remote_config = get_remote_config_and_login_if_necessary(
+                    mcurl=mcurl, email=email)
+
+            if parent_path is None:
+                self.tmpdir = tempfile.TemporaryDirectory()
+                self.proj = clifuncs.clone_project(remote_config, proj_id, self.tmpdir.name)
+
+            else:
+                self.tmpdir = None
+                parent_path = str(parent_path)
+
+                # check if project already exists, then construct or clone
+                client = remote_config.make_client()
+                proj = client.get_project(proj_id)
+                path = os.path.join(parent_path, proj.name)
+                if clifuncs.project_exists(path):
+                    self.proj = clifuncs.make_local_project(path, proj._data)
+                else:
+                    self.proj = clifuncs.clone_project(remote_config, proj_id, parent_path)
+
+        self.local_path = pathlib.Path(self.proj.local_path)
+
+    def glob(self, pattern):
+        """Helper to construct paths for upload or download
+
+        Args:
+            pattern (str): Pattern, relative to local project directory root,
+                passed as argument to `self.local_path.glob(pattern)`.
+
+        Returns:
+            List of str, File paths found from use of `glob`, made relative to
+            self.local_path and converted to str.
+
+        """
+        return [str(file.relative_to(self.local_path)) for file in self.local_path.glob(pattern)]
+
+    def download(self, *paths, recursive=False, only_print=False, force=False,
+                 output=None, globus=False, label=None, no_compare=False):
+        """Download requested files from the Materials Commons project
+
+        Args:
+            recursive (bool): Download directory contents recursively
+            force (bool): Force overwrite of existing files
+            only_print (bool): Print file, do not write
+            output (str): Download file name. Only allowed if `len(paths) == 1`.
+            globus (bool): Use globus to download files
+            label (str): Globus transfer label to make finding tasks simpler
+            no_compare (bool): Download remote without checking if local is
+                equivalent
+            *paths (str): Files or directories to download, specified either
+                using absolute paths or paths relative to the project root
+                directory (`self.local_path`).
+        """
+        # TODO: convert to direct function calls rather than arg parsing
+
+        working_dir = self.local_path
+
+        argv = []
+        if recursive is True:
+            argv.append("-r")
+        if only_print is True:
+            argv.append("-p")
+        if force is True:
+            argv.append("-f")
+        if output is not None:
+            argv.append("-o")
+            argv.append(str(output))
+        if globus is True:
+            argv.append("-g")
+        if label is not None:
+            argv.append("--label")
+            argv.append(str(label))
+        if no_compare is True:
+            argv.append("--no-compare")
+        if len(paths):
+            # using relpaths is more robust within the working_dir context
+            # argv += [str(os.path.relpath(os.path.abspath(path), self.local_path)) for path in paths]
+            argv += [os.path.normpath(os.path.join(working_dir, path)) for path in paths]
+        try:
+            down_subcommand(argv, working_dir)
+        except SystemExit as e:
+            print("Invalid download request")
+
+    def upload(self, *paths, recursive=False, limit=None, globus=False,
+               label=None, no_compare=False, upload_as=None):
+        """Upload requested files to Materials Commons
+
+        Args:
+            recursive (bool): Download directory contents recursively
+            limit (str): File size upload limit (MB). Default="750" (750MB). Does
+                not apply to Globus uploads.
+            globus (bool): Use globus to download files
+            label (str): Globus transfer label to make finding tasks simpler
+            no_compare (bool): Download remote without checking if local is
+                equivalent
+            upload_as (str): Upload a file or directory to a particular location in the project. Raises if `len(paths) != 1`.
+            *paths (str): Files or directories to upload, specified either
+                using absolute paths or paths relative to the project root
+                directory (`self.local_path`).
+        """
+        # TODO: convert to direct function calls rather than arg parsing
+
+        working_dir = self.local_path
+
+        argv = []
+        if recursive is True:
+            argv.append("-r")
+        if limit is not None:
+            argv.append("--limit")
+            argv.append(str(limit))
+        if globus is True:
+            argv.append("-g")
+        if label is not None:
+            argv.append("--label")
+            argv.append(str(label))
+        if no_compare is True:
+            argv.append("--no-compare")
+        if upload_as is not None:
+            argv.append("--upload-as")
+            argv.append(str(upload_as))
+        if len(paths):
+            # using relpaths is more robust within the working_dir context
+            # argv += [str(os.path.relpath(os.path.abspath(path), self.local_path)) for path in paths]
+            argv += [os.path.normpath(os.path.join(working_dir, path)) for path in paths]
+        try:
+            up_subcommand(argv, working_dir)
+        except SystemExit as e:
+            print("Invalid upload request")

materials_commons_cli-2.3.0b4/materials_commons/cli/old/exceptions.py

@@ -0,0 +1,12 @@
+
+class MCCLIException(BaseException):
+    pass
+
+class MissingRemoteException(BaseException):
+    pass
+
+class MultipleRemoteException(BaseException):
+    pass
+
+class NoDefaultRemoteException(BaseException):
+    pass

materials_commons_cli-2.3.0b4/materials_commons/cli/old/file_functions.py

@@ -0,0 +1,137 @@
+import io
+import os.path
+import requests
+import materials_commons.api as mcapi
+from materials_commons.cli.old.exceptions import MCCLIException
+
+def isfile(file_or_dir):
+    return isinstance(file_or_dir, mcapi.File) and file_or_dir.mime_type != "directory"
+
+def isdir(file_or_dir):
+    return isinstance(file_or_dir, mcapi.File) and file_or_dir.mime_type == "directory"
+
+def get_parent_id(file_or_dir):
+    """Get file_or_dir.directory_id, else raise"""
+    if hasattr(file_or_dir, 'directory_id'):
+        if file_or_dir.directory_id == file_or_dir.id:
+            raise MCCLIException("directory_id == id")
+        return file_or_dir.directory_id
+    else:
+        raise MCCLIException("file_or_dir is missing attribute directory_id")
+
+def make_local_abspath(proj_local_path, mcpath):
+    return os.path.normpath(os.path.join(proj_local_path, os.path.relpath(mcpath, "/")))
+
+def path_in_project(proj_local_path, local_abspath):
+    """Check if local path is within the local project directory
+
+    Args:
+        proj_local_path (str): Path to project, i.e. "/path/to/project"
+        local_abspath (str): Path to file in project, i.e. "/path/to/project/
+            path/to/file"
+
+    Returns: True, if local_abspath is within the local project directory.
+    """
+    return os.path.commonpath([proj_local_path, local_abspath]) == os.path.normpath(proj_local_path)
+
+def make_mcpath(proj_local_path, local_abspath):
+    """
+
+    Args:
+        proj_local_path (str): Path to project, i.e. "/path/to/project"
+        local_abspath (str): Path to file in project, i.e. "/path/to/project/
+            path/to/file"
+
+    Returns:
+        mcpath (str): Returns a Materials Commons style path, i.e. "/path/to/
+            file"
+
+    Raises:
+        MCCLIException, if any path in clipaths is not within the local project
+        directory.
+    """
+    if not path_in_project(proj_local_path, local_abspath):
+        raise MCCLIException("make_mcpath error: path not in project")
+    relpath = os.path.relpath(local_abspath, proj_local_path)
+    if relpath == ".":
+        mcpath = "/"
+    else:
+        mcpath = os.path.join("/", relpath)
+    return mcpath
+
+def get_by_path_if_exists(client, project_id, file_path):
+    """
+    Get file (or directory) by path in project, if it exists.
+    :param int project_id: The id of the project containing the file or directory
+    :param file_path: The Materials Commons path to the file or directory
+    :return: The file or None
+    :rtype File or None
+    """
+    try:
+        return client.get_file_by_path(project_id, file_path)
+    except mcapi.MCAPIError as e:
+        if e.response.status_code == 404:
+            return None
+        raise e
+
+def _check_file_selection_dirs(path, file_selection, orig_path=None):
+    """Recursively checks if a path is included in a dataset file selection, and why"""
+    if path == orig_path:
+        selected_by = "(self)"
+    else:
+        selected_by = path
+    if path in file_selection['include_dirs']:
+        return (True, selected_by)
+    elif path in file_selection['exclude_dirs']:
+        return (False, selected_by)
+    else:
+        parent = os.path.dirname(path)
+        if parent == "/":
+            return (False, None)
+        else:
+            return _check_file_selection_dirs(parent, file_selection, orig_path=orig_path)
+
+def check_file_selection(path, file_selection):
+    """Check if a file or directory is selected in a dataset file selection, and why.
+
+    Notes:
+        A file is selected in a dataset's file selection if it is listed explicitly in "include_files". It is not selected if it is listed in "exclude_files". If it is neither explicitly included or excluded, its parent directories are searched up to the project root directory to see if they are included or excluded. If the first parent directory listed is in "include_dirs", then the file is also selected. If the first parent directory listed is in "exclude_dirs", or if no parent directory is listed at all, the file is not selected.
+
+    Args:
+        path (str): Materials Commons path of file or directory to check
+        file_selection (dict): A file selection dict. Expected format: ::
+
+            {
+                "include_files": [... list of file paths ...],
+                "exclude_files": [... list of file paths ...],
+                "include_dirs": [... list of directory paths ...],
+                "exclude_dirs": [... list of directory paths ...]
+            }
+
+    Returns:
+        (selected, selected_by):
+
+        selected (bool): True if selected, False if not selected
+
+        selected_by (str or None): One of "(self)" if included/excluded explicitly; Else, "<path>", the path of the first parent directory that is included or excluded; Otherwise, None, to indicate that it is not selected, but neither included nor excluded.
+    """
+    result = None
+    if path in file_selection['include_files']:
+        result = (True, "(self)")
+    elif path in file_selection['exclude_files']:
+        result = (False, "(self)")
+    else:
+        result = _check_file_selection_dirs(path, file_selection, orig_path=path)
+    return result
+
+def download_file_as_string(client, project_id, file_id):
+    f = io.BytesIO()
+    client._throttle()
+    urlpart = "/projects/" + str(project_id) + "/files/" + str(file_id) + "/download"
+    url = client.base_url + urlpart
+    with requests.get(url, stream=True, verify=False, headers=client.headers) as r:
+        client._handle(r)
+        for chunk in r.iter_content(chunk_size=8192):
+            if chunk:
+                f.write(chunk)
+    return f.getvalue().decode('utf-8')