materials-commons-cli 2.1.2__tar.gz → 2.1.4__tar.gz

{materials_commons-cli-2.1.2 → materials_commons_cli-2.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
-Name: materials_commons-cli
-Version: 2.1.2
+Name: materials_commons_cli
+Version: 2.1.4
 Summary: Materials Commons CLI
 Home-page: https://materials-commons.github.io/materials-commons-cli/html/index.html
 Author: Materials Commons development team

materials_commons_cli-2.1.4/materials_commons/cli/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.1.4'

{materials_commons-cli-2.1.2 → materials_commons_cli-2.1.4}/materials_commons/cli/cloned_project.py

@@ -183,7 +183,7 @@ class ClonedProject(object):
 
         Args:
             recursive (bool): Download directory contents recursively
-            limit (str): File size upload limit (MB). Default="50" (50MB). Does
+            limit (str): File size upload limit (MB). Default="750" (750MB). Does
                 not apply to Globus uploads.
             globus (bool): Use globus to download files
             label (str): Globus transfer label to make finding tasks simpler

materials_commons_cli-2.1.4/materials_commons/cli/parser.py

@@ -0,0 +1,187 @@
+import argparse
+import json
+import os
+import sys
+from datetime import datetime, timedelta
+from io import StringIO
+
+import materials_commons.api as mcapi
+import pkg_resources
+import requests
+
+import materials_commons.cli.functions as clifuncs
+from materials_commons.cli.exceptions import MCCLIException, MissingRemoteException, \
+    MultipleRemoteException, NoDefaultRemoteException
+from materials_commons.cli.subcommands.clone import clone_subcommand
+from materials_commons.cli.subcommands.config import config_subcommand
+from materials_commons.cli.subcommands.dataset import DatasetSubcommand
+from materials_commons.cli.subcommands.down import down_subcommand
+from materials_commons.cli.subcommands.expt import ExptSubcommand
+from materials_commons.cli.subcommands.fetch import fetch_subcommand
+# from materials_commons.cli.subcommands.samp import SampSubcommand
+from materials_commons.cli.subcommands.globus import globus_subcommand
+from materials_commons.cli.subcommands.init import init_subcommand
+from materials_commons.cli.subcommands.ls import ls_subcommand
+from materials_commons.cli.subcommands.mkdir import mkdir_subcommand
+from materials_commons.cli.subcommands.mv import mv_subcommand
+# from materials_commons.cli.subcommands.proc import ProcSubcommand
+from materials_commons.cli.subcommands.proj import ProjSubcommand
+from materials_commons.cli.subcommands.remote import remote_subcommand
+from materials_commons.cli.subcommands.rm import rm_subcommand
+from materials_commons.cli.subcommands.up import up_subcommand
+from materials_commons.cli.subcommands.versions import versions_subcommand
+from materials_commons.cli.user_config import Config
+
+standard_usage = [
+    {'name': 'remote', 'desc': 'List servers', 'subcommand': remote_subcommand},
+    {'name': 'proj', 'desc': 'List projects', 'subcommand': ProjSubcommand()},
+    {'name': 'dataset', 'desc': 'List datasets', 'subcommand': DatasetSubcommand()},
+    {'name': 'expt', 'desc': 'List experiments', 'subcommand': ExptSubcommand()},
+    {'name': 'init', 'desc': 'Initialize a new project', 'subcommand': init_subcommand},
+    {'name': 'clone', 'desc': 'Clone an existing project', 'subcommand': clone_subcommand},
+    {'name': 'ls', 'desc': 'List directory contents', 'subcommand': ls_subcommand},
+    {'name': 'mkdir', 'desc': 'Make directories', 'subcommand': mkdir_subcommand},
+    {'name': 'rm', 'desc': 'Remove files and directories', 'subcommand': rm_subcommand},
+    {'name': 'mv', 'desc': 'Move files', 'subcommand': mv_subcommand},
+    {'name': 'fetch', 'desc': 'Remote data fetching and configuration', 'subcommand': fetch_subcommand},
+    {'name': 'up', 'desc': 'Upload files', 'subcommand': up_subcommand},
+    {'name': 'down', 'desc': 'Download files', 'subcommand': down_subcommand},
+    {'name': 'globus', 'desc': 'Manage Globus uploads and downloads', 'subcommand': globus_subcommand},
+    {'name': 'versions', 'desc': 'List file versions', 'subcommand': versions_subcommand},
+    # {'name': 'proc', 'desc': 'List processes', 'subcommand': ProcSubcommand()},
+    # {'name': 'samp', 'desc': 'List samples', 'subcommand': SampSubcommand()},
+    {'name': 'config', 'desc': 'Configure `mc`', 'subcommand': config_subcommand}
+]
+standard_interfaces = {d['name']: d for d in standard_usage}
+
+
+def make_parser():
+    usage_help = StringIO()
+    usage_help.write("mc <command> [<args>]\n\n")
+    usage_help.write("The standard mc commands are:\n")
+
+    for name, interface in standard_interfaces.items():
+        usage_help.write("  {:10} {:40}\n".format(name, interface['desc']))
+
+    parser = argparse.ArgumentParser(
+        description='Materials Commons command line interface',
+        usage=usage_help.getvalue())
+    parser.add_argument('command', help='Subcommand to run')
+    parser.add_argument('--version', '-v', action='version',
+                        version=pkg_resources.get_distribution('materials-commons-cli').version)
+
+    return parser
+
+
+CLI_VERSION_CACHE_FILE = os.path.expanduser('~/.materialscommons/.cli-version-cache.json')
+
+
+def should_check_version():
+    try:
+        if os.path.exists(CLI_VERSION_CACHE_FILE):
+            with open(CLI_VERSION_CACHE_FILE) as f:
+                cache = json.load(f)
+            last_check = datetime.fromisoformat(cache['last_check'])
+            # Only check once per day
+            if datetime.now() - last_check < timedelta(days=1):
+                return False
+    except Exception:
+        pass
+    return True
+
+
+def update_version_cache():
+    try:
+        with open(CLI_VERSION_CACHE_FILE, 'w') as f:
+            json.dump({
+                'last_check': datetime.now().isoformat()
+            }, f)
+    except Exception:
+        pass
+
+
+def check_package_version():
+    if not should_check_version():
+        return
+
+    package_name = 'materials-commons-cli'
+    try:
+        current_version = pkg_resources.get_distribution(package_name).version
+        response = requests.get(f"https://pypi.org/pypi/{package_name}/json")
+        latest_version = response.json()["info"]["version"]
+
+        if current_version != latest_version:
+            print(
+                f"\nWarning: You are using an older version of the Materials Commons CLI ({current_version}).\n"
+                f"A newer version is available ({latest_version}).\n"
+                f"It is recommended you upgrade to the newest version: pip install --upgrade {package_name}\n",
+                file=sys.stderr,
+            )
+
+        update_version_cache()
+
+    except Exception as e:
+        pass
+
+
+def main(argv=None, working_dir=None):
+    if argv is None:
+        argv = sys.argv
+    if working_dir is None:
+        working_dir = os.getcwd()
+    try:
+
+        config = Config()
+        if config.REST_logging:
+            mcapi.Client.set_debug_on()
+            pass
+
+        parser = make_parser()
+
+        if len(argv) < 2:
+            parser.print_help()
+            return 1
+
+        # parse_args defaults to [1:] for args, but you need to
+        # exclude the rest of the args too, or validation will fail
+        args = parser.parse_args(argv[1:2])
+
+        if args.command in standard_interfaces:
+            result = standard_interfaces[args.command]['subcommand'](argv[2:], working_dir)
+            check_package_version()
+            return result
+
+        else:
+            print('Unrecognized command')
+            parser.print_help()
+            return 1
+
+    except MissingRemoteException as e:
+        print("Error:", e)
+        clifuncs.print_remote_help()
+        return 1
+
+    except MultipleRemoteException as e:
+        print("Error:", e)
+        print(
+            '** Please edit .mc/config.json to include `"remote":{"mcurl": "Materials Commons https address here","email": "YOUR_EMAIL_HERE"}` **')
+        return 1
+
+    except NoDefaultRemoteException as e:
+        print("Error:", e)
+        print("Set the default remote with:")
+        print("    mc remote --set-default EMAIL URL")
+        clifuncs.print_remote_help()
+        return 1
+
+    except MCCLIException as e:
+        print("CLI Error:", e)
+        return 1
+
+    except mcapi.MCAPIError as e:
+        import json
+        print("API Error:", e)
+        print("Writing error message to 'mcapi_error.json'")
+        with open('mcapi_error.json', 'w') as f:
+            json.dump(e.response.json(), f, indent=2)
+        return 1

{materials_commons-cli-2.1.2 → materials_commons_cli-2.1.4}/materials_commons/cli/subcommands/up.py

@@ -1,15 +1,12 @@
 import argparse
-import os
-import sys
-import time
 
 import materials_commons.cli.exceptions as cliexcept
 import materials_commons.cli.functions as clifuncs
 import materials_commons.cli.globus as cliglobus
-import materials_commons.cli.file_functions as filefuncs
 import materials_commons.cli.tree_functions as treefuncs
 from materials_commons.cli.treedb import LocalTree, RemoteTree
 
+
 def make_parser():
     """Make argparse.ArgumentParser for `mc up`"""
 
@@ -29,8 +26,8 @@ def make_parser():
     parser.add_argument('paths', nargs='*', default=None, help='Files or directories')
     parser.add_argument('-r', '--recursive', action="store_true", default=False,
                         help='Upload directory contents recursively')
-    parser.add_argument('--limit', nargs=1, type=float, default=[250],
-                        help='File size upload limit (MB). Default=250MB. Does not apply to Globus uploads.')
+    parser.add_argument('--limit', nargs=1, type=float, default=[750],
+                        help='File size upload limit (MB). Default=750MB. Max size is also 750MB. Does not apply to Globus uploads.')
     parser.add_argument('-g', '--globus', action="store_true", default=False,
                         help=globus_help)
     parser.add_argument('--label', nargs=1, type=str,

{materials_commons-cli-2.1.2 → materials_commons_cli-2.1.4}/materials_commons/cli/tree_functions.py

@@ -112,7 +112,7 @@ def make_mcpaths_for_upload(proj_local_path, paths):
             _paths.append(path)
     return _paths
 
-def upload_file(proj, local_abspath, mcpath, working_dir, parent_id=None, limit=50, remotetree=None, update_remotetree=True):
+def upload_file(proj, local_abspath, mcpath, working_dir, parent_id=None, limit=750, remotetree=None, update_remotetree=True):
     """Upload one file
 
     Notes:
@@ -208,7 +208,7 @@ def upload_file(proj, local_abspath, mcpath, working_dir, parent_id=None, limit=
         print("uploaded:", printpath, "as", printdestpath)
     return (file_result, error_result)
 
-def check_and_upload_file(proj, local_abspath, working_dir, limit=50, no_compare=False,
+def check_and_upload_file(proj, local_abspath, working_dir, limit=750, no_compare=False,
     upload_as=None, localtree=None, remotetree=None, parent_id=None, child_data=None,
     update_remotetree=True):
     """Checks validity and upload one file
@@ -257,8 +257,8 @@ def check_and_upload_file(proj, local_abspath, working_dir, limit=50, no_compare
         error_results: str
             Error messages for unsuccessful file uploads
     """
-    file_result = None
-    error_result = None
+    file_results = {}
+    error_results = {}
 
     printpath = os.path.relpath(local_abspath, start=working_dir)
 
@@ -281,14 +281,14 @@ def check_and_upload_file(proj, local_abspath, working_dir, limit=50, no_compare
         if child_data[mcpath]['r_type'] == 'directory':
             msg = printpath + ": remote is directory (skipping)"
             print(msg)
-            error_resuts[local_abspath] = msg
+            error_results[local_abspath] = msg
             return (file_results, error_results)
 
         # if local and remote files exists, and checksums known and match -> skip, continue
         if 'eq' in child_data[mcpath] and child_data[mcpath]['eq'] is True:
             msg = printpath + ": local is equivalent to remote (skipping)"
             print(msg)
-            error_resuts[child_local_abspath] = msg
+            # error_results[child_local_abspath] = msg
             return (file_results, error_results)
 
         # else, get parent_id if not already known (might be None)
@@ -305,7 +305,7 @@ def check_and_upload_file(proj, local_abspath, working_dir, limit=50, no_compare
         if mcpath in dirs_data and dirs_data[mcpath]['r_type'] == 'directory':
             msg = printpath + ": remote is directory (skipping)"
             print(msg)
-            return (file_result, msg)
+            return (file_results, msg)
 
         # if remote file exists
         if mcpath in files_data:
@@ -315,7 +315,7 @@ def check_and_upload_file(proj, local_abspath, working_dir, limit=50, no_compare
             if 'eq' in file_data and file_data['eq'] is True:
                 msg = printpath + ": local is equivalent to remote (skipping)"
                 print(msg)
-                return (file_result, msg)
+                return (file_results, msg)
 
             # else, get parent_id if not already known (still might be None)
             if parent_id is None:
@@ -354,7 +354,7 @@ def filter_local_abspaths(proj_local_path, local_abspaths, working_dir):
     return _local_abspaths
 
 
-def check_and_upload_directory(proj, local_abspath, working_dir, limit=50,
+def check_and_upload_directory(proj, local_abspath, working_dir, limit=750,
     no_compare=False, upload_as=None, localtree=None, remotetree=None, parent_id=None):
     """Checks validity and uploads a directory and contents recursively
 
@@ -466,7 +466,7 @@ def check_and_upload_directory(proj, local_abspath, working_dir, limit=50,
         if os.path.isfile(child_local_abspath):
 
             file_result, error_msg = check_and_upload_file(proj, child_local_abspath, working_dir,
-                limit=50, no_compare=no_compare, upload_as=child_upload_as, localtree=localtree,
+                limit=limit, no_compare=no_compare, upload_as=child_upload_as, localtree=localtree,
                 remotetree=remotetree, parent_id=id, child_data=child_data, update_remotetree=True)
 
             if file_result is not None:
@@ -494,7 +494,7 @@ def check_and_upload_directory(proj, local_abspath, working_dir, limit=50,
     return (file_results, error_results)
 
 
-def standard_upload_v2(proj, paths, working_dir, recursive=False, limit=50, no_compare=False, upload_as=None, localtree=None, remotetree=None):
+def standard_upload_v2(proj, paths, working_dir, recursive=False, limit=750, no_compare=False, upload_as=None, localtree=None, remotetree=None):
     """Upload files and directories to Materials Commons
 
     Args:
@@ -590,7 +590,7 @@ def standard_upload_v2(proj, paths, working_dir, recursive=False, limit=50, no_c
 
 
 
-def standard_upload(proj, paths, working_dir, recursive=False, limit=50, no_compare=False, upload_as=None, localtree=None, remotetree=None):
+def standard_upload(proj, paths, working_dir, recursive=False, limit=750, no_compare=False, upload_as=None, localtree=None, remotetree=None):
     """Upload files to Materials Commons
 
     Args:
@@ -1270,7 +1270,7 @@ class _Mover(object):
                 print(dest_printpath + ": does not exist on remote (may not rename multiple src)")
                 valid_usage = False
         elif self.dest_remote_type != 'directory':
-            raise cliexcept.MCCLIException("Error in mv: dest_path='" + dest_path + "', dest_remote_type='" + str(dest_remote_type) + "'")
+            raise cliexcept.MCCLIException("Error in mv: dest_path='" + dest_path + "', dest_remote_type='" + str(self.dest_remote_type) + "'")
 
         # check local dest type
         if not self.remote_only:
@@ -1283,7 +1283,7 @@ class _Mover(object):
                     print(dest_printpath + ": does not exist locally (may not rename multiple src)")
                     valid_usage = False
             elif self.dest_local_type != 'directory':
-                raise cliexcept.MCCLIException("Error in mv: dest_path='" + dest_path + "', dest_remote_type='" + str(dest_local_type) + "'")
+                raise cliexcept.MCCLIException("Error in mv: dest_path='" + dest_path + "', dest_remote_type='" + str(self.dest_local_type) + "'")
 
             if self.dest_remote_type != self.dest_local_type:
                 print(dest_printpath + ": local and remote types do not match")

{materials_commons-cli-2.1.2 → materials_commons_cli-2.1.4}/materials_commons_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: materials-commons-cli
-Version: 2.1.2
+Version: 2.1.4
 Summary: Materials Commons CLI
 Home-page: https://materials-commons.github.io/materials-commons-cli/html/index.html
 Author: Materials Commons development team

{materials_commons-cli-2.1.2 → materials_commons_cli-2.1.4}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_namespace_packages
 from materials_commons.cli import __version__
 
 setup(
-    name='materials_commons-cli',
+    name='materials_commons_cli',
     version=__version__,
     description='Materials Commons CLI',
     long_description="""This package contains the materials_commons.cli module. This module is an interface

materials_commons-cli-2.1.2/materials_commons/cli/__init__.py

@@ -1 +0,0 @@
-__version__ = '2.1.2'

materials_commons-cli-2.1.2/materials_commons/cli/parser.py

@@ -1,185 +0,0 @@
-import argparse
-import json
-import os
-import sys
-from datetime import datetime, timedelta
-from io import StringIO
-
-import materials_commons.api as mcapi
-import pkg_resources
-import requests
-
-import materials_commons.cli.functions as clifuncs
-from materials_commons.cli.exceptions import MCCLIException, MissingRemoteException, \
-	MultipleRemoteException, NoDefaultRemoteException
-from materials_commons.cli.subcommands.clone import clone_subcommand
-from materials_commons.cli.subcommands.config import config_subcommand
-from materials_commons.cli.subcommands.dataset import DatasetSubcommand
-from materials_commons.cli.subcommands.down import down_subcommand
-from materials_commons.cli.subcommands.expt import ExptSubcommand
-from materials_commons.cli.subcommands.fetch import fetch_subcommand
-# from materials_commons.cli.subcommands.samp import SampSubcommand
-from materials_commons.cli.subcommands.globus import globus_subcommand
-from materials_commons.cli.subcommands.init import init_subcommand
-from materials_commons.cli.subcommands.ls import ls_subcommand
-from materials_commons.cli.subcommands.mkdir import mkdir_subcommand
-from materials_commons.cli.subcommands.mv import mv_subcommand
-# from materials_commons.cli.subcommands.proc import ProcSubcommand
-from materials_commons.cli.subcommands.proj import ProjSubcommand
-from materials_commons.cli.subcommands.remote import remote_subcommand
-from materials_commons.cli.subcommands.rm import rm_subcommand
-from materials_commons.cli.subcommands.up import up_subcommand
-from materials_commons.cli.subcommands.versions import versions_subcommand
-from materials_commons.cli.user_config import Config
-
-standard_usage = [
-	{'name': 'remote', 'desc': 'List servers', 'subcommand': remote_subcommand},
-	{'name': 'proj', 'desc': 'List projects', 'subcommand': ProjSubcommand()},
-	{'name': 'dataset', 'desc': 'List datasets', 'subcommand': DatasetSubcommand()},
-	{'name': 'expt', 'desc': 'List experiments', 'subcommand': ExptSubcommand()},
-	{'name': 'init', 'desc': 'Initialize a new project', 'subcommand': init_subcommand},
-	{'name': 'clone', 'desc': 'Clone an existing project', 'subcommand': clone_subcommand},
-	{'name': 'ls', 'desc': 'List directory contents', 'subcommand': ls_subcommand},
-	{'name': 'mkdir', 'desc': 'Make directories', 'subcommand': mkdir_subcommand},
-	{'name': 'rm', 'desc': 'Remove files and directories', 'subcommand': rm_subcommand},
-	{'name': 'mv', 'desc': 'Move files', 'subcommand': mv_subcommand},
-	{'name': 'fetch', 'desc': 'Remote data fetching and configuration', 'subcommand': fetch_subcommand},
-	{'name': 'up', 'desc': 'Upload files', 'subcommand': up_subcommand},
-	{'name': 'down', 'desc': 'Download files', 'subcommand': down_subcommand},
-	{'name': 'globus', 'desc': 'Manage Globus uploads and downloads', 'subcommand': globus_subcommand},
-	{'name': 'versions', 'desc': 'List file versions', 'subcommand': versions_subcommand},
-	# {'name': 'proc', 'desc': 'List processes', 'subcommand': ProcSubcommand()},
-	# {'name': 'samp', 'desc': 'List samples', 'subcommand': SampSubcommand()},
-	{'name': 'config', 'desc': 'Configure `mc`', 'subcommand': config_subcommand}
-]
-standard_interfaces = {d['name']: d for d in standard_usage}
-
-
-def make_parser():
-	usage_help = StringIO()
-	usage_help.write("mc <command> [<args>]\n\n")
-	usage_help.write("The standard mc commands are:\n")
-
-	for name, interface in standard_interfaces.items():
-		usage_help.write("  {:10} {:40}\n".format(name, interface['desc']))
-
-	parser = argparse.ArgumentParser(
-		description='Materials Commons command line interface',
-		usage=usage_help.getvalue())
-	parser.add_argument('command', help='Subcommand to run')
-
-	return parser
-
-
-CLI_VERSION_CACHE_FILE = os.path.expanduser('~/.materialscommons/.cli-version-cache.json')
-
-
-def should_check_version():
-	try:
-		if os.path.exists(CLI_VERSION_CACHE_FILE):
-			with open(CLI_VERSION_CACHE_FILE) as f:
-				cache = json.load(f)
-			last_check = datetime.fromisoformat(cache['last_check'])
-			# Only check once per day
-			if datetime.now() - last_check < timedelta(days=1):
-				return False
-	except Exception:
-		pass
-	return True
-
-
-def update_version_cache():
-	try:
-		with open(CLI_VERSION_CACHE_FILE, 'w') as f:
-			json.dump({
-				'last_check': datetime.now().isoformat()
-			}, f)
-	except Exception:
-		pass
-
-
-def check_package_version():
-	if not should_check_version():
-		return
-
-	package_name = 'materials-commons-cli'
-	try:
-		current_version = pkg_resources.get_distribution(package_name).version
-		response = requests.get(f"https://pypi.org/pypi/{package_name}/json")
-		latest_version = response.json()["info"]["version"]
-
-		if current_version != latest_version:
-			print(
-				f"\nWarning: You are using an older version of the Materials Commons CLI ({current_version}).\n"
-				f"A newer version is available ({latest_version}).\n"
-				f"It is recommended you upgrade to the newest version: pip install --upgrade {package_name}\n",
-				file=sys.stderr,
-			)
-
-		update_version_cache()
-
-	except Exception as e:
-		pass
-
-
-def main(argv=None, working_dir=None):
-	if argv is None:
-		argv = sys.argv
-	if working_dir is None:
-		working_dir = os.getcwd()
-	try:
-
-		config = Config()
-		if config.REST_logging:
-			mcapi.Client.set_debug_on()
-			pass
-
-		parser = make_parser()
-
-		if len(argv) < 2:
-			parser.print_help()
-			return 1
-
-		# parse_args defaults to [1:] for args, but you need to
-		# exclude the rest of the args too, or validation will fail
-		args = parser.parse_args(argv[1:2])
-
-		if args.command in standard_interfaces:
-			result = standard_interfaces[args.command]['subcommand'](argv[2:], working_dir)
-			check_package_version()
-			return result
-
-		else:
-			print('Unrecognized command')
-			parser.print_help()
-			return 1
-
-	except MissingRemoteException as e:
-		print("Error:", e)
-		clifuncs.print_remote_help()
-		return 1
-
-	except MultipleRemoteException as e:
-		print("Error:", e)
-		print(
-			'** Please edit .mc/config.json to include `"remote":{"mcurl": "Materials Commons https address here","email": "YOUR_EMAIL_HERE"}` **')
-		return 1
-
-	except NoDefaultRemoteException as e:
-		print("Error:", e)
-		print("Set the default remote with:")
-		print("    mc remote --set-default EMAIL URL")
-		clifuncs.print_remote_help()
-		return 1
-
-	except MCCLIException as e:
-		print("CLI Error:", e)
-		return 1
-
-	except mcapi.MCAPIError as e:
-		import json
-		print("API Error:", e)
-		print("Writing error message to 'mcapi_error.json'")
-		with open('mcapi_error.json', 'w') as f:
-			json.dump(e.response.json(), f, indent=2)
-		return 1