materials-commons-cli 2.1.8__tar.gz → 2.2.0__tar.gz

{materials_commons_cli-2.1.8 → materials_commons_cli-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: materials_commons_cli
-Version: 2.1.8
+Version: 2.2.0
 Summary: Materials Commons CLI
 Home-page: https://materials-commons.github.io/materials-commons-cli/html/index.html
 Author: Materials Commons development team

materials_commons_cli-2.2.0/materials_commons/cli/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.2.0'

{materials_commons_cli-2.1.8 → materials_commons_cli-2.2.0}/materials_commons/cli/functions.py

@@ -134,15 +134,23 @@ def format_time(time_value, fmt="%Y %b %d %H:%M:%S"):
     else:
         return str(type(time_value))
 
-def checksum(path, chunk_size=131072):
-    """Generate MD5 checksum for the file at "path" """
+def checksum(path, chunk_size=8192):
+    """Generate MD5 checksum for the file at "path"
+
+    Args:
+        path (str): Path to file
+        chunk_size (int): Size of chunks to read (default 8192 bytes)
+
+    Returns:
+        str: MD5 checksum hexdigest
+    """
     md5 = hashlib.md5()
     with open(path, 'rb') as f:
         while True:
-            data = f.read(chunk_size)
-            if not data:
+            chunk = f.read(chunk_size)
+            if not chunk:
                 break
-            md5.update(data)
+            md5.update(chunk)
     return md5.hexdigest()
 
 def random_name(n=3, max_letters=6, sep='-'):
@@ -449,7 +457,7 @@ def make_local_project(path, data=None):
             else:
                 proj = models.Project(data=data)
         except requests.exceptions.ConnectionError as e:
-            raise MCCLIException("Could not connect to " + remote.config.mcurl)
+            raise MCCLIException("Could not connect to " + project_config.remote.mcurl)
         except requests.exceptions.HTTPError as e:
             raise MCCLIException("HTTPError: " + str(e))
 

{materials_commons_cli-2.1.8 → materials_commons_cli-2.2.0}/materials_commons/cli/parser.py

@@ -31,6 +31,7 @@ from materials_commons.cli.subcommands.rm import rm_subcommand
 from materials_commons.cli.subcommands.up import up_subcommand
 from materials_commons.cli.subcommands.versions import versions_subcommand
 from materials_commons.cli.user_config import Config
+from materials_commons.cli.subcommands.server import server_subcommand
 
 standard_usage = [
     {'name': 'remote', 'desc': 'List servers', 'subcommand': remote_subcommand},
@@ -50,7 +51,8 @@ standard_usage = [
     {'name': 'versions', 'desc': 'List file versions', 'subcommand': versions_subcommand},
     # {'name': 'proc', 'desc': 'List processes', 'subcommand': ProcSubcommand()},
     # {'name': 'samp', 'desc': 'List samples', 'subcommand': SampSubcommand()},
-    {'name': 'config', 'desc': 'Configure `mc`', 'subcommand': config_subcommand}
+    {'name': 'config', 'desc': 'Configure `mc`', 'subcommand': config_subcommand},
+    {'name': 'server', 'desc': 'Run MC Server', 'subcommand': server_subcommand}
 ]
 standard_interfaces = {d['name']: d for d in standard_usage}
 
@@ -60,8 +62,13 @@ def make_parser():
     usage_help.write("mc <command> [<args>]\n\n")
     usage_help.write("The standard mc commands are:\n")
 
+    beta = os.getenv('MC_BETA')
     for name, interface in standard_interfaces.items():
-        usage_help.write("  {:10} {:40}\n".format(name, interface['desc']))
+        if name == 'server':
+            if beta:
+                usage_help.write("  {:10} {:40}\n".format(name, interface['desc']))
+        else:
+            usage_help.write("  {:10} {:40}\n".format(name, interface['desc']))
 
     parser = argparse.ArgumentParser(
         description='Materials Commons command line interface',
@@ -130,7 +137,6 @@ def main(argv=None, working_dir=None):
     if working_dir is None:
         working_dir = os.getcwd()
     try:
-
         config = Config()
         if config.REST_logging:
             mcapi.Client.set_debug_on()

materials_commons_cli-2.2.0/materials_commons/cli/subcommands/server.py

@@ -0,0 +1,75 @@
+import argparse
+import asyncio
+import signal
+import uuid
+from typing import Awaitable, Callable, Dict, Any
+
+from materials_commons.cli.desktop.command_handlers import register_handlers
+from materials_commons.cli.desktop.websocket_server import WebSocketCommandListener
+from materials_commons.cli.user_config import Config
+from materials_commons.cli.desktop.local_rest_server import LocalRestServer
+
+CommandHandler = Callable[[Dict[str, Any]], Awaitable[None]]
+
+DEFAULT_WS_URL = "wss://materialscommons.org/ws"
+DEFAULT_RECONNECT_MIN_SEC = 1
+DEFAULT_RECONNECT_MAX_SEC = 30
+
+
+def _install_signal_handlers(loop: asyncio.AbstractEventLoop):
+    # Installs signal handlers for graceful shutdown
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        try:
+            loop.add_signal_handler(sig, lambda s=sig: asyncio.create_task(_graceful_stop(loop)), None)
+        except NotImplementedError:
+            signal.signal(sig, lambda *_: asyncio.create_task(_graceful_stop(loop)))
+
+
+async def _graceful_stop(loop: asyncio.AbstractEventLoop):
+    """Cancels pending tasks then stops the event loop"""
+    tasks = [t for t in asyncio.all_tasks(loop) if t is not asyncio.current_task(loop)]
+    for t in tasks:
+        t.cancel()
+    await asyncio.gather(*tasks, return_exceptions=True)
+    loop.stop()
+
+
+def server_subcommand(argv, working_dir=None):
+    parser = argparse.ArgumentParser(description='Start the Materials Commons server')
+    parser.add_argument('--ws-url', type=str, default=DEFAULT_WS_URL, help='WebSocket URL for commands')
+    args = parser.parse_args(argv)
+
+    config = Config()
+    if config.client_uuid is None:
+        config.client_uuid = str(uuid.uuid4())
+        config.save()
+
+    # Runs local REST server and remote command listener
+    async def main():
+        queue = asyncio.Queue()
+        running_loop = asyncio.get_running_loop()
+        local_rest_server = LocalRestServer(loop=running_loop, queue=queue)
+        local_rest_server.start()
+
+        listener = WebSocketCommandListener(args.ws_url, config.default_remote.mcapikey, config.client_uuid,
+                                            register_handlers(), queue)
+        try:
+            await listener.run()
+        finally:
+            await listener.shutdown()
+            local_rest_server.stop()
+
+    # Creates event loop; runs main; handles interrupts
+    try:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        _install_signal_handlers(loop)
+        loop.run_until_complete(main())
+    except KeyboardInterrupt:
+        pass
+    finally:
+        try:
+            loop.run_until_complete(_graceful_stop(loop))
+        except Exception:
+            pass
+        loop.close()

{materials_commons_cli-2.1.8 → materials_commons_cli-2.2.0}/materials_commons/cli/subcommands/up.py

@@ -1,10 +1,16 @@
 import argparse
+import logging
+import sys
 
 import materials_commons.cli.exceptions as cliexcept
 import materials_commons.cli.functions as clifuncs
 import materials_commons.cli.globus as cliglobus
 import materials_commons.cli.tree_functions as treefuncs
 from materials_commons.cli.treedb import LocalTree, RemoteTree
+from materials_commons.cli.subcommands.server import DEFAULT_WS_URL
+from materials_commons.cli.desktop.uploader.api import ws_upload_synchronous
+import os
+from pathlib import Path
 
 
 def make_parser():
@@ -14,11 +20,14 @@ def make_parser():
 
     mc_up_usage = """
     mc up [-r] [--no-compare] [--limit] <pathspec> [<pathspec> ...]
-    mc up -g [-r] [--no-compare] [--label] <pathspec> [<pathspec> ...]"""
+    mc up -g [-r] [--no-compare] [--label] <pathspec> [<pathspec> ...]
+    mc up --websocket [-r] <pathspec> [<pathspec> ...]"""
 
     globus_help = """Use globus to upload files. Uses the current active upload or creates a new upload.
      Use `globus task list` to monitor transfer tasks. Use `mc globus upload` to manage uploads."""
 
+    websocket_help = """Use websockets to upload files. Files are uploaded in parallel and progress is reported."""
+
     parser = argparse.ArgumentParser(
         description=mc_up_description,
         usage=mc_up_usage,
@@ -30,24 +39,40 @@ def make_parser():
                         help='File size upload limit (MB). Default=750MB. Max size is also 750MB. Does not apply to Globus uploads.')
     parser.add_argument('-g', '--globus', action="store_true", default=False,
                         help=globus_help)
+    parser.add_argument('--websocket', '--ws', action="store_true", default=False, help=websocket_help)
+    parser.add_argument('--ws-url', type=str, default=DEFAULT_WS_URL, help='WebSocket URL for commands')
     parser.add_argument('--label', nargs=1, type=str,
                         help='Globus transfer label to make finding tasks simpler. Default is `<project name>-<upload name>.')
     parser.add_argument('--no-compare', action="store_true", default=False,
                         help='Upload without checking if remote is equivalent.')
-    parser.add_argument('--upload-as', nargs=1, default=None, help='Upload to a different location than standard upload. Specified as if it were a local path.')
+    parser.add_argument('--upload-as', nargs=1, default=None,
+                        help='Upload to a different location than standard upload. Specified as if it were a local path.')
+    parser.add_argument('--chunk-size', type=int, default=1024 * 1024,
+                        help='Chunk size for websocket uploads (default: 1MB)')
+    parser.add_argument('--max-concurrent', type=int, default=3,
+                        help='Maximum concurrent uploads for websocket mode (default: 3)')
     return parser
 
+
 def up_subcommand(argv, working_dir):
     """
     upload files to Materials Commons
 
     mc up [-r] [--no-compare] [--limit] <pathspec> [<pathspec> ...]
     mc up -g [-r] [--no-compare] [--label] <pathspec> [<pathspec> ...]
-
+    mc up --ws [-r] <pathspec> [<pathspec> ...]
     """
     parser = make_parser()
     args = parser.parse_args(argv)
 
+    showLogs = os.getenv("MC_SHOW_LOGS")
+    if showLogs == "true":
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            handlers=[logging.StreamHandler(sys.stdout)]
+        )
+
     proj = clifuncs.make_local_project(working_dir)
 
     pconfig = clifuncs.read_project_config(proj.local_path)
@@ -62,6 +87,12 @@ def up_subcommand(argv, working_dir):
     if args.upload_as and args.globus:
         print("--upload-as option is not supported with --globus")
         raise cliexcept.MCCLIException("Invalid upload request")
+    if args.upload_as and args.websocket:
+        print("--upload-as option is not supported with --websocket")
+        raise cliexcept.MCCLIException("Invalid upload request")
+    if args.globus and args.websocket:
+        print("--globus option is not supported with --websocket")
+        raise cliexcept.MCCLIException("Invalid upload request")
 
     upload_as = None
     if args.upload_as:
@@ -69,7 +100,56 @@ def up_subcommand(argv, working_dir):
                                                   args.upload_as,
                                                   working_dir)[0]
 
-    if args.globus:
+    if args.websocket:
+        # convert input paths (absolute or relative to working_dir) to local_abspath
+        local_abspaths = treefuncs.clipaths_to_local_abspaths(
+            proj.local_path, args.paths, working_dir)
+
+        # filter, skipping .mc, those specified by .mcignore
+        local_abspaths = treefuncs.filter_local_abspaths(
+            proj.local_path, local_abspaths, working_dir)
+
+        mcpaths = treefuncs.clipaths_to_mcpaths(proj.local_path, local_abspaths, working_dir)
+
+        file_paths = []
+        project_paths = []
+
+        for local_path, mc_path in zip(local_abspaths, mcpaths):
+            if treefuncs.os.path.isdir(local_path):
+                if args.recursive:
+                    for root, dirs, files in os.walk(local_path):
+                        for filename in files:
+                            file_path = os.path.join(root, filename)
+
+                            # Calculate relative path and construct mc_path
+                            relative = Path(file_path).relative_to(local_path)
+                            file_mc_path = str(Path(mc_path) / relative)
+                            file_paths.append(file_path)
+                            project_paths.append(file_mc_path)
+                else:
+                    print("Skipping directory because --recursive option not specified: " + local_path)
+            else:
+                file_paths.append(local_path)
+                project_paths.append(mc_path)
+        if not file_paths:
+            print("No files to upload.")
+            return
+
+        status = ws_upload_synchronous(
+            file_paths=file_paths,
+            project_paths=project_paths,
+            project_id=proj.id,
+            chunk_size=args.chunk_size,
+            max_concurrent=args.max_concurrent,
+            ws_url=args.ws_url
+        )
+        if not status:
+            print("\nSome uploads failed.")
+            raise cliexcept.MCCLIException(
+                "Upload failed. Check server logs for details.")
+
+        print("\nAll uploads completed successfully!")
+    elif args.globus:
 
         # convert input paths (absolute or relative to working_dir) to local_abspath
         local_abspaths = treefuncs.clipaths_to_local_abspaths(
@@ -81,7 +161,7 @@ def up_subcommand(argv, working_dir):
 
         mcpaths = treefuncs.clipaths_to_mcpaths(proj.local_path, local_abspaths, working_dir)
 
-        all_uploads = {upload.id:upload for upload in proj.remote.get_all_globus_upload_requests(proj.id)}
+        all_uploads = {upload.id: upload for upload in proj.remote.get_all_globus_upload_requests(proj.id)}
 
         globus_upload_id = None
         if pconfig.globus_upload_id:
@@ -99,8 +179,9 @@ def up_subcommand(argv, working_dir):
             upload = all_uploads[globus_upload_id]
             print("Using current globus upload (name=" + upload.name + ", id=" + str(upload.id) + ").")
 
-        if upload.status != 2:    # TODO clean up status code / message
-            raise cliexcept.MCCLIException("Current Globus upload (id=" + str(globus_upload_id) + ") not ready for uploading.")
+        if upload.status != 2:  # TODO clean up status code / message
+            raise cliexcept.MCCLIException(
+                "Current Globus upload (id=" + str(globus_upload_id) + ") not ready for uploading.")
 
         label = proj.name + "-" + upload.name
         if args.label:
@@ -117,7 +198,7 @@ def up_subcommand(argv, working_dir):
             print("Use `mc globus upload` to manage Globus uploads.")
             print("Multiple transfer tasks may be initiated.")
             print("When all tasks finish uploading, use `mc globus upload --id " + str(upload.id) +
-                " --finish` " + "to import all uploaded files into the Materials Commons project.")
+                  " --finish` " + "to import all uploaded files into the Materials Commons project.")
 
     else:
         localtree = None
@@ -125,9 +206,9 @@ def up_subcommand(argv, working_dir):
             localtree = LocalTree(proj.local_path)
 
         treefuncs.standard_upload_v2(proj, args.paths, working_dir,
-                                  recursive=args.recursive, limit=args.limit[0],
-                                  no_compare=args.no_compare,
-                                  upload_as=upload_as, localtree=localtree,
-                                  remotetree=remotetree)
+                                     recursive=args.recursive, limit=args.limit[0],
+                                     no_compare=args.no_compare,
+                                     upload_as=upload_as, localtree=localtree,
+                                     remotetree=remotetree)
 
     return

{materials_commons_cli-2.1.8 → materials_commons_cli-2.2.0}/materials_commons/cli/user_config.py

@@ -1,10 +1,10 @@
 import getpass
+import json
 import os
-import requests
-import warnings
+import re
 from os.path import join
-import json
 
+import requests
 from materials_commons.api.client import Client
 
 
@@ -94,7 +94,8 @@ class Config(object):
                 'mcurl': None,
                 'email': None,
                 'remotes': {},
-                'globus': {}
+                'globus': {},
+                'client_uuid': None,
             }
 
         # check for recognized environment variables
@@ -137,6 +138,7 @@ class Config(object):
 
         self.developer_mode = config.get('developer_mode', False)
         self.REST_logging = config.get('REST_logging', False)
+        self.client_uuid = config.get('client_uuid', None)
 
     def save(self):
         config = {
@@ -144,7 +146,8 @@ class Config(object):
             'remotes': [vars(value) for value in self.remotes],
             'globus': vars(self.globus),
             'developer_mode': self.developer_mode,
-            'REST_logging': self.REST_logging
+            'REST_logging': self.REST_logging,
+            'client_uuid': self.client_uuid or '',
         }
         if not os.path.exists(self.config_file):
             user = getpass.getuser()

{materials_commons_cli-2.1.8 → materials_commons_cli-2.2.0}/materials_commons_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: materials-commons-cli
-Version: 2.1.8
+Version: 2.2.0
 Summary: Materials Commons CLI
 Home-page: https://materials-commons.github.io/materials-commons-cli/html/index.html
 Author: Materials Commons development team

{materials_commons_cli-2.1.8 → materials_commons_cli-2.2.0}/materials_commons_cli.egg-info/SOURCES.txt

@@ -32,6 +32,7 @@ materials_commons/cli/subcommands/mv.py
 materials_commons/cli/subcommands/proj.py
 materials_commons/cli/subcommands/remote.py
 materials_commons/cli/subcommands/rm.py
+materials_commons/cli/subcommands/server.py
 materials_commons/cli/subcommands/up.py
 materials_commons/cli/subcommands/versions.py
 materials_commons_cli.egg-info/PKG-INFO

{materials_commons_cli-2.1.8 → materials_commons_cli-2.2.0}/materials_commons_cli.egg-info/requires.txt

@@ -8,3 +8,4 @@ requests
 setuptools
 sortedcontainers
 tabulate
+websockets

{materials_commons_cli-2.1.8 → materials_commons_cli-2.2.0}/setup.py

@@ -49,6 +49,7 @@ setup(
     ],
     keywords='materials science mc materials-commons prisms',
     install_requires=[
+        "websockets",
         "globus-cli",
         "globus-sdk",
         "igittigitt==2.1.0",

materials_commons_cli-2.1.8/materials_commons/cli/__init__.py

@@ -1 +0,0 @@
-__version__ = '2.1.8'