materials-commons-cli 2.3.0b10__tar.gz → 2.3.0b12__tar.gz

{materials_commons_cli-2.3.0b10 → materials_commons_cli-2.3.0b12}/PKG-INFO

@@ -1,13 +1,12 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: materials_commons_cli
-Version: 2.3.0b10
+Version: 2.3.0b12
 Summary: Materials Commons CLI
 Home-page: https://materials-commons.github.io/materials-commons-cli/html/index.html
 Author: Materials Commons development team
 Author-email: materials-commons-help@umich.edu
 License: MIT
 Keywords: materials science mc materials-commons prisms
-Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
@@ -16,10 +15,31 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8
 License-File: LICENSE
+Requires-Dist: websockets~=16.0
+Requires-Dist: aiofiles~=25.1
+Requires-Dist: globus-cli~=3.9
+Requires-Dist: globus-sdk~=3.13
+Requires-Dist: igittigitt==2.1
+Requires-Dist: materials-commons-api>=2.0.1
+Requires-Dist: python-dateutil~=2.8
+Requires-Dist: pyyaml~=6.0
+Requires-Dist: requests~=2.28
+Requires-Dist: sortedcontainers~=2.4
+Requires-Dist: tabulate~=0.9
+Requires-Dist: aiosqlite~=0.22
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary
 
 This package contains the materials_commons.cli module. This module is an interface
     to the Materials Commons project. We assume you have used (or are otherwise familiar with) the Materials
     Commons web site, https://materialscommons.org/, or a similar site based on the
     Materials Commons code (https://github.com/materials-commons/materialscommons), and intend to use these
     tools in that context.
-

materials_commons_cli-2.3.0b12/materials_commons/cli/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.3.0b12'

{materials_commons_cli-2.3.0b10 → materials_commons_cli-2.3.0b12}/materials_commons/cli/parser.py

@@ -1,16 +1,15 @@
 import argparse
 import json
 import logging
+import materials_commons.api as mcapi
 import os
 import sys
 from datetime import datetime, timedelta
 from importlib.metadata import PackageNotFoundError, version as pkg_version
 from io import StringIO
 
-import materials_commons.api as mcapi
-import requests
-
 import materials_commons.cli.old.functions as clifuncs
+import requests
 from materials_commons.cli.old.exceptions import MCCLIException, MissingRemoteException, \
     MultipleRemoteException, NoDefaultRemoteException
 from materials_commons.cli.old.user_config import Config
@@ -18,15 +17,12 @@ from materials_commons.cli.subcommands.clone import clone_subcommand
 from materials_commons.cli.subcommands.config import config_subcommand
 from materials_commons.cli.subcommands.dataset import DatasetSubcommand
 from materials_commons.cli.subcommands.down import down_subcommand
-from materials_commons.cli.subcommands.down2 import down2_subcommand
 from materials_commons.cli.subcommands.expt import ExptSubcommand
 from materials_commons.cli.subcommands.fetch import fetch_subcommand
 # from materials_commons.cli.subcommands.samp import SampSubcommand
 from materials_commons.cli.subcommands.globus import globus_subcommand
 from materials_commons.cli.subcommands.init import init_subcommand
 from materials_commons.cli.subcommands.ls import ls_subcommand
-from materials_commons.cli.subcommands.ls2 import ls2_subcommand
-from materials_commons.cli.subcommands.mcp import mcp_subcommand
 from materials_commons.cli.subcommands.mkdir import mkdir_subcommand
 from materials_commons.cli.subcommands.mv import mv_subcommand
 # from materials_commons.cli.subcommands.proc import ProcSubcommand
@@ -61,7 +57,6 @@ standard_usage = [
     # {'name': 'down2', 'desc': 'Donwload files (experimental)', 'subcommand': down2_subcommand},
     {'name': 'config', 'desc': 'Configure `mc`', 'subcommand': config_subcommand},
     {'name': 'server', 'desc': 'Run MC Server', 'subcommand': server_subcommand},
-    {'name': 'mcp', 'desc': 'Run MCP Server', 'subcommand': mcp_subcommand},
 ]
 standard_interfaces = {d['name']: d for d in standard_usage}
 
@@ -169,7 +164,7 @@ def main(argv=None, working_dir=None):
 
         if args.command in standard_interfaces:
             result = standard_interfaces[args.command]['subcommand'](argv[2:], working_dir)
-            check_package_version()
+            # check_package_version()
             return result
 
         else:

{materials_commons_cli-2.3.0b10 → materials_commons_cli-2.3.0b12}/materials_commons/cli/reconcile3.py

@@ -472,5 +472,6 @@ class SingleFileReconciler:
 
         result = await asyncio.to_thread(checksum, path.as_posix(), 1024 * 1024, progress_callback)
 
-        await TerminalProgress.render(progress_id, make_line(total_bytes, "checksum complete"))
+        await TerminalProgress.render(progress_id, make_line(total_bytes, "checksum complete"),
+                                      done=True)
         return result

{materials_commons_cli-2.3.0b10 → materials_commons_cli-2.3.0b12}/materials_commons/cli/server/downloader/file_downloader.py

@@ -368,4 +368,5 @@ class FileDownloader:
             f"{status}"
         )
 
-        await TerminalProgress.render(f"download:{self.transfer_id}", line)
+        await TerminalProgress.render(f"download:{self.transfer_id}", line,
+                                      done=status not in ("downloading", "starting"))

{materials_commons_cli-2.3.0b10 → materials_commons_cli-2.3.0b12}/materials_commons/cli/server/uploader/file_uploader.py

@@ -74,7 +74,7 @@ class FileUploader:
         """
         try:
             if self.file_size == 0:
-                print(f"Skipping {self.file_path.as_posix()} (empty file)...")
+                await TerminalProgress.write_line(f"Skipping {self.file_path.as_posix()} (empty file)...")
                 return True
 
             await self._render_progress(status="starting")
@@ -209,6 +209,21 @@ class FileUploader:
                 logger.error(f"Transfer rejected: {reason}")
             return False
 
+        elif msg["command"] == "TRANSFER_ALREADY_UPLOADED":
+            payload = msg["payload"]
+            updated_record = replace(self.upload_request.updated_record,
+                                     local_last_seen_ts=int(time.time()),
+                                     remote_checksum=payload.get("file_checksum", ""),
+                                     remote_size=payload.get("file_size", 0),
+                                     remote_file_id=payload.get("file_id", 0),
+                                     remote_ctime_ns=payload.get("file_created_at_ns", 0))
+            db_write_request = DBWriteRequest(project=self.upload_request.project,
+                                              data=updated_record,
+                                              command="single")
+            await self.db_write_queue.put(db_write_request)
+            self._already_uploaded = True
+            return False
+
         logger.error(f"Unexpected response: {msg['command']}")
         return False
 
@@ -522,7 +537,10 @@ class FileUploader:
             f"{status}"
         )
 
-        await TerminalProgress.render(self.transfer_id, line)
+        # Anything other than an in-flight status is terminal for this file, so
+        # the row is retired rather than held open for the rest of the run.
+        await TerminalProgress.render(self.transfer_id, line,
+                                      done=status not in ("uploading", "starting"))
 
     def _calculate_md5(self, chunk_size=8192) -> str:
         """Calculate md5 hash of file"""

{materials_commons_cli-2.3.0b10 → materials_commons_cli-2.3.0b12}/materials_commons/cli/server/websocket_server.py

@@ -241,7 +241,7 @@ class WebSocketCommandListener:
 
         headers["MC-Client-ID"] = self.client_uuid
         headers["MC-Client-Hostname"] = socket.gethostname()
-        headers["MC-Connection-Type"] = "cli"
+        headers["MC-Connection-Type"] = "pycli"
 
         try:
             projects = server.list_local_projects()

{materials_commons_cli-2.3.0b10 → materials_commons_cli-2.3.0b12}/materials_commons/cli/subcommands/up.py

@@ -192,6 +192,9 @@ async def ws_upload(args, working_dir):
     ignore_parser = igittigitt.IgnoreParser()
     ignore_parser.parse_rule_files(base_dir=proj.local_path, filename=".mcignore", add_default_patterns=False)
 
+    def ignore(path, _ignore):
+        return ignore_parser.match(path)
+
     # Start services
     container = ServiceContainer.create(ws_url=args.ws_url)
     service_runtime = ServiceRuntime(container)
@@ -204,7 +207,7 @@ async def ws_upload(args, working_dir):
             p = Path(path)
             if p.is_dir():
                 async for current_path, path_entries in async_reconciler.walk(path=path, listdir_fn=local_listdir,
-                                                                              recursive=args.recursive, ignore_fn=None):
+                                                                              recursive=args.recursive, ignore_fn=ignore):
                     for entry_name in sorted(path_entries):
                         file_state = path_entries[entry_name]
                         if file_state.exception:
@@ -218,6 +221,8 @@ async def ws_upload(args, working_dir):
                             transfer_id = await container.file_upload_manager.upload_file(upload_request)
                             transfer_ids.append(transfer_id)
             elif p.is_file():
+                if ignore(p, True):
+                    continue
                 file_state = await async_reconciler.reconcile_file(p)
                 if file_state.exception:
                     logger.error(f"Error encountered while processing {p}: {file_state.exception}")

materials_commons_cli-2.3.0b12/materials_commons/cli/terminal_progress.py

@@ -0,0 +1,107 @@
+import asyncio
+import shutil
+import sys
+from collections import OrderedDict
+from typing import Iterable, Optional
+
+def format_bytes(value: float) -> str:
+    """Format bytes as a human-readable value."""
+    units = ("B", "KB", "MB", "GB", "TB")
+    size = float(value)
+
+    for unit in units:
+        if size < 1024.0 or unit == units[-1]:
+            return f"{size:.1f} {unit}"
+        size /= 1024.0
+
+    return f"{size:.1f} TB"
+
+class TerminalProgress:
+    """Renders in-progress transfers as a block of lines pinned to the bottom of
+    the terminal.
+
+    The block is repainted in its entirety on every update, using only cursor
+    movements relative to the line just below the block. Absolute row addressing
+    is deliberately avoided: it breaks as soon as the terminal scrolls, the
+    window is resized, or more transfers are tracked than the screen has rows.
+    """
+
+    _lock: Optional[asyncio.Lock] = None
+    _active: "OrderedDict[str, str]" = OrderedDict()
+
+    # Number of terminal rows the live block currently occupies. The cursor is
+    # always left at column 0 of the row immediately below the block.
+    _painted = 0
+
+    @classmethod
+    def _get_lock(cls) -> asyncio.Lock:
+        # Created lazily so the lock binds to the running loop, not to whatever
+        # loop happened to exist at import time.
+        if cls._lock is None:
+            cls._lock = asyncio.Lock()
+        return cls._lock
+
+    @staticmethod
+    def _fit(line: str, width: int) -> str:
+        # Stay one column short of the edge; writing into the last column
+        # triggers deferred wrapping on most terminals, which would make the
+        # line occupy two rows and throw off the repaint arithmetic.
+        limit = max(width - 1, 8)
+        if len(line) <= limit:
+            return line
+        return line[:limit - 3] + "..."
+
+    @classmethod
+    def _repaint(cls, permanent_lines: Iterable[str] = ()) -> None:
+        size = shutil.get_terminal_size(fallback=(80, 24))
+        max_rows = max(size.lines - 1, 1)
+
+        out = []
+        if cls._painted:
+            out.append(f"\033[{cls._painted}A")
+        out.append("\r")
+
+        # Lines that are done scroll away into the terminal's scrollback.
+        for text in permanent_lines:
+            out.append("\033[2K" + cls._fit(text, size.columns) + "\n")
+
+        # Only as many rows as fit on screen; anything above the top would be
+        # clamped by the terminal and repaint over itself.
+        visible = list(cls._active.values())[-max_rows:]
+        for text in visible:
+            out.append("\033[2K" + cls._fit(text, size.columns) + "\n")
+
+        # Clear whatever the previous, taller block left behind.
+        out.append("\033[J")
+
+        cls._painted = len(visible)
+        sys.stdout.write("".join(out))
+        sys.stdout.flush()
+
+    @classmethod
+    async def render(cls, progress_id: str, line: str, done: bool = False):
+        """Update the row for ``progress_id``.
+
+        When ``done`` is set the row is retired: it is written out as permanent
+        output above the live block and no longer takes up a row.
+        """
+        if not sys.stdout.isatty():
+            return
+
+        async with cls._get_lock():
+            if done:
+                cls._active.pop(progress_id, None)
+                cls._repaint([line])
+            else:
+                cls._active[progress_id] = line
+                cls._repaint()
+
+    @classmethod
+    async def write_line(cls, line: str):
+        """Emit a normal output line without corrupting the progress block."""
+        if not sys.stdout.isatty():
+            print(line)
+            return
+
+        async with cls._get_lock():
+            cls._repaint([line])

{materials_commons_cli-2.3.0b10 → materials_commons_cli-2.3.0b12}/materials_commons_cli.egg-info/PKG-INFO

@@ -1,13 +1,12 @@
-Metadata-Version: 2.1
-Name: materials-commons-cli
-Version: 2.3.0b10
+Metadata-Version: 2.4
+Name: materials_commons_cli
+Version: 2.3.0b12
 Summary: Materials Commons CLI
 Home-page: https://materials-commons.github.io/materials-commons-cli/html/index.html
 Author: Materials Commons development team
 Author-email: materials-commons-help@umich.edu
 License: MIT
 Keywords: materials science mc materials-commons prisms
-Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
@@ -16,10 +15,31 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8
 License-File: LICENSE
+Requires-Dist: websockets~=16.0
+Requires-Dist: aiofiles~=25.1
+Requires-Dist: globus-cli~=3.9
+Requires-Dist: globus-sdk~=3.13
+Requires-Dist: igittigitt==2.1
+Requires-Dist: materials-commons-api>=2.0.1
+Requires-Dist: python-dateutil~=2.8
+Requires-Dist: pyyaml~=6.0
+Requires-Dist: requests~=2.28
+Requires-Dist: sortedcontainers~=2.4
+Requires-Dist: tabulate~=0.9
+Requires-Dist: aiosqlite~=0.22
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary
 
 This package contains the materials_commons.cli module. This module is an interface
     to the Materials Commons project. We assume you have used (or are otherwise familiar with) the Materials
     Commons web site, https://materialscommons.org/, or a similar site based on the
     Materials Commons code (https://github.com/materials-commons/materialscommons), and intend to use these
     tools in that context.
-

{materials_commons_cli-2.3.0b10 → materials_commons_cli-2.3.0b12}/materials_commons_cli.egg-info/SOURCES.txt

@@ -69,7 +69,6 @@ materials_commons/cli/subcommands/globus.py
 materials_commons/cli/subcommands/init.py
 materials_commons/cli/subcommands/ls.py
 materials_commons/cli/subcommands/ls2.py
-materials_commons/cli/subcommands/mcp.py
 materials_commons/cli/subcommands/mkdir.py
 materials_commons/cli/subcommands/mv.py
 materials_commons/cli/subcommands/proj.py
@@ -101,4 +100,6 @@ test/test_mc_up.py
 test/test_treecompare_no_cache.py
 test/test_treecompare_with_cache.py
 test/test_treedb.py
-test/test_user_config.py
+test/test_user_config.py
+tests/test_file_uploader.py
+tests/test_reconcile3.py

{materials_commons_cli-2.3.0b10 → materials_commons_cli-2.3.0b12}/materials_commons_cli.egg-info/requires.txt

@@ -7,8 +7,6 @@ materials-commons-api>=2.0.1
 python-dateutil~=2.8
 pyyaml~=6.0
 requests~=2.28
-setuptools~=60.2
 sortedcontainers~=2.4
 tabulate~=0.9
 aiosqlite~=0.22
-fastmcp~=3.4

{materials_commons_cli-2.3.0b10 → materials_commons_cli-2.3.0b12}/setup.py

@@ -68,10 +68,8 @@ setup(
         "python-dateutil~=2.8",
         "pyyaml~=6.0",
         "requests~=2.28",
-        "setuptools~=60.2",
         "sortedcontainers~=2.4",
         "tabulate~=0.9",
         "aiosqlite~=0.22",
-        "fastmcp~=3.4",
     ]
 )