materials-commons-cli 2.3.0b9__tar.gz → 2.3.0b10__tar.gz

{materials_commons_cli-2.3.0b9 → materials_commons_cli-2.3.0b10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: materials_commons_cli
-Version: 2.3.0b9
+Version: 2.3.0b10
 Summary: Materials Commons CLI
 Home-page: https://materials-commons.github.io/materials-commons-cli/html/index.html
 Author: Materials Commons development team

materials_commons_cli-2.3.0b10/materials_commons/cli/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.3.0b10'

{materials_commons_cli-2.3.0b9 → materials_commons_cli-2.3.0b10}/materials_commons/cli/old/functions.py

@@ -144,28 +144,33 @@ def format_time(time_value, fmt="%Y %b %d %H:%M:%S") -> str:
         return str(type(time_value))
 
 
-def checksum(path, chunk_size=8192):
-    """Generate MD5 checksum for the file at "path"
+def checksum(path, chunk_size=1024*1024, progress_callback=None):
+    """Generate MD5 checksum for the file at path
 
     Args:
         path (str): Path to file
         chunk_size (int): Size of chunks to read (default 8192 bytes)
+        progress_callback (callable): Callback function to report progress as progress_callback(bytes_read)
 
     Returns:
         str: MD5 checksum hexdigest
     """
     md5 = hashlib.md5()
+    bytes_read = 0
     with open(path, 'rb') as f:
         while True:
             chunk = f.read(chunk_size)
             if not chunk:
                 break
             md5.update(chunk)
+            bytes_read += len(chunk)
+            if progress_callback:
+                progress_callback(bytes_read)
     return md5.hexdigest()
 
 
-async def checksum_async(path, chunk_size=8192):
-    """Generate MD5 checksum for the file at "path"""
+async def checksum_async(path, chunk_size=1024*1024):
+    """Generate MD5 checksum for the file at path"""
     return await asyncio.to_thread(checksum, path, chunk_size)
 
 

{materials_commons_cli-2.3.0b9 → materials_commons_cli-2.3.0b10}/materials_commons/cli/parser.py

@@ -1,5 +1,6 @@
 import argparse
 import json
+import logging
 import os
 import sys
 from datetime import datetime, timedelta
@@ -144,6 +145,8 @@ def check_package_version():
 
 
 def main(argv=None, working_dir=None):
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.WARNING)
     if argv is None:
         argv = sys.argv
     if working_dir is None:

{materials_commons_cli-2.3.0b9 → materials_commons_cli-2.3.0b10}/materials_commons/cli/reconcile3.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import time
 from dataclasses import replace
 from typing import Literal, Optional
 
@@ -9,6 +10,7 @@ from materials_commons.api import models as mcmodel
 from materials_commons.cli.local_project import LocalProject
 from materials_commons.cli.models import FileRecord, Observation, FileDecision
 from materials_commons.cli.old.functions import checksum
+from materials_commons.cli.terminal_progress import TerminalProgress
 
 Action = Literal["skip", "upload", "download", "conflict", "adopt", "db_update"]
 ObservationState = Literal["neither", "local_only", "remote_only", "both"]
@@ -141,12 +143,12 @@ class SingleFileReconciler:
             if observation.local_entry_matches_record() and observation.file_record.local_checksum:
                 return self._upload(record, "local only")
             else:
-                local_checksum = await asyncio.to_thread(checksum, observation.local_entry.path.as_posix())
+                local_checksum = await self._checksum_with_feedback(observation)
                 record = replace(record, local_checksum=local_checksum)
                 reason = "local only - update local record"
                 return self._upload(record, reason)
         else:
-            local_checksum = await asyncio.to_thread(checksum, observation.local_entry.path.as_posix())
+            local_checksum = await self._checksum_with_feedback(observation)
             record = replace(record, local_checksum=local_checksum)
             reason = "local only - add local record"
             return self._upload(record, reason)
@@ -157,13 +159,13 @@ class SingleFileReconciler:
             if observation.local_entry_matches_record():
                 return self._skip(record, "local entry matches existing record; no remote file to download")
             else:
-                local_checksum = await asyncio.to_thread(checksum, observation.local_entry.path.as_posix())
+                local_checksum = await self._checksum_with_feedback(observation)
                 record = replace(record, local_checksum=local_checksum)
                 reason = "local only, uploadable - update local record"
                 return self._db_update(record, reason)
         else:
             # No record in the project database
-            local_checksum = await asyncio.to_thread(checksum, observation.local_entry.path.as_posix())
+            local_checksum = await self._checksum_with_feedback(observation)
             record = replace(record, local_checksum=local_checksum)
             reason = "local only, uploadable - add local record"
             return self._db_update(record, reason)
@@ -265,14 +267,14 @@ class SingleFileReconciler:
                 # No action needed, local has been uploaded, but remote changed, and we could download it.
                 return self._skip(updated_record, "local matches remote, but remote file differs (downloadable)")
         # Local entry is different. Compute new checksum and upload.
-        local_checksum = await asyncio.to_thread(checksum, observation.local_entry.path.as_posix())
+        local_checksum = await self._checksum_with_feedback(observation)
         updated_record = replace(updated_record, local_checksum=local_checksum)
         return self._upload(updated_record, "local file changed")
 
     async def _reconcile_both_sides_upload_no_record(self, observation: Observation,
                                                      updated_record: FileRecord) -> FileDecision:
         # No file record. Let's compute the checksum and see if that matches the remote
-        local_checksum = await asyncio.to_thread(checksum, observation.local_entry.path.as_posix())
+        local_checksum = await self._checksum_with_feedback(observation)
         if local_checksum == observation.remote_entry.checksum:
             return self._db_update(updated_record, "local matches remote, add file record")
         updated_record = replace(updated_record, local_checksum=local_checksum)
@@ -304,7 +306,7 @@ class SingleFileReconciler:
         else:
             # The file record and the local file don't match. We need to compute the checksum for the
             # local to determine if the local file has ever been uploaded.
-            local_checksum = await asyncio.to_thread(checksum, observation.local_entry.path.as_posix())
+            local_checksum = await self._checksum_with_feedback(observation)
             # Check if the local checksum matches the file record checksum
             if local_checksum == observation.file_record.local_checksum:
                 # The checksum matches, so this file has been uploaded. So we can safely download the remote.
@@ -328,7 +330,7 @@ class SingleFileReconciler:
         """
         # There is no file_record; that means we don't know the state of the local file. First, lets compute
         # the checksum of the local file.
-        local_checksum = await asyncio.to_thread(checksum, observation.local_entry.path.as_posix())
+        local_checksum = await self._checksum_with_feedback(observation)
         if local_checksum == observation.remote_entry.checksum:
             # Need to create a file_record
             return self._db_update(updated_record, "local file matches remote checksum")
@@ -372,7 +374,8 @@ class SingleFileReconciler:
 
         # Get previous versions and look for a match
         try:
-            previous_versions = await asyncio.to_thread(self._proj.remote.get_file_versions, self._proj.id, remote_file_id)
+            previous_versions = await asyncio.to_thread(self._proj.remote.get_file_versions, self._proj.id,
+                                                        remote_file_id)
         except Exception as e:
             previous_versions = None
 
@@ -432,3 +435,42 @@ class SingleFileReconciler:
 
     def _db_update(self, record: FileRecord, reason: str) -> FileDecision:
         return self._decision("db_update", record, reason, updated=True)
+
+    async def _checksum_with_feedback(self, observation: Observation) -> str:
+        path = observation.local_entry.path
+        total_bytes = observation.local_entry.size
+        progress_id = f"checksum:{path.as_posix()}"
+        loop = asyncio.get_running_loop()
+
+        def make_line(bytes_read: int, status: str = "computing checksum") -> str:
+            percent = 100.0 if total_bytes == 0 else min((bytes_read / total_bytes) * 100.0, 100.0)
+            return (
+                f"{path.as_posix()} | "
+                f"checksum | "
+                f"{bytes_read} / {total_bytes} bytes | "
+                f"{percent:6.2f}% | "
+                f"{status}"
+            )
+
+        last_render_at = 0.0
+
+        def progress_callback(bytes_read: int) -> None:
+            nonlocal last_render_at
+
+            now = time.monotonic()
+            if now - last_render_at < 1.0 and bytes_read < total_bytes:
+                return
+
+            last_render_at = now
+            line = make_line(bytes_read)
+            asyncio.run_coroutine_threadsafe(
+                TerminalProgress.render(progress_id, line),
+                loop,
+            )
+
+        await TerminalProgress.render(progress_id, make_line(0))
+
+        result = await asyncio.to_thread(checksum, path.as_posix(), 1024 * 1024, progress_callback)
+
+        await TerminalProgress.render(progress_id, make_line(total_bytes, "checksum complete"))
+        return result

{materials_commons_cli-2.3.0b9 → materials_commons_cli-2.3.0b10}/materials_commons/cli/server/downloader/file_download_manager.py

@@ -1,9 +1,11 @@
 import asyncio
 from typing import Dict, Any, Optional, Callable
+import logging
 
 from materials_commons.cli.requests import DownloadRequest
 from materials_commons.cli.server.downloader.file_downloader import FileDownloader, logger
 
+logger.setLevel(logging.WARNING)
 
 class FileDownloadManager:
     """Manages multiple concurrent file downloads"""

{materials_commons_cli-2.3.0b9 → materials_commons_cli-2.3.0b10}/materials_commons/cli/server/downloader/file_downloader.py

@@ -13,6 +13,7 @@ import requests
 from materials_commons.cli.old.functions import checksum
 from materials_commons.cli.requests import DownloadRequest, DBWriteRequest
 from materials_commons.cli.server import projects
+from materials_commons.cli.terminal_progress import TerminalProgress
 
 logger = logging.getLogger(__name__)
 
@@ -154,25 +155,35 @@ class FileDownloader:
 
     async def _download_with_ranges(self, resume_from: int = 0) -> bool:
         """Download the file using HTTP Range requests with streaming"""
+
+        # Get the event loop so we can schedule progress rendering. Since the routine
+        # _download_with_ranges_blocking is not async it will need the loop do the
+        # progress. We have to get the async loop here in the async routine, and then
+        # pass it to the blocking download routine.
+        loop = asyncio.get_running_loop()
         try:
-            success = await asyncio.to_thread(self._download_with_ranges_blocking, resume_from)
+            await self._render_progress(status="starting")
+            success = await asyncio.to_thread(self._download_with_ranges_blocking, resume_from, loop)
 
             if not success:
                 await self._save_metadata()
+                await self._render_progress(status="failed downloading")
+                return False
 
             return success
 
         except requests.exceptions.RequestException as e:
             logger.error(f"HTTP request failed: {e}")
             await self._save_metadata()
+            await self._render_progress(status=f"failed with HTTP exception: {e}")
             return False
         except Exception as e:
             logger.error(f"Download error: {e}")
             await self._save_metadata()
+            await self._render_progress(status=f"failed with exception: {e}")
             return False
 
-    def _download_with_ranges_blocking(self, resume_from: int = 0) -> bool:
-        "Blocking download. Called from asyncio.to_thread()"
+    def _download_with_ranges_blocking(self, resume_from: int = 0, loop: Optional[asyncio.AbstractEventLoop] = None) -> bool:
         """Blocking download implementation. Runs in a worker thread."""
         headers = {
             'Authorization': f'Bearer {self.apitoken}',
@@ -213,6 +224,26 @@ class FileDownloader:
 
             # Open the file in append mode if resuming, write mode otherwise
             mode = 'ab' if resume_from > 0 else 'wb'
+            last_progress_render_at = 0.0
+            progress_interval_seconds = 0.25
+
+            def schedule_progress_render(status: str = "downloading", force: bool = False) -> None:
+                nonlocal last_progress_render_at
+
+                if loop is None:
+                    return
+
+                now = time.monotonic()
+                if not force and now - last_progress_render_at < progress_interval_seconds:
+                    return
+
+                last_progress_render_at = now
+                asyncio.run_coroutine_threadsafe(
+                    self._render_progress(self.bytes_received, self.expected_size, status=status),
+                    loop,
+                )
+
+            schedule_progress_render(status="downloading", force=True)
 
             with open(self.part_file, mode) as f:
                 for chunk in response.iter_content(chunk_size=self.chunk_size):
@@ -222,6 +253,7 @@ class FileDownloader:
 
                     if self.cancelled:
                         logger.info("Download cancelled")
+                        schedule_progress_render(status="cancelled", force=True)
                         return False
 
                     if chunk:
@@ -229,6 +261,8 @@ class FileDownloader:
                         f.write(chunk)
                         self.bytes_received += len(chunk)
 
+                        schedule_progress_render()
+
                         # Progress callback
                         if self.progress_callback:
                             self.progress_callback(self.bytes_received, self.expected_size)
@@ -237,6 +271,8 @@ class FileDownloader:
                         if self.bytes_received % (self.chunk_size * 10) == 0:
                             self._save_metadata_blocking()
 
+            schedule_progress_render(status="download complete", force=True)
+
             return True
 
         finally:
@@ -313,3 +349,23 @@ class FileDownloader:
         """Cancel the download"""
         self.cancelled = True
         logger.info(f"Download cancelled: {self.file_path}")
+
+    async def _render_progress(self,
+                               bytes_received: Optional[int] = None,
+                               total_bytes: Optional[int] = None,
+                               status: str = "downloading") -> None:
+        """Render this download's progress on a stable terminal row."""
+        bytes_received = self.bytes_received if bytes_received is None else bytes_received
+        total_bytes = self.expected_size if total_bytes is None else total_bytes
+
+        percent = 100.0 if total_bytes == 0 else min((bytes_received / total_bytes) * 100.0, 100.0)
+
+        line = (
+            f"{self.file_path.as_posix()} | "
+            f"download | "
+            f"{bytes_received} / {total_bytes} bytes | "
+            f"{percent:6.2f}% | "
+            f"{status}"
+        )
+
+        await TerminalProgress.render(f"download:{self.transfer_id}", line)

{materials_commons_cli-2.3.0b9 → materials_commons_cli-2.3.0b10}/materials_commons/cli/server/uploader/file_uploader.py

@@ -1,23 +1,32 @@
 import asyncio
 import hashlib
 import logging
+import sys
 import time
 import uuid
 from dataclasses import replace
 from datetime import datetime, timezone
 from typing import Optional, Callable, Dict, Any
+from materials_commons.cli.terminal_progress import TerminalProgress, format_bytes
 
 import aiofiles
 
-from materials_commons.cli.filedb import FileIndexDB
 from materials_commons.cli.requests import UploadRequest, DBWriteRequest
 
 logger = logging.getLogger(__name__)
+logger.setLevel(logging.WARNING)
 
 
 class FileUploader:
     """Handles upload of a single file via websocket using queue pattern"""
 
+    # These class variables are used to track progress of file uploads. They are shared across all
+    # instances of FileUploader. The lock keeps different writers to the terminal from interfering
+    # with each other.
+    _progress_lock = asyncio.Lock()
+    _progress_rows: Dict[str, int] = {}
+    _progress_row_count = 0
+
     def __init__(
             self,
             ws_send_queue: asyncio.Queue,
@@ -40,6 +49,8 @@ class FileUploader:
         self.bytes_sent = 0
         self.paused = False
         self.cancelled = False
+        self.upload_started_at = time.monotonic()
+        self._last_progress_render_at = 0.0
 
         # These fields are referenced multiple times. They are pulled out to make code more readable.
         self.file_path = upload_request.observation.path
@@ -66,20 +77,25 @@ class FileUploader:
                 print(f"Skipping {self.file_path.as_posix()} (empty file)...")
                 return True
 
+            await self._render_progress(status="starting")
+
             # Step 1: Initialize the transfer
             if not await self._send_transfer_init():
+                await self._render_progress(status="failed on initialization")
                 return False
 
             # Step 2: Wait for acceptance
             if not await self._wait_for_acceptance():
                 if self._already_uploaded:
                     logger.info(f"File {self.file_path} already uploaded")
-                    print(f"Uploaded {self.file_path}...")
+                    await self._render_progress(self.file_size, self.file_size, status="already uploaded")
                     return True
+                await self._render_progress(status="failed on acceptance")
                 return False
 
             # # Step 3: Send file chunks
             if not await self._send_chunks2():
+                await self._render_progress(status="failed on chunk sending")
                 return False
 
             # ### New
@@ -99,19 +115,21 @@ class FileUploader:
 
             # Step 4: Send completion
             if not await self._send_transfer_complete():
+                await self._render_progress(status="failed on completion")
                 return False
 
             # Step 5: Wait for finalization
             if not await self._wait_for_finalization():
+                await self._render_progress(status="failed on finalization")
                 return False
 
             logger.info(f"File upload complete for {self.file_path}")
-            print(f"Uploaded {self.file_path}...")
+            await self._render_progress(self.file_size, self.file_size, status="complete")
             return True
 
         except Exception as e:
             logger.error(f"Error uploading file {self.file_path}: {e}")
-            print(f"Error uploading file {self.file_path}: {e}")
+            await self._render_progress(status=f"error: {e}")
             return False
 
     async def resume(self, resume_from_byte: int, resume_from_chunk: int) -> bool:
@@ -323,6 +341,8 @@ class FileUploader:
 
                     self.bytes_sent = bytes_received
 
+                await self._render_progress(self.bytes_sent, self.file_size, status="uploading")
+
                 # Progress callback outside lock
                 if self.progress_callback:
                     self.progress_callback(self.bytes_sent, self.file_size)
@@ -389,6 +409,8 @@ class FileUploader:
                     self.bytes_sent = ack_msg["payload"]["bytes_received"]
                     sequence += 1
 
+                    await self._render_progress(self.bytes_sent, self.file_size, status="uploading")
+
                     # Progress callback
                     if self.progress_callback:
                         self.progress_callback(self.bytes_sent, self.file_size)
@@ -473,6 +495,35 @@ class FileUploader:
         self.cancelled = True
         logger.info(f"Upload cancelled: {self.file_path.name}")
 
+    async def _render_progress(self,
+                               bytes_sent: Optional[int] = None,
+                               total_bytes: Optional[int] = None,
+                               status: str = "uploading") -> None:
+        """Render this upload's progress on a stable terminal row."""
+
+        bytes_sent = self.bytes_sent if bytes_sent is None else bytes_sent
+        total_bytes = self.file_size if total_bytes is None else total_bytes
+
+        force_render = status != "uploading" or bytes_sent >= total_bytes
+        now = time.monotonic()
+        if not force_render and now - self._last_progress_render_at < 0.25:
+            return
+        self._last_progress_render_at = now
+
+        percent = 100.0 if total_bytes == 0 else min((bytes_sent / total_bytes) * 100.0, 100.0)
+        elapsed = max(time.monotonic() - self.upload_started_at, 0.001)
+        rate = bytes_sent / elapsed
+
+        line = (
+            f"{self.file_path.as_posix()} | "
+            f"{format_bytes(bytes_sent)} / {format_bytes(total_bytes)} | "
+            f"{percent:6.2f}% | "
+            f"{format_bytes(rate)}/s | "
+            f"{status}"
+        )
+
+        await TerminalProgress.render(self.transfer_id, line)
+
     def _calculate_md5(self, chunk_size=8192) -> str:
         """Calculate md5 hash of file"""
         md5_hash = hashlib.md5()

{materials_commons_cli-2.3.0b9 → materials_commons_cli-2.3.0b10}/materials_commons/cli/subcommands/up.py

@@ -22,6 +22,7 @@ from materials_commons.cli.subcommands.server import DEFAULT_WS_URL
 from materials_commons.cli.walk import local_listdir
 
 logger = logging.getLogger(__name__)
+logger.setLevel(logging.WARNING)
 
 
 def make_parser():

materials_commons_cli-2.3.0b10/materials_commons/cli/terminal_progress.py

@@ -0,0 +1,45 @@
+import asyncio
+import sys
+from typing import Dict
+
+def format_bytes(value: float) -> str:
+    """Format bytes as a human-readable value."""
+    units = ("B", "KB", "MB", "GB", "TB")
+    size = float(value)
+
+    for unit in units:
+        if size < 1024.0 or unit == units[-1]:
+            return f"{size:.1f} {unit}"
+        size /= 1024.0
+
+    return f"{size:.1f} TB"
+
+class TerminalProgress:
+    _lock = asyncio.Lock()
+    _rows: Dict[str, int] = {}
+    _row_count = 0
+
+    @classmethod
+    async def render(cls, progress_id: str, line: str):
+        if not sys.stdout.isatty():
+            return
+
+        async with cls._lock:
+            if progress_id not in cls._rows:
+                cls._rows[progress_id] = cls._row_count
+                cls._row_count += 1
+                print()
+
+            row = cls._rows[progress_id]
+            rows_from_cursor = cls._row_count - row
+
+            terminal_width = 120
+            if len(line) > terminal_width:
+                line = line[:terminal_width - 3] + "..."
+
+            sys.stdout.write("\033[s")
+            sys.stdout.write(f"\033[{rows_from_cursor}A")
+            sys.stdout.write("\r\033[K")
+            sys.stdout.write(line)
+            sys.stdout.write("\033[u")
+            sys.stdout.flush()

{materials_commons_cli-2.3.0b9 → materials_commons_cli-2.3.0b10}/materials_commons_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: materials-commons-cli
-Version: 2.3.0b9
+Version: 2.3.0b10
 Summary: Materials Commons CLI
 Home-page: https://materials-commons.github.io/materials-commons-cli/html/index.html
 Author: Materials Commons development team

{materials_commons_cli-2.3.0b9 → materials_commons_cli-2.3.0b10}/materials_commons_cli.egg-info/SOURCES.txt

@@ -15,6 +15,7 @@ materials_commons/cli/reconcile2.py
 materials_commons/cli/reconcile3.py
 materials_commons/cli/requests.py
 materials_commons/cli/run.py
+materials_commons/cli/terminal_progress.py
 materials_commons/cli/walk.py
 materials_commons/cli/old/__init__.py
 materials_commons/cli/old/cloned_project.py

materials_commons_cli-2.3.0b9/materials_commons/cli/__init__.py

@@ -1 +0,0 @@
-__version__ = '2.3.0b9'