materials-commons-cli 2.2.1__tar.gz → 2.2.3__tar.gz

{materials_commons_cli-2.2.1 → materials_commons_cli-2.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: materials_commons_cli
-Version: 2.2.1
+Version: 2.2.3
 Summary: Materials Commons CLI
 Home-page: https://materials-commons.github.io/materials-commons-cli/html/index.html
 Author: Materials Commons development team

materials_commons_cli-2.2.3/materials_commons/cli/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.2.3'

materials_commons_cli-2.2.3/materials_commons/cli/filedb.py

@@ -0,0 +1,110 @@
+import hashlib
+import os
+import sqlite3
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable, Optional, Tuple
+
+@dataclass(frozen=True)
+class FileRecord:
+    path: str
+    size: int
+    mtime_ns: int
+    ctime_ns: int
+    last_seen_ts: int
+    checksum: Optional[str] = None
+    status: Optional[str] = None
+    remote_file_id: Optional[int] = None
+    transfer_id: Optional[str] = None
+
+
+class FileIndexDB:
+    def __init__(self, db_path: Path):
+        self.db_path = db_path
+        self._init_db()
+
+    def _init_db(self):
+        with self._connect() as conn:
+            conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS files (
+                    path TEXT PRIMARY KEY,
+                    size INTEGER NOT NULL,
+                    mtime_ns INTEGER NOT NULL,
+                    ctime_ns INTEGER NOT NULL,
+                    last_seen_ts INTEGER NOT NULL,
+                    checksum TEXT,
+                    status TEXT,
+                    remote_file_id INTEGER,
+                    transfer_id TEXT
+                ) STRICT;
+                """
+            )
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_files_status ON files(status);")
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_files_last_seen ON files(last_seen_ts);")
+
+
+    def _connect(self):
+        conn = sqlite3.connect(self.db_path)
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA journal_mode=WAL;")
+        conn.execute("PRAGMA busy_timeout=5000;")
+        conn.execute("PRAGMA synchronous=NORMAL;")
+        conn.execute("PRAGMA cache_size=100000;")
+        conn.execute("PRAGMA temp_store=MEMORY;")
+        conn.execute("PRAGMA locking_mode=EXCLUSIVE;")
+        conn.execute("PRAGMA foreign_keys=ON;")
+        return conn
+
+    def upsert(self, record: FileRecord):
+        with self._connect() as conn:
+            conn.execute(
+                """
+                INSERT INTO files(path, size, mtime_ns, ctime_ns, last_seen_ts, checksum, status, remote_file_id, transfer_id) 
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(path) DO UPDATE SET
+                    size = excluded.size,
+                    mtime_ns = excluded.mtime_ns,
+                    ctime_ns = excluded.ctime_ns,
+                    last_seen_ts = excluded.last_seen_ts,
+                    checksum = COALESCE(excluded.checksum, files.checksum),
+                    status = COALESCE(excluded.status, files.status),
+                    remote_file_id = COALESCE(excluded.remote_file_id, files.remote_file_id),
+                    transfer_id = COALESCE(excluded.transfer_id, files.transfer_id)
+                """,
+                (
+                    record.path,
+                    record.size,
+                    record.mtime_ns,
+                    record.ctime_ns,
+                    record.last_seen_ts,
+                    record.checksum,
+                    record.status,
+                    record.remote_file_id,
+                    record.transfer_id,
+                )
+            )
+
+    def get_file_by_path(self, path: str) -> Optional[FileRecord]:
+        with self._connect() as conn:
+            cursor = conn.execute("SELECT * FROM files WHERE path=?", (path,))
+            row = cursor.fetchone()
+            if row is None:
+                return None
+            return FileRecord(**row)
+
+def walk(root: Path) -> Iterable[Path]:
+    for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
+        base = Path(dirpath)
+        for name in filenames:
+            yield base / name
+
+def build_file_db_from_scan(db: FileIndexDB, root: Path, compute_checksums: bool = True):
+    scan_started = int(time.time())
+    count = 0
+    for p in walk(root):
+        if not p.is_file():
+            continue
+
+

{materials_commons_cli-2.2.1 → materials_commons_cli-2.2.3}/materials_commons/cli/functions.py

@@ -1,4 +1,6 @@
 import datetime
+
+import aiofiles
 import dateutil
 import hashlib
 import json
@@ -153,6 +155,17 @@ def checksum(path, chunk_size=8192):
             md5.update(chunk)
     return md5.hexdigest()
 
+async def checksum_async(path, chunk_size=8192):
+    """Generate MD5 checksum for the file at "path"""
+    md5 = hashlib.md5()
+    async with aiofiles.open(path, 'rb') as f:
+        while True:
+            chunk = await f.read(chunk_size)
+            if not chunk:
+                break
+            md5.update(chunk)
+    return md5.hexdigest()
+
 def random_name(n=3, max_letters=6, sep='-'):
     """Generates a random name for "n" words of max "max_letters" length, joined by "sep" """
     import random

{materials_commons_cli-2.2.1 → materials_commons_cli-2.2.3}/materials_commons/cli/parser.py

@@ -6,7 +6,7 @@ from datetime import datetime, timedelta
 from io import StringIO
 
 import materials_commons.api as mcapi
-import pkg_resources
+from importlib.metadata import PackageNotFoundError, version as pkg_version
 import requests
 
 import materials_commons.cli.functions as clifuncs
@@ -30,6 +30,7 @@ from materials_commons.cli.subcommands.remote import remote_subcommand
 from materials_commons.cli.subcommands.rm import rm_subcommand
 from materials_commons.cli.subcommands.up import up_subcommand
 from materials_commons.cli.subcommands.versions import versions_subcommand
+from materials_commons.cli.subcommands.scan import scan_subcommand
 from materials_commons.cli.user_config import Config
 from materials_commons.cli.subcommands.server import server_subcommand
 
@@ -47,6 +48,7 @@ standard_usage = [
     {'name': 'fetch', 'desc': 'Remote data fetching and configuration', 'subcommand': fetch_subcommand},
     {'name': 'up', 'desc': 'Upload files', 'subcommand': up_subcommand},
     {'name': 'down', 'desc': 'Download files', 'subcommand': down_subcommand},
+    # {'name': 'scan', 'desc': 'Scan local directory for files to upload', 'subcommand': scan_subcommand},
     {'name': 'globus', 'desc': 'Manage Globus uploads and downloads', 'subcommand': globus_subcommand},
     {'name': 'versions', 'desc': 'List file versions', 'subcommand': versions_subcommand},
     # {'name': 'proc', 'desc': 'List processes', 'subcommand': ProcSubcommand()},
@@ -74,8 +76,12 @@ def make_parser():
         description='Materials Commons command line interface',
         usage=usage_help.getvalue())
     parser.add_argument('command', help='Subcommand to run')
-    parser.add_argument('--version', '-v', action='version',
-                        version=pkg_resources.get_distribution('materials-commons-cli').version)
+    package_name = 'materials-commons-cli'
+    try:
+        installed_version = pkg_version(package_name)
+    except PackageNotFoundError:
+        installed_version = "unknown"
+    parser.add_argument('--version', '-v', action='version', version=installed_version)
 
     return parser
 
@@ -113,7 +119,7 @@ def check_package_version():
 
     package_name = 'materials-commons-cli'
     try:
-        current_version = pkg_resources.get_distribution(package_name).version
+        current_version = pkg_version(package_name)
         response = requests.get(f"https://pypi.org/pypi/{package_name}/json")
         latest_version = response.json()["info"]["version"]
 

{materials_commons_cli-2.2.1/materials_commons/cli/desktop → materials_commons_cli-2.2.3/materials_commons/cli/server}/command_handlers.py

@@ -1,6 +1,6 @@
 import asyncio
 from typing import Dict, Any, Callable, Awaitable
-from materials_commons.cli import desktop
+from materials_commons.cli import server
 import os
 import signal
 import logging
@@ -74,7 +74,7 @@ async def handle_list_directory(queue: asyncio.Queue, cmd: Dict[str, Any]) -> No
 async def handle_list_projects(queue: asyncio.Queue, cmd: Dict[str, Any]) -> None:
     print(f"[handler] list_projects -> {cmd}")
     payload = cmd.get("payload") or {}
-    projects = desktop.list_local_projects()
+    projects = server.list_local_projects()
     response_payload = {
         "projects": projects,
         "request_id": payload.get("request_id"),
@@ -121,7 +121,7 @@ async def handle_upload_file(queue: asyncio.Queue, cmd: Dict[str, Any]) -> None:
     # To do this, we get the local path for the project, then join project_path and the local project's path
     # to get the full local path. Note that project_path may start with a /, so we strip it off before joining.
     if not file_path:
-        p = desktop.get_local_project_by_id(project_id)
+        p = server.get_local_project_by_id(project_id)
         if not p:
             logger.error(f"Project {project_id} not found in local projects")
             return
@@ -178,7 +178,7 @@ async def handle_upload_directory(queue: asyncio.Queue, cmd: Dict[str, Any]):
         logger.error("Missing required field mc_project_path in UPLOAD_DIRECTORY command")
         return
 
-    p = desktop.get_local_project_by_id(project_id)
+    p = server.get_local_project_by_id(project_id)
     if not p:
         logger.error(f"Project {project_id} not found in local projects")
         return
@@ -341,7 +341,7 @@ async def handle_download_file(queue: asyncio.Queue, cmd: Dict[str, Any]) -> Non
     if project_path:
         # If we have a project path, then we need to translate it to a
         # local file path
-        p = desktop.get_local_project_by_id(project_id)
+        p = server.get_local_project_by_id(project_id)
         if not p:
             logger.error(f"Project {project_id} not found in local projects")
             return

{materials_commons_cli-2.2.1/materials_commons/cli/desktop → materials_commons_cli-2.2.3/materials_commons/cli/server}/downloader/file_download_manager.py

@@ -1,9 +1,7 @@
 import asyncio
-import os
-from pathlib import Path
 from typing import Dict, Any, Optional, Callable
 
-from materials_commons.cli.desktop.downloader.file_downloader import FileDownloader, logger
+from materials_commons.cli.server.downloader.file_downloader import FileDownloader, logger
 
 
 class FileDownloadManager:

{materials_commons_cli-2.2.1/materials_commons/cli/desktop → materials_commons_cli-2.2.3/materials_commons/cli/server}/downloader/file_downloader.py

@@ -1,4 +1,3 @@
-# materials_commons/cli/desktop/downloader/file_downloader.py
 import asyncio
 import hashlib
 import json
@@ -6,7 +5,8 @@ import logging
 import uuid
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Optional, Callable, Dict, Any
+from typing import Optional, Callable
+
 import requests
 
 logger = logging.getLogger(__name__)

materials_commons_cli-2.2.3/materials_commons/cli/server/indexer/file_index_manager.py

@@ -0,0 +1,84 @@
+import asyncio
+import os
+
+from datetime import datetime
+from materials_commons.cli.filedb import FileIndexDB, FileRecord
+from materials_commons.cli.functions import checksum
+
+
+class FileIndexManager:
+    """Manages multiple concurrent file indexers"""
+
+    def __init__(self, db: FileIndexDB, queue: asyncio.Queue, max_concurrent: int = 3):
+        self.db = db
+        self._index_queue = queue
+        self._workers_running = False
+        self._max_concurrent = max_concurrent
+        self.db_write_queue = asyncio.Queue()
+
+    async def start_workers(self):
+        """Start background workers to process the index queue"""
+        self._workers_running = True
+        workers = [
+            asyncio.create_task(self._index_file_worker())
+            for i in range(self._max_concurrent)
+        ]
+        db_queue_worker = asyncio.create_task(self._db_queue_worker())
+        workers.append(db_queue_worker)
+        return workers
+
+    async def _index_file_worker(self):
+        """Worker that processes index requests from queue"""
+        print("Index worker started")
+        while self._workers_running:
+            try:
+                (file_path, project_path) = await asyncio.wait_for(self._index_queue.get(), timeout=1.0)
+            except asyncio.TimeoutError:
+                continue
+
+            try:
+                await self._index(file_path, project_path)
+            except Exception as e:
+                print(f"Error indexing {file_path}: {e}")
+            finally:
+                self._index_queue.task_done()
+
+    async def _index(self, file_path: str, project_path: str):
+        """Index a file"""
+        print(f"Indexing {file_path} in {project_path}")
+        hash = await asyncio.to_thread(checksum, file_path)
+        finfo = os.stat(file_path)
+        file_record = FileRecord(
+            path = file_path,
+            size = finfo.st_size,
+            mtime_ns = finfo.st_mtime_ns,
+            ctime_ns = finfo.st_ctime_ns,
+            last_seen_ts = int(datetime.now().timestamp()),
+            checksum = hash,
+            status = "indexed",
+        )
+        await self.db_write_queue.put(file_record)
+        print(f"Done indexing {file_path} in {project_path} got Hash: {hash}")
+        pass
+
+    async def _db_queue_worker(self):
+        """Worker that processes db writes from queue"""
+        while self._workers_running:
+            try:
+                db_entry = await asyncio.wait_for(self.db_write_queue.get(), timeout=1.0)
+                print(f"Writing to db: {db_entry}")
+            except asyncio.TimeoutError:
+                continue
+            except asyncio.CancelledError:
+                return
+
+            try:
+                self.db.upsert(db_entry)
+            except Exception as e:
+                pass
+            finally:
+                self.db_write_queue.task_done()
+
+    async def stop_workers(self):
+        """Stop background workers"""
+        self._workers_running = False

{materials_commons_cli-2.2.1/materials_commons/cli/desktop → materials_commons_cli-2.2.3/materials_commons/cli/server}/local_rest_server.py

@@ -5,7 +5,7 @@ import asyncio
 import threading
 from pathlib import Path
 
-from materials_commons.cli import desktop
+from materials_commons.cli import server
 
 
 class LocalRestRequestHandler(BaseHTTPRequestHandler):
@@ -27,7 +27,7 @@ class LocalRestRequestHandler(BaseHTTPRequestHandler):
 
     def _refresh_projects(self):
         try:
-            projects = desktop.list_local_projects()
+            projects = server.list_local_projects()
             project_ids = [str(p["project_id"]) for p in projects]
             payload = {"type": "refresh-projects", "project_ids": project_ids}
 
@@ -54,7 +54,7 @@ class LocalRestRequestHandler(BaseHTTPRequestHandler):
 
 class LocalRestServer:
     """
-    Local REST server for handling requests from the desktop client. Writes to the server.json state file so
+    Local REST server for handling requests from the server client. Writes to the server.json state file so
     the CLI can find the port.
     """
 

{materials_commons_cli-2.2.1/materials_commons/cli/desktop → materials_commons_cli-2.2.3/materials_commons/cli/server}/uploader/api.py

@@ -1,8 +1,8 @@
 import asyncio
 import logging
 from typing import List, Optional
-from materials_commons.cli.desktop.websocket_server import WebSocketCommandListener
-from materials_commons.cli.desktop.command_handlers import register_handlers
+from materials_commons.cli.server.websocket_server import WebSocketCommandListener
+from materials_commons.cli.server.command_handlers import register_handlers
 from materials_commons.cli.user_config import Config
 import ssl
 import websockets

{materials_commons_cli-2.2.1/materials_commons/cli/desktop → materials_commons_cli-2.2.3/materials_commons/cli/server}/uploader/file_transfer_manager.py

@@ -5,7 +5,7 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import Dict, Any, Optional, Callable
 
-from materials_commons.cli.desktop.uploader.file_uploader import FileUploader, logger
+from materials_commons.cli.server.uploader.file_uploader import FileUploader, logger
 
 
 class FileTransferManager:

{materials_commons_cli-2.2.1/materials_commons/cli/desktop → materials_commons_cli-2.2.3/materials_commons/cli/server}/uploader/file_uploader.py

@@ -6,6 +6,8 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import Optional, Callable, Dict, Any
 
+import aiofiles
+
 logger = logging.getLogger(__name__)
 
 
@@ -43,6 +45,7 @@ class FileUploader:
         self.in_flight_chunks = 0
         self.last_acked_chunk = -1
         self._state_lock = asyncio.Lock()
+        self._already_uploaded = False
 
         # Response handling
         self.response_queue: asyncio.Queue = asyncio.Queue()
@@ -59,6 +62,10 @@ class FileUploader:
 
             # Step 2: Wait for acceptance
             if not await self._wait_for_acceptance():
+                if self._already_uploaded:
+                    logger.info(f"File {self.file_path} already uploaded")
+                    print(f"File {self.file_path} already uploaded...")
+                    return True
                 return False
 
             # # Step 3: Send file chunks
@@ -89,10 +96,12 @@ class FileUploader:
                 return False
 
             logger.info(f"File upload complete for {self.file_path}")
+            print(f"Uploaded {self.file_path}...")
             return True
 
         except Exception as e:
             logger.error(f"Error uploading file {self.file_path}: {e}")
+            print(f"Error uploading file {self.file_path}: {e}")
             return False
 
     async def resume(self, resume_from_byte: int, resume_from_chunk: int) -> bool:
@@ -161,7 +170,11 @@ class FileUploader:
 
         elif msg["command"] == "TRANSFER_REJECT":
             reason = msg["payload"].get("reason", "unknown")
-            logger.error(f"Transfer rejected: {reason}")
+            if reason == "file already uploaded":
+                # TODO: This is a hack to prevent uploading the same file twice. We need a better solution.
+                self._already_uploaded = True
+            else:
+                logger.error(f"Transfer rejected: {reason}")
             return False
 
         logger.error(f"Unexpected response: {msg['command']}")
@@ -189,10 +202,10 @@ class FileUploader:
         """Send file chunks using sliding window (don't wait for each ACK)"""
         total_chunks = (self.file_size + self.chunk_size - 1) // self.chunk_size
 
-        with open(self.file_path, 'rb') as f:
+        async with aiofiles.open(self.file_path, 'rb') as f:
             # Single seek to start position (for resume)
             if start_chunk > 0:
-                f.seek(start_chunk * self.chunk_size)
+                await f.seek(start_chunk * self.chunk_size)
 
             async with self._state_lock:
                 self.next_chunk_to_send = start_chunk
@@ -227,7 +240,7 @@ class FileUploader:
                     continue
 
                 # Read the next chunk (outside lock - file I/O can be slow)
-                chunk = f.read(self.chunk_size)
+                chunk = await f.read(self.chunk_size)
                 if not chunk:
                     break
 

{materials_commons_cli-2.2.1/materials_commons/cli/desktop → materials_commons_cli-2.2.3/materials_commons/cli/server}/websocket_server.py

@@ -9,9 +9,9 @@ from typing import Dict, Any, Optional, Callable, Awaitable
 
 import websockets
 from websockets.exceptions import ConnectionClosedError, ConnectionClosedOK, InvalidStatus
-from materials_commons.cli import desktop
-from materials_commons.cli.desktop.uploader.file_transfer_manager import FileTransferManager
-from materials_commons.cli.desktop.downloader.file_download_manager import FileDownloadManager
+from materials_commons.cli import server
+from materials_commons.cli.server.uploader.file_transfer_manager import FileTransferManager
+from materials_commons.cli.server.downloader.file_download_manager import FileDownloadManager
 from materials_commons.cli.user_config import Config
 
 # Type alias for handler functions
@@ -233,7 +233,7 @@ class WebSocketCommandListener:
         headers["MC-Connection-Type"] = "cli"
 
         try:
-            projects = desktop.list_local_projects()
+            projects = server.list_local_projects()
             headers["MC-Client-Projects"] = ",".join([str(p["project_id"]) for p in projects])
         except Exception:
             # Fallback if listing projects fails

materials_commons_cli-2.2.3/materials_commons/cli/subcommands/scan.py

@@ -0,0 +1,83 @@
+import argparse
+import asyncio
+import os
+
+import materials_commons.cli.functions as clifuncs
+import materials_commons.cli.tree_functions as treefuncs
+from pathlib import Path
+import igittigitt
+from materials_commons.cli.filedb import FileIndexDB
+from materials_commons.cli.server.indexer.file_index_manager import FileIndexManager
+
+
+def make_parser():
+    mc_scan_usage = 'mc scan'
+    parser = argparse.ArgumentParser(
+        description='Scan a project and build a database of current files and directories. This speeds uploads by storing state on the file.',
+        usage=mc_scan_usage,
+        prog='mc scan')
+
+    return parser
+
+async def scan_files_async(local_path, mc_path, file_index_queue, ignore_parser):
+    """Scan files and directories in a project asynchronously"""
+
+    loop = asyncio.get_event_loop()
+
+    def walk_and_enqueue():
+
+        for root, dirs, files in os.walk(local_path):
+            current_dir = os.path.basename(root)
+            if current_dir == ".mc":
+                continue
+            for filename in files:
+                if filename == ".DS_Store":
+                    continue
+                if ignore_parser.match(filename):
+                    continue
+                file_path = os.path.join(root, filename)
+                # Calculate relative path and construct mc_path
+                relative = Path(file_path).relative_to(local_path)
+                file_mc_path = str(Path(mc_path) / relative)
+                # print(f"Indexing {file_mc_path}, {file_path}")
+                asyncio.run_coroutine_threadsafe(file_index_queue.put((file_path, file_mc_path)), loop).result()
+
+    await asyncio.to_thread(walk_and_enqueue)
+
+async def scan_subcommand_async(argv, working_dir):
+    """Builds and populates the file index database for project scan; skips ignored directories and files"""
+    parser = make_parser()
+    parser.parse_args(argv)
+
+    proj = clifuncs.make_local_project(working_dir)
+
+    filedb = FileIndexDB(db_path=Path(proj.local_path) / ".mc" / "mc2.sqlite")
+
+    ignore_parser = igittigitt.IgnoreParser()
+    ignore_parser.parse_rule_files(base_dir=proj.local_path, filename=".mcignore",
+                                   add_default_patterns=False)
+
+    file_index_queue = asyncio.Queue()
+    file_index_manager = FileIndexManager(filedb, file_index_queue)
+
+    workers = await file_index_manager.start_workers()
+
+    local_abspaths = treefuncs.clipaths_to_local_abspaths(proj.local_path, [proj.local_path], proj.local_path)
+    local_abspaths = treefuncs.filter_local_abspaths(proj.local_path, local_abspaths, working_dir)
+    mcpaths = treefuncs.clipaths_to_mcpaths(proj.local_path, local_abspaths, working_dir)
+    for local_path, mc_path in zip(local_abspaths, mcpaths):
+        await scan_files_async(local_path, mc_path, file_index_queue, ignore_parser)
+
+    # Wait for all tasks to complete
+    await file_index_queue.join()
+    await file_index_manager.stop_workers()
+    print("called stop_workers")
+    for worker in workers:
+        try:
+            await worker
+        except asyncio.CancelledError:
+            pass
+
+def scan_subcommand(argv, working_dir):
+    """Runs the scan subcommand asynchronously"""
+    asyncio.run(scan_subcommand_async(argv, working_dir))

{materials_commons_cli-2.2.1 → materials_commons_cli-2.2.3}/materials_commons/cli/subcommands/server.py

@@ -4,10 +4,10 @@ import signal
 import uuid
 from typing import Awaitable, Callable, Dict, Any
 
-from materials_commons.cli.desktop.command_handlers import register_handlers
-from materials_commons.cli.desktop.websocket_server import WebSocketCommandListener
+from materials_commons.cli.server.command_handlers import register_handlers
+from materials_commons.cli.server.websocket_server import WebSocketCommandListener
 from materials_commons.cli.user_config import Config
-from materials_commons.cli.desktop.local_rest_server import LocalRestServer
+from materials_commons.cli.server.local_rest_server import LocalRestServer
 
 CommandHandler = Callable[[Dict[str, Any]], Awaitable[None]]
 

{materials_commons_cli-2.2.1 → materials_commons_cli-2.2.3}/materials_commons/cli/subcommands/up.py

@@ -2,13 +2,15 @@ import argparse
 import logging
 import sys
 
+import igittigitt
+
 import materials_commons.cli.exceptions as cliexcept
 import materials_commons.cli.functions as clifuncs
 import materials_commons.cli.globus as cliglobus
 import materials_commons.cli.tree_functions as treefuncs
 from materials_commons.cli.treedb import LocalTree, RemoteTree
 from materials_commons.cli.subcommands.server import DEFAULT_WS_URL
-from materials_commons.cli.desktop.uploader.api import ws_upload_synchronous
+from materials_commons.cli.server.uploader.api import ws_upload_synchronous
 import os
 from pathlib import Path
 
@@ -106,11 +108,14 @@ def up_subcommand(argv, working_dir):
             proj.local_path, args.paths, working_dir)
 
         # filter, skipping .mc, those specified by .mcignore
-        local_abspaths = treefuncs.filter_local_abspaths(
-            proj.local_path, local_abspaths, working_dir)
+        local_abspaths = treefuncs.filter_local_abspaths(proj.local_path, local_abspaths, working_dir)
 
         mcpaths = treefuncs.clipaths_to_mcpaths(proj.local_path, local_abspaths, working_dir)
 
+        ignore_parser = igittigitt.IgnoreParser()
+        ignore_parser.parse_rule_files(base_dir=proj.local_path, filename=".mcignore",
+                                       add_default_patterns=False)
+
         file_paths = []
         project_paths = []
 
@@ -118,7 +123,14 @@ def up_subcommand(argv, working_dir):
             if treefuncs.os.path.isdir(local_path):
                 if args.recursive:
                     for root, dirs, files in os.walk(local_path):
+                        current_dir = os.path.basename(root)
+                        if current_dir == ".mc":
+                            continue
                         for filename in files:
+                            if filename == ".DS_Store":
+                                continue
+                            # if ignore_parser.match(filename):
+                            #     continue
                             file_path = os.path.join(root, filename)
 
                             # Calculate relative path and construct mc_path
@@ -126,6 +138,7 @@ def up_subcommand(argv, working_dir):
                             file_mc_path = str(Path(mc_path) / relative)
                             file_paths.append(file_path)
                             project_paths.append(file_mc_path)
+                            # print(f"Indexing {file_mc_path}, {file_path}")
                 else:
                     print("Skipping directory because --recursive option not specified: " + local_path)
             else:

{materials_commons_cli-2.2.1 → materials_commons_cli-2.2.3}/materials_commons_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: materials-commons-cli
-Version: 2.2.1
+Version: 2.2.3
 Summary: Materials Commons CLI
 Home-page: https://materials-commons.github.io/materials-commons-cli/html/index.html
 Author: Materials Commons development team

{materials_commons_cli-2.2.1 → materials_commons_cli-2.2.3}/materials_commons_cli.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ materials_commons/cli/__init__.py
 materials_commons/cli/cloned_project.py
 materials_commons/cli/exceptions.py
 materials_commons/cli/file_functions.py
+materials_commons/cli/filedb.py
 materials_commons/cli/functions.py
 materials_commons/cli/globus.py
 materials_commons/cli/list_objects.py
@@ -17,18 +18,20 @@ materials_commons/cli/tmp_functions.py
 materials_commons/cli/tree_functions.py
 materials_commons/cli/treedb.py
 materials_commons/cli/user_config.py
-materials_commons/cli/desktop/__init__.py
-materials_commons/cli/desktop/command_handlers.py
-materials_commons/cli/desktop/local_rest_server.py
-materials_commons/cli/desktop/projects.py
-materials_commons/cli/desktop/websocket_server.py
-materials_commons/cli/desktop/downloader/__init__.py
-materials_commons/cli/desktop/downloader/file_download_manager.py
-materials_commons/cli/desktop/downloader/file_downloader.py
-materials_commons/cli/desktop/uploader/__init__.py
-materials_commons/cli/desktop/uploader/api.py
-materials_commons/cli/desktop/uploader/file_transfer_manager.py
-materials_commons/cli/desktop/uploader/file_uploader.py
+materials_commons/cli/server/__init__.py
+materials_commons/cli/server/command_handlers.py
+materials_commons/cli/server/local_rest_server.py
+materials_commons/cli/server/projects.py
+materials_commons/cli/server/websocket_server.py
+materials_commons/cli/server/downloader/__init__.py
+materials_commons/cli/server/downloader/file_download_manager.py
+materials_commons/cli/server/downloader/file_downloader.py
+materials_commons/cli/server/indexer/__init__.py
+materials_commons/cli/server/indexer/file_index_manager.py
+materials_commons/cli/server/uploader/__init__.py
+materials_commons/cli/server/uploader/api.py
+materials_commons/cli/server/uploader/file_transfer_manager.py
+materials_commons/cli/server/uploader/file_uploader.py
 materials_commons/cli/subcommands/__init__.py
 materials_commons/cli/subcommands/clone.py
 materials_commons/cli/subcommands/config.py
@@ -44,6 +47,7 @@ materials_commons/cli/subcommands/mv.py
 materials_commons/cli/subcommands/proj.py
 materials_commons/cli/subcommands/remote.py
 materials_commons/cli/subcommands/rm.py
+materials_commons/cli/subcommands/scan.py
 materials_commons/cli/subcommands/server.py
 materials_commons/cli/subcommands/up.py
 materials_commons/cli/subcommands/versions.py

{materials_commons_cli-2.2.1 → materials_commons_cli-2.2.3}/materials_commons_cli.egg-info/requires.txt

@@ -1,3 +1,5 @@
+websockets
+aiofiles
 globus-cli
 globus-sdk
 igittigitt==2.1.0
@@ -8,4 +10,3 @@ requests
 setuptools
 sortedcontainers
 tabulate
-websockets

{materials_commons_cli-2.2.1 → materials_commons_cli-2.2.3}/setup.py

@@ -21,7 +21,7 @@ setup(
         "": ["*.txt"]
     },
     packages=find_namespace_packages(
-        include=['materials_commons.cli', 'materials_commons.cli.subcommands', 'materials_commons.cli.desktop', 'materials_commons.cli.desktop.uploader', 'materials_commons.cli.desktop.downloader']
+        include=['materials_commons.cli', 'materials_commons.cli.subcommands', 'materials_commons.cli.server', 'materials_commons.cli.server.uploader', 'materials_commons.cli.server.downloader', 'materials_commons.cli.server.indexer']
     ),
     entry_points={
         'console_scripts': ['mc=materials_commons.cli.parser:main']
@@ -50,6 +50,7 @@ setup(
     keywords='materials science mc materials-commons prisms',
     install_requires=[
         "websockets",
+        "aiofiles",
         "globus-cli",
         "globus-sdk",
         "igittigitt==2.1.0",

materials_commons_cli-2.2.1/materials_commons/cli/__init__.py

@@ -1 +0,0 @@
-__version__ = '2.2.1'