materials-commons-cli 2.3.0b6__tar.gz → 2.3.0b8__tar.gz

{materials_commons_cli-2.3.0b6 → materials_commons_cli-2.3.0b8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: materials_commons_cli
-Version: 2.3.0b6
+Version: 2.3.0b8
 Summary: Materials Commons CLI
 Home-page: https://materials-commons.github.io/materials-commons-cli/html/index.html
 Author: Materials Commons development team

materials_commons_cli-2.3.0b8/materials_commons/cli/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.3.0b8'

{materials_commons_cli-2.3.0b6 → materials_commons_cli-2.3.0b8}/materials_commons/cli/async_reconciler.py

@@ -17,12 +17,14 @@ class AsyncReconciler:
                  db: FileIndexDB,
                  reconcile_mode: ReconcileMode,
                  # recompute_checksum: bool = True,
-                 max_concurrent: int = 10):
+                 max_concurrent: int = 10,
+                 batch_size: Optional[int] = None):
         self.proj = proj
         self.db = db
         # self.recompute_checksum = recompute_checksum
         self.max_concurrent = max_concurrent
         self.reconcile_mode = reconcile_mode
+        self.batch_size = batch_size or max_concurrent
         self.reconciler = SingleFileReconciler(proj=self.proj, mode=reconcile_mode)
 
     async def walk(self, path: str | Path, listdir_fn: ListDirFunc, recursive: bool = False,
@@ -35,36 +37,39 @@ class AsyncReconciler:
 
         async for current_path, observations in async_walk(path, recursive=recursive, listdir_fn=listdir_fn,
                                                            ignore_fn=ignore_fn):
-            path_states: dict[str, FileState] = {}
             remote_dir = projects.local_to_remote_project_path(Path(self.proj.local_path), Path(current_path))
 
             # Create a map of file_records to name
             file_records = await self.db.get_files_by_dir(remote_dir.as_posix())
             file_records_map = {file_record.name: file_record for file_record in file_records}
 
-            # Go through all the observations and match to the file records
-            for obs in observations:
-                obs.file_record = file_records_map.get(obs.name, None)
-                obs.remote_path = remote_dir / obs.name
-                path_states[obs.name] = FileState(observation=obs)
+            # Handle a batch of records rather than the entire directory
+            for start in range(0, len(observations), self.batch_size):
+                batch_observations = observations[start:start + self.batch_size]
+                path_states: dict[str, FileState] = {}
 
-            results = await asyncio.gather(
-                *[single_entry_reconcile(state.observation) for state in path_states.values()],
-                return_exceptions=True)
+                # Go through all the observations and match to the file records
+                for obs in batch_observations:
+                    obs.file_record = file_records_map.get(obs.name, None)
+                    obs.remote_path = remote_dir / obs.name
+                    path_states[obs.name] = FileState(observation=obs)
 
-            # At this point path_entries contains entries in one of 4 states:
-            # 1. Both remote and local entries exist
-            # 2. Only remote entry exists
-            # 3. Only local entry exists
-            # 4. Lookup failed, and we got an exception
+                results = await asyncio.gather(
+                    *[single_entry_reconcile(state.observation) for state in path_states.values()],
+                    return_exceptions=True)
 
-            for state, result in zip(path_states.values(), results):
-                if isinstance(result, Exception):
-                    state.exception = result
-                else:
-                    state.file_decision = result
+                # At this point path_entries contains entries in one of 4 states:
+                # 1. Both remote and local entries exist
+                # 2. Only remote entry exists
+                # 3. Only local entry exists
+                # 4. Lookup failed, and we got an exception
+                for state, result in zip(path_states.values(), results):
+                    if isinstance(result, Exception):
+                        state.exception = result
+                    else:
+                        state.file_decision = result
 
-            yield current_path, path_states
+                yield current_path, path_states
 
     async def reconcile_file(self, path: str | Path) -> FileState:
         remote_project_path = projects.local_to_remote_project_path(Path(self.proj.local_path), Path(path))

{materials_commons_cli-2.3.0b6 → materials_commons_cli-2.3.0b8}/materials_commons/cli/downloader.py

@@ -1,4 +1,5 @@
 import logging
+import traceback
 from pathlib import Path
 
 from igittigitt import IgnoreParser, igittigitt
@@ -45,13 +46,7 @@ class Downloader:
         try:
             for path in paths:
                 p = Path(path)
-                if p.is_dir():
-                    # Walk and download local dir
-                    await self._download_dir(p, recursive=recursive)
-                elif p.is_file():
-                    # Download local file
-                    await self._download_file(p)
-                else:
+                if not p.exists():
                     # If we are here, then the path entry is a remote path. We look up the path to
                     # determine the type of download to perform.
                     remote_path = self.proj.to_remote_path(p)
@@ -71,9 +66,16 @@ class Downloader:
                     else:
                         # Remote entry is a file
                         await self._download_file(p, force=force)
+                elif p.is_dir():
+                    # Walk and download local dir
+                    await self._download_dir(p, recursive=recursive)
+                elif p.is_file():
+                    # Download local file
+                    await self._download_file(p)
 
         except Exception as e:
             logger.error(f"Unexpected error during download: {e}")
+            traceback.print_exc()
             return False
 
         finally:
@@ -114,3 +116,5 @@ class Downloader:
                     print(f"Downloading {entry_name} to {current_path / entry_name}...")
                     transfer_id = await self.container.file_download_manager.download_file(download_request)
                     self.transfer_ids.append(transfer_id)
+                else:
+                    print(f"Skipping download of {entry_name} because {file_state.file_decision.reason}")

{materials_commons_cli-2.3.0b6 → materials_commons_cli-2.3.0b8}/materials_commons/cli/parser.py

@@ -25,6 +25,7 @@ from materials_commons.cli.subcommands.globus import globus_subcommand
 from materials_commons.cli.subcommands.init import init_subcommand
 from materials_commons.cli.subcommands.ls import ls_subcommand
 from materials_commons.cli.subcommands.ls2 import ls2_subcommand
+from materials_commons.cli.subcommands.mcp import mcp_subcommand
 from materials_commons.cli.subcommands.mkdir import mkdir_subcommand
 from materials_commons.cli.subcommands.mv import mv_subcommand
 # from materials_commons.cli.subcommands.proc import ProcSubcommand
@@ -58,7 +59,8 @@ standard_usage = [
     # {'name': 'ls2', 'desc': 'List directory contents (experimental)', 'subcommand': ls2_subcommand},
     # {'name': 'down2', 'desc': 'Donwload files (experimental)', 'subcommand': down2_subcommand},
     {'name': 'config', 'desc': 'Configure `mc`', 'subcommand': config_subcommand},
-    {'name': 'server', 'desc': 'Run MC Server', 'subcommand': server_subcommand}
+    {'name': 'server', 'desc': 'Run MC Server', 'subcommand': server_subcommand},
+    {'name': 'mcp', 'desc': 'Run MCP Server', 'subcommand': mcp_subcommand},
 ]
 standard_interfaces = {d['name']: d for d in standard_usage}
 

{materials_commons_cli-2.3.0b6 → materials_commons_cli-2.3.0b8}/materials_commons/cli/server/downloader/file_download_manager.py

@@ -25,6 +25,9 @@ class FileDownloadManager:
 
     async def start_workers(self) -> None:
         """Start background workers to process download queue"""
+        if self._workers_running:
+            logger.info("Download workers already running")
+            return
         self._workers_running = True
         self._worker_tasks = [
             asyncio.create_task(self._download_worker(i))

{materials_commons_cli-2.3.0b6 → materials_commons_cli-2.3.0b8}/materials_commons/cli/server/downloader/file_downloader.py

@@ -12,6 +12,7 @@ import requests
 
 from materials_commons.cli.old.functions import checksum
 from materials_commons.cli.requests import DownloadRequest, DBWriteRequest
+from materials_commons.cli.server import projects
 
 logger = logging.getLogger(__name__)
 
@@ -32,7 +33,10 @@ class FileDownloader:
         self.db_write_queue = db_write_queue
         self.download_request = download_request
         self.file_id = download_request.observation.remote_entry.remote_file_id
-        self.file_path = download_request.observation.local_path
+        if download_request.observation.local_path is None:
+            self.file_path = projects.remote_to_local_project_path(download_request.project.local_path, download_request.observation.remote_path)
+        else:
+            self.file_path = download_request.observation.local_path
         self.base_url = download_request.project.remote.base_url
         self.apitoken = download_request.project.remote.apikey
         self.project_id = download_request.project.id

{materials_commons_cli-2.3.0b6 → materials_commons_cli-2.3.0b8}/materials_commons/cli/server/projects.py

@@ -133,13 +133,16 @@ def local_to_remote_project_path(proj_base: Path, full_path: Path) -> Path:
     Returns:
         Path: The remote path corresponding to the local path.
     """
-    full_path = full_path.resolve()
-    # remote_path will be the relative path from proj_base to full_path, i.e., it
-    # won't start with a slash, so we need to add one to get the correct path.
-    if full_path.as_posix() == proj_base.as_posix():
-        return Path("/")
-    remote_path = full_path.relative_to(proj_base)
-    return Path("/" + remote_path.as_posix())
+    try:
+        full_path_resolved = full_path.resolve()
+        # remote_path will be the relative path from proj_base to full_path, i.e., it
+        # won't start with a slash, so we need to add one to get the correct path.
+        if full_path_resolved.as_posix() == proj_base.as_posix():
+            return Path("/")
+        remote_path = full_path_resolved.relative_to(proj_base)
+        return Path("/" + remote_path.as_posix())
+    except ValueError:
+        return full_path
 
 
 def remote_to_local_project_path(proj_base: Path, remote_path: Path) -> Path:

{materials_commons_cli-2.3.0b6 → materials_commons_cli-2.3.0b8}/materials_commons/cli/server/uploader/file_uploader.py

@@ -74,6 +74,7 @@ class FileUploader:
             if not await self._wait_for_acceptance():
                 if self._already_uploaded:
                     logger.info(f"File {self.file_path} already uploaded")
+                    print(f"Uploaded {self.file_path}...")
                     return True
                 return False
 

{materials_commons_cli-2.3.0b6 → materials_commons_cli-2.3.0b8}/materials_commons/cli/subcommands/ls2.py

@@ -90,21 +90,26 @@ async def ls2_subcommand_async(args, working_dir):
     async_reconciler = AsyncReconciler(db=db, proj=proj, reconcile_mode="status")
     which_class = LSAction if args.action else LSEntry
     listdir_fn = make_merged_listdir_func(proj)
-    for path in args.paths:
-        lstable = LSTable()
-        p = Path(path)
-        if p.is_dir():
-            async for current_path, path_entries in async_reconciler.walk(path=path, listdir_fn=listdir_fn,
-                                                                          recursive=False, ignore_fn=None):
-                for entry_name in sorted(path_entries):
-                    entry = path_entries[entry_name]
-                    if entry.exception:
-                        print(f"Error reconciling file: {entry.exception}")
-                        continue
-                    lstable.add_row(which_class.from_file_state(entry))
-                lstable.print_table()
-        elif p.is_file():
-            state = await async_reconciler.reconcile_file(p)
+    try:
+        for path in args.paths:
             lstable = LSTable()
-            lstable.add_row(which_class.from_file_state(state))
-            lstable.print_table()
+            p = Path(path)
+            if p.is_dir():
+                async for current_path, path_entries in async_reconciler.walk(path=path, listdir_fn=listdir_fn,
+                                                                              recursive=False, ignore_fn=None):
+                    for entry_name in sorted(path_entries):
+                        entry = path_entries[entry_name]
+                        if entry.exception:
+                            print(f"Error reconciling file: {entry.exception}")
+                            continue
+                        lstable.add_row(which_class.from_file_state(entry))
+                    lstable.print_table()
+            elif p.is_file():
+                state = await async_reconciler.reconcile_file(p)
+                lstable = LSTable()
+                lstable.add_row(which_class.from_file_state(state))
+                lstable.print_table()
+    except Exception as e:
+        pass
+    finally:
+        await db.close()

materials_commons_cli-2.3.0b8/materials_commons/cli/subcommands/mcp.py

@@ -0,0 +1,86 @@
+import argparse
+import functools
+import inspect
+
+from mcp.server.fastmcp import FastMCP
+from materials_commons.cli.config import Config
+from materials_commons.api.client import Client
+
+
+mcp = FastMCP(
+    name="mcmcp",
+    instructions=""
+)
+
+
+def make_parser():
+    parser = argparse.ArgumentParser(
+        description="MCP Server for Materials Commons",
+        usage="mc mcp",
+    )
+    parser.add_argument("--port", type=int, default=8000)
+    parser.add_argument("--mc-url", type=str, default="https://materialscommons.org/api")
+    return parser
+
+def mcp_subcommand(argv, working_dir=None):
+    parser = make_parser()
+    args = parser.parse_args(argv)
+    config = Config.load()
+    client = config.default_remote.make_client()
+    register_mcapi_tools(client)
+    mcp.run()
+    return args
+
+def register_mcapi_tools(client: Client):
+    for method_name, method in inspect.getmembers(client, predicate=callable):
+        if method_name.startswith("_"):
+            continue
+        if not is_get_or_list_method(method_name):
+            continue
+
+        doc = inspect.getdoc(method)
+        if doc is None:
+            continue
+
+        signature = inspect.signature(method)
+
+        @functools.wraps(method)
+        def tool_wrapper(*args, __method=method, **kwargs):
+            result = __method(*args, **kwargs)
+            return to_jsonable(result)
+
+        tool_wrapper.__name__ = method_name
+        tool_wrapper.__doc__ = doc
+        tool_wrapper.__signature__ = signature
+
+        mcp.tool(name=method_name)(tool_wrapper)
+
+def is_get_or_list_method(method_name: str):
+    return method_name.startswith("get_") or method_name.startswith("list_")
+
+def to_jsonable(value):
+    if value is None:
+        return None
+
+    if isinstance(value, (str, int, float, bool)):
+        return value
+
+    if isinstance(value, list):
+        return [to_jsonable(item) for item in value]
+
+    if isinstance(value, tuple):
+        return [to_jsonable(item) for item in value]
+
+    if isinstance(value, dict):
+        return {
+            str(key): to_jsonable(item)
+            for key, item in value.items()
+        }
+
+    if hasattr(value, "to_dict"):
+        return to_jsonable(value.to_dict())
+
+    if hasattr(value, "__dict__"):
+        return to_jsonable(value.__dict__)
+
+    return str(value)

{materials_commons_cli-2.3.0b6 → materials_commons_cli-2.3.0b8}/materials_commons/cli/subcommands/up.py

@@ -208,6 +208,7 @@ async def ws_upload(args, working_dir):
                         file_state = path_entries[entry_name]
                         if file_state.exception:
                             logger.error(f"Error encountered while processing {entry_name}: {file_state.exception}")
+                            # print(f"Error encountered while processing {entry_name}: {file_state.exception}")
                             continue
                         if file_state.file_decision.action == "upload":
                             upload_request = UploadRequest(observation=file_state.observation,
@@ -219,7 +220,7 @@ async def ws_upload(args, working_dir):
                 file_state = await async_reconciler.reconcile_file(p)
                 if file_state.exception:
                     logger.error(f"Error encountered while processing {p}: {file_state.exception}")
-                    print(f"Error encountered while processing {p}: {file_state.exception}")
+                    # print(f"Error encountered while processing {p}: {file_state.exception}")
                     continue
                 if file_state.file_decision.action == "upload":
                     upload_request = UploadRequest(observation=file_state.observation,

{materials_commons_cli-2.3.0b6 → materials_commons_cli-2.3.0b8}/materials_commons/cli/walk.py

@@ -118,20 +118,23 @@ async def local_listdir(path: str | Path) -> list[Observation]:
     def _scan() -> list[Observation]:
         items: list[Observation] = []
 
-        for entry in root.iterdir():
-            local = path_to_local_file_entry(entry)
-
-            items.append(
-                Observation(
-                    local_path=entry,
-                    remote_path=None,
-                    local_entry=local,
-                    file_record=None,
-                    remote_entry=None,
+        try:
+            for entry in root.iterdir():
+                local = path_to_local_file_entry(entry)
+
+                items.append(
+                    Observation(
+                        local_path=entry,
+                        remote_path=None,
+                        local_entry=local,
+                        file_record=None,
+                        remote_entry=None,
+                    )
                 )
-            )
-
-        return items
+        except:
+            pass
+        finally:
+            return items
 
     return await asyncio.to_thread(_scan)
 
@@ -202,6 +205,7 @@ async def merged_local_remote_listdir(
     Merge local and remote directory listings, prioritizing local entries.
     """
     local_entries = await local_listdir_fn(path)
+
     project_path = projects.local_to_remote_project_path(Path(proj.local_path), path)
     remote_entries = await remote_listdir_fn(project_path)
 

{materials_commons_cli-2.3.0b6 → materials_commons_cli-2.3.0b8}/materials_commons_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: materials-commons-cli
-Version: 2.3.0b6
+Version: 2.3.0b8
 Summary: Materials Commons CLI
 Home-page: https://materials-commons.github.io/materials-commons-cli/html/index.html
 Author: Materials Commons development team

{materials_commons_cli-2.3.0b6 → materials_commons_cli-2.3.0b8}/materials_commons_cli.egg-info/SOURCES.txt

@@ -68,6 +68,7 @@ materials_commons/cli/subcommands/globus.py
 materials_commons/cli/subcommands/init.py
 materials_commons/cli/subcommands/ls.py
 materials_commons/cli/subcommands/ls2.py
+materials_commons/cli/subcommands/mcp.py
 materials_commons/cli/subcommands/mkdir.py
 materials_commons/cli/subcommands/mv.py
 materials_commons/cli/subcommands/proj.py

{materials_commons_cli-2.3.0b6 → materials_commons_cli-2.3.0b8}/materials_commons_cli.egg-info/requires.txt

@@ -1,5 +1,5 @@
+websockets
 aiofiles
-aiosqlite
 globus-cli
 globus-sdk
 igittigitt==2.1.0
@@ -10,4 +10,5 @@ requests
 setuptools
 sortedcontainers
 tabulate
-websockets
+aiosqlite
+fastmcp

{materials_commons_cli-2.3.0b6 → materials_commons_cli-2.3.0b8}/setup.py

@@ -72,5 +72,6 @@ setup(
         "sortedcontainers",
         "tabulate",
         "aiosqlite",
+        "fastmcp",
     ]
 )

materials_commons_cli-2.3.0b6/materials_commons/cli/__init__.py

@@ -1 +0,0 @@
-__version__ = '2.3.0b6'