materials-commons-cli 2.3.0b11__tar.gz → 2.3.0b12__tar.gz

{materials_commons_cli-2.3.0b11/materials_commons_cli.egg-info → materials_commons_cli-2.3.0b12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: materials_commons_cli
-Version: 2.3.0b11
+Version: 2.3.0b12
 Summary: Materials Commons CLI
 Home-page: https://materials-commons.github.io/materials-commons-cli/html/index.html
 Author: Materials Commons development team
@@ -27,7 +27,6 @@ Requires-Dist: requests~=2.28
 Requires-Dist: sortedcontainers~=2.4
 Requires-Dist: tabulate~=0.9
 Requires-Dist: aiosqlite~=0.22
-Requires-Dist: fastmcp~=3.4
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

materials_commons_cli-2.3.0b12/materials_commons/cli/__init__.py

@@ -0,0 +1 @@
+__version__ = '2.3.0b12'

{materials_commons_cli-2.3.0b11 → materials_commons_cli-2.3.0b12}/materials_commons/cli/parser.py

@@ -1,16 +1,15 @@
 import argparse
 import json
 import logging
+import materials_commons.api as mcapi
 import os
 import sys
 from datetime import datetime, timedelta
 from importlib.metadata import PackageNotFoundError, version as pkg_version
 from io import StringIO
 
-import materials_commons.api as mcapi
-import requests
-
 import materials_commons.cli.old.functions as clifuncs
+import requests
 from materials_commons.cli.old.exceptions import MCCLIException, MissingRemoteException, \
     MultipleRemoteException, NoDefaultRemoteException
 from materials_commons.cli.old.user_config import Config
@@ -18,15 +17,12 @@ from materials_commons.cli.subcommands.clone import clone_subcommand
 from materials_commons.cli.subcommands.config import config_subcommand
 from materials_commons.cli.subcommands.dataset import DatasetSubcommand
 from materials_commons.cli.subcommands.down import down_subcommand
-from materials_commons.cli.subcommands.down2 import down2_subcommand
 from materials_commons.cli.subcommands.expt import ExptSubcommand
 from materials_commons.cli.subcommands.fetch import fetch_subcommand
 # from materials_commons.cli.subcommands.samp import SampSubcommand
 from materials_commons.cli.subcommands.globus import globus_subcommand
 from materials_commons.cli.subcommands.init import init_subcommand
 from materials_commons.cli.subcommands.ls import ls_subcommand
-from materials_commons.cli.subcommands.ls2 import ls2_subcommand
-from materials_commons.cli.subcommands.mcp import mcp_subcommand
 from materials_commons.cli.subcommands.mkdir import mkdir_subcommand
 from materials_commons.cli.subcommands.mv import mv_subcommand
 # from materials_commons.cli.subcommands.proc import ProcSubcommand
@@ -61,7 +57,6 @@ standard_usage = [
     # {'name': 'down2', 'desc': 'Donwload files (experimental)', 'subcommand': down2_subcommand},
     {'name': 'config', 'desc': 'Configure `mc`', 'subcommand': config_subcommand},
     {'name': 'server', 'desc': 'Run MC Server', 'subcommand': server_subcommand},
-    {'name': 'mcp', 'desc': 'Run MCP Server', 'subcommand': mcp_subcommand},
 ]
 standard_interfaces = {d['name']: d for d in standard_usage}
 
@@ -169,7 +164,7 @@ def main(argv=None, working_dir=None):
 
         if args.command in standard_interfaces:
             result = standard_interfaces[args.command]['subcommand'](argv[2:], working_dir)
-            check_package_version()
+            # check_package_version()
             return result
 
         else:

{materials_commons_cli-2.3.0b11 → materials_commons_cli-2.3.0b12}/materials_commons/cli/server/uploader/file_uploader.py

@@ -209,6 +209,21 @@ class FileUploader:
                 logger.error(f"Transfer rejected: {reason}")
             return False
 
+        elif msg["command"] == "TRANSFER_ALREADY_UPLOADED":
+            payload = msg["payload"]
+            updated_record = replace(self.upload_request.updated_record,
+                                     local_last_seen_ts=int(time.time()),
+                                     remote_checksum=payload.get("file_checksum", ""),
+                                     remote_size=payload.get("file_size", 0),
+                                     remote_file_id=payload.get("file_id", 0),
+                                     remote_ctime_ns=payload.get("file_created_at_ns", 0))
+            db_write_request = DBWriteRequest(project=self.upload_request.project,
+                                              data=updated_record,
+                                              command="single")
+            await self.db_write_queue.put(db_write_request)
+            self._already_uploaded = True
+            return False
+
         logger.error(f"Unexpected response: {msg['command']}")
         return False
 

{materials_commons_cli-2.3.0b11 → materials_commons_cli-2.3.0b12}/materials_commons/cli/server/websocket_server.py

@@ -241,7 +241,7 @@ class WebSocketCommandListener:
 
         headers["MC-Client-ID"] = self.client_uuid
         headers["MC-Client-Hostname"] = socket.gethostname()
-        headers["MC-Connection-Type"] = "cli"
+        headers["MC-Connection-Type"] = "pycli"
 
         try:
             projects = server.list_local_projects()

{materials_commons_cli-2.3.0b11 → materials_commons_cli-2.3.0b12}/materials_commons/cli/subcommands/up.py

@@ -192,6 +192,9 @@ async def ws_upload(args, working_dir):
     ignore_parser = igittigitt.IgnoreParser()
     ignore_parser.parse_rule_files(base_dir=proj.local_path, filename=".mcignore", add_default_patterns=False)
 
+    def ignore(path, _ignore):
+        return ignore_parser.match(path)
+
     # Start services
     container = ServiceContainer.create(ws_url=args.ws_url)
     service_runtime = ServiceRuntime(container)
@@ -204,7 +207,7 @@ async def ws_upload(args, working_dir):
             p = Path(path)
             if p.is_dir():
                 async for current_path, path_entries in async_reconciler.walk(path=path, listdir_fn=local_listdir,
-                                                                              recursive=args.recursive, ignore_fn=None):
+                                                                              recursive=args.recursive, ignore_fn=ignore):
                     for entry_name in sorted(path_entries):
                         file_state = path_entries[entry_name]
                         if file_state.exception:
@@ -218,6 +221,8 @@ async def ws_upload(args, working_dir):
                             transfer_id = await container.file_upload_manager.upload_file(upload_request)
                             transfer_ids.append(transfer_id)
             elif p.is_file():
+                if ignore(p, True):
+                    continue
                 file_state = await async_reconciler.reconcile_file(p)
                 if file_state.exception:
                     logger.error(f"Error encountered while processing {p}: {file_state.exception}")

{materials_commons_cli-2.3.0b11 → materials_commons_cli-2.3.0b12/materials_commons_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: materials_commons_cli
-Version: 2.3.0b11
+Version: 2.3.0b12
 Summary: Materials Commons CLI
 Home-page: https://materials-commons.github.io/materials-commons-cli/html/index.html
 Author: Materials Commons development team
@@ -27,7 +27,6 @@ Requires-Dist: requests~=2.28
 Requires-Dist: sortedcontainers~=2.4
 Requires-Dist: tabulate~=0.9
 Requires-Dist: aiosqlite~=0.22
-Requires-Dist: fastmcp~=3.4
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{materials_commons_cli-2.3.0b11 → materials_commons_cli-2.3.0b12}/materials_commons_cli.egg-info/SOURCES.txt

@@ -69,7 +69,6 @@ materials_commons/cli/subcommands/globus.py
 materials_commons/cli/subcommands/init.py
 materials_commons/cli/subcommands/ls.py
 materials_commons/cli/subcommands/ls2.py
-materials_commons/cli/subcommands/mcp.py
 materials_commons/cli/subcommands/mkdir.py
 materials_commons/cli/subcommands/mv.py
 materials_commons/cli/subcommands/proj.py

{materials_commons_cli-2.3.0b11 → materials_commons_cli-2.3.0b12}/materials_commons_cli.egg-info/requires.txt

@@ -10,4 +10,3 @@ requests~=2.28
 sortedcontainers~=2.4
 tabulate~=0.9
 aiosqlite~=0.22
-fastmcp~=3.4

{materials_commons_cli-2.3.0b11 → materials_commons_cli-2.3.0b12}/setup.py

@@ -71,6 +71,5 @@ setup(
         "sortedcontainers~=2.4",
         "tabulate~=0.9",
         "aiosqlite~=0.22",
-        "fastmcp~=3.4",
     ]
 )

{materials_commons_cli-2.3.0b11 → materials_commons_cli-2.3.0b12}/tests/test_file_uploader.py

@@ -7,11 +7,13 @@ from unittest.mock import AsyncMock
 import pytest
 
 from materials_commons.cli.server.uploader.file_uploader import FileUploader
+from materials_commons.cli.subcommands.up import ws_upload
 
 
 @dataclass(frozen=True)
 class FakeUpdatedRecord:
     local_checksum: str = "local-checksum"
+    local_last_seen_ts: int = 0
     remote_checksum: str = ""
     remote_size: int = 0
     remote_file_id: int = 0
@@ -81,6 +83,276 @@ def drain_queue(queue):
         items.append(queue.get_nowait())
     return items
 
+def test_wait_for_acceptance_already_uploaded_writes_updated_record_to_db_queue(tmp_path):
+    """
+    Intent:
+    Verify TRANSFER_ALREADY_UPLOADED updates the file database queue.
+
+    When the server reports that a file has already been uploaded, the uploader
+    should treat acceptance as complete, mark the upload as already uploaded, and
+    enqueue the updated remote metadata for persistence.
+    """
+
+    async def run_test():
+        uploader, _, db_write_queue = make_uploader(tmp_path, file_bytes=b"hello")
+
+        await uploader.handle_response(
+            {
+                "command": "TRANSFER_ALREADY_UPLOADED",
+                "payload": {
+                    "file_checksum": "remote-checksum",
+                    "file_size": 5,
+                    "file_id": 4321,
+                    "file_created_at_ns": 987654321,
+                },
+            }
+        )
+
+        result = await uploader._wait_for_acceptance()
+
+        assert result is False
+        assert uploader._already_uploaded is True
+        assert uploader.waiting_for_response is None
+
+        db_write_request = await db_write_queue.get()
+        assert db_write_request.project == uploader.upload_request.project
+        assert db_write_request.command == "single"
+        assert db_write_request.data.remote_checksum == "remote-checksum"
+        assert db_write_request.data.remote_size == 5
+        assert db_write_request.data.remote_file_id == 4321
+        assert db_write_request.data.remote_ctime_ns == 987654321
+
+    asyncio.run(run_test())
+
+
+def test_upload_returns_true_and_updates_db_when_server_says_already_uploaded(tmp_path):
+    """
+    Intent:
+    Verify the upload-level TRANSFER_ALREADY_UPLOADED path.
+
+    upload() should return success without sending chunks or TRANSFER_COMPLETE,
+    and it should preserve the database update produced during acceptance.
+    """
+
+    async def run_test():
+        uploader, ws_send_queue, db_write_queue = make_uploader(
+            tmp_path,
+            file_bytes=b"hello",
+        )
+
+        await uploader.handle_response(
+            {
+                "command": "TRANSFER_ALREADY_UPLOADED",
+                "payload": {
+                    "file_checksum": "remote-checksum",
+                    "file_size": 5,
+                    "file_id": 2468,
+                    "file_created_at_ns": 13579,
+                },
+            }
+        )
+
+        result = await uploader.upload()
+
+        assert result is True
+
+        sent_messages = drain_queue(ws_send_queue)
+        assert len(sent_messages) == 1
+        assert sent_messages[0]["command"] == "TRANSFER_INIT"
+
+        db_write_request = await db_write_queue.get()
+        assert db_write_request.command == "single"
+        assert db_write_request.data.remote_checksum == "remote-checksum"
+        assert db_write_request.data.remote_size == 5
+        assert db_write_request.data.remote_file_id == 2468
+        assert db_write_request.data.remote_ctime_ns == 13579
+
+    asyncio.run(run_test())
+
+
+def test_ws_upload_single_file_in_mcignore_is_not_uploaded_or_reconciled(tmp_path, monkeypatch):
+    """
+    Intent:
+    Verify websocket single-file upload honors .mcignore.
+
+    If the requested file matches .mcignore, ws_upload() should skip it before
+    reconciliation and should not enqueue an upload.
+    """
+
+    async def run_test():
+        project_root = tmp_path
+        ignored_file = project_root / "ignored.txt"
+        ignored_file.write_text("do not upload")
+        (project_root / ".mcignore").write_text("ignored.txt\n")
+
+        db = SimpleNamespace(close=AsyncMock())
+        proj = SimpleNamespace(local_path=project_root, get_filedb=AsyncMock(return_value=db))
+
+        class FakeLocalProject:
+            @staticmethod
+            def load(_working_dir):
+                return proj
+
+        class FakeServiceRuntime:
+            def __init__(self, _container):
+                pass
+
+            async def start(self, websocket_listener=True):
+                return None
+
+            async def stop(self, drain=True):
+                return None
+
+        upload_file = AsyncMock()
+
+        fake_container = SimpleNamespace(
+            file_upload_manager=SimpleNamespace(
+                upload_file=upload_file,
+                wait_all=AsyncMock(),
+            )
+        )
+
+        class FakeServiceContainer:
+            @staticmethod
+            def create(ws_url):
+                return fake_container
+
+        reconcile_file = AsyncMock()
+
+        class FakeAsyncReconciler:
+            def __init__(self, db, proj, reconcile_mode):
+                pass
+
+            async def reconcile_file(self, path):
+                return await reconcile_file(path)
+
+        monkeypatch.setattr("materials_commons.cli.subcommands.up.LocalProject", FakeLocalProject)
+        monkeypatch.setattr("materials_commons.cli.subcommands.up.ServiceRuntime", FakeServiceRuntime)
+        monkeypatch.setattr("materials_commons.cli.subcommands.up.ServiceContainer", FakeServiceContainer)
+        monkeypatch.setattr("materials_commons.cli.subcommands.up.AsyncReconciler", FakeAsyncReconciler)
+
+        args = SimpleNamespace(
+            paths=[ignored_file.as_posix()],
+            recursive=False,
+            ws_url="ws://example.test",
+        )
+
+        await ws_upload(args, project_root)
+
+        reconcile_file.assert_not_awaited()
+        upload_file.assert_not_awaited()
+        fake_container.file_upload_manager.wait_all.assert_awaited_once()
+        db.close.assert_awaited_once()
+
+    asyncio.run(run_test())
+
+
+def test_ws_upload_directory_walk_does_not_upload_files_in_mcignore(tmp_path, monkeypatch):
+    """
+    Intent:
+    Verify websocket directory upload honors .mcignore during walk.
+
+    The directory walk should receive and use the ignore function, causing ignored
+    files to be excluded from upload while non-ignored files are still uploaded.
+    """
+
+    async def run_test():
+        project_root = tmp_path
+        allowed_file = project_root / "allowed.txt"
+        ignored_file = project_root / "ignored.txt"
+        ignored_dir = project_root / "ignored-dir"
+        ignored_dir_file = ignored_dir / "nested.txt"
+
+        allowed_file.write_text("upload me")
+        ignored_file.write_text("do not upload me")
+        ignored_dir.mkdir()
+        ignored_dir_file.write_text("do not walk me")
+        (project_root / ".mcignore").write_text("ignored.txt\nignored-dir/\n")
+
+        db = SimpleNamespace(close=AsyncMock())
+        proj = SimpleNamespace(local_path=project_root, get_filedb=AsyncMock(return_value=db))
+
+        class FakeLocalProject:
+            @staticmethod
+            def load(_working_dir):
+                return proj
+
+        class FakeServiceRuntime:
+            def __init__(self, _container):
+                pass
+
+            async def start(self, websocket_listener=True):
+                return None
+
+            async def stop(self, drain=True):
+                return None
+
+        upload_file = AsyncMock(side_effect=["transfer-allowed"])
+
+        fake_container = SimpleNamespace(
+            file_upload_manager=SimpleNamespace(
+                upload_file=upload_file,
+                wait_all=AsyncMock(),
+            )
+        )
+
+        class FakeServiceContainer:
+            @staticmethod
+            def create(ws_url):
+                return fake_container
+
+        observed_ignore_results = {}
+
+        class FakeAsyncReconciler:
+            def __init__(self, db, proj, reconcile_mode):
+                pass
+
+            async def walk(self, path, listdir_fn, recursive=False, ignore_fn=None):
+                observed_ignore_results["allowed_file"] = ignore_fn(allowed_file, False)
+                observed_ignore_results["ignored_file"] = ignore_fn(ignored_file, False)
+                observed_ignore_results["ignored_dir"] = ignore_fn(ignored_dir, True)
+                observed_ignore_results["ignored_dir_file"] = ignore_fn(ignored_dir_file, False)
+
+                yield Path(path), {
+                    "allowed.txt": SimpleNamespace(
+                        exception=None,
+                        observation=SimpleNamespace(name="allowed.txt"),
+                        file_decision=SimpleNamespace(
+                            action="upload",
+                            updated_record=FakeUpdatedRecord(local_checksum="allowed-checksum"),
+                        ),
+                    )
+                }
+
+        monkeypatch.setattr("materials_commons.cli.subcommands.up.LocalProject", FakeLocalProject)
+        monkeypatch.setattr("materials_commons.cli.subcommands.up.ServiceRuntime", FakeServiceRuntime)
+        monkeypatch.setattr("materials_commons.cli.subcommands.up.ServiceContainer", FakeServiceContainer)
+        monkeypatch.setattr("materials_commons.cli.subcommands.up.AsyncReconciler", FakeAsyncReconciler)
+
+        args = SimpleNamespace(
+            paths=[project_root.as_posix()],
+            recursive=True,
+            ws_url="ws://example.test",
+        )
+
+        await ws_upload(args, project_root)
+
+        assert observed_ignore_results == {
+            "allowed_file": False,
+            "ignored_file": True,
+            "ignored_dir": True,
+            "ignored_dir_file": True,
+        }
+
+        upload_file.assert_awaited_once()
+        upload_request = upload_file.await_args.args[0]
+        assert upload_request.observation.name == "allowed.txt"
+        assert upload_request.updated_record.local_checksum == "allowed-checksum"
+
+        fake_container.file_upload_manager.wait_all.assert_awaited_once()
+        db.close.assert_awaited_once()
+
+    asyncio.run(run_test())
 
 def test_send_transfer_init_puts_expected_message_on_websocket_queue(tmp_path):
     """

materials_commons_cli-2.3.0b11/materials_commons/cli/__init__.py

@@ -1 +0,0 @@
-__version__ = '2.3.0b11'

materials_commons_cli-2.3.0b11/materials_commons/cli/subcommands/mcp.py

@@ -1,86 +0,0 @@
-import argparse
-import functools
-import inspect
-
-from mcp.server.fastmcp import FastMCP
-from materials_commons.cli.config import Config
-from materials_commons.api.client import Client
-
-
-mcp = FastMCP(
-    name="mcmcp",
-    instructions=""
-)
-
-
-def make_parser():
-    parser = argparse.ArgumentParser(
-        description="MCP Server for Materials Commons",
-        usage="mc mcp",
-    )
-    parser.add_argument("--port", type=int, default=8000)
-    parser.add_argument("--mc-url", type=str, default="https://materialscommons.org/api")
-    return parser
-
-def mcp_subcommand(argv, working_dir=None):
-    parser = make_parser()
-    args = parser.parse_args(argv)
-    config = Config.load()
-    client = config.default_remote.make_client()
-    register_mcapi_tools(client)
-    mcp.run()
-    return args
-
-def register_mcapi_tools(client: Client):
-    for method_name, method in inspect.getmembers(client, predicate=callable):
-        if method_name.startswith("_"):
-            continue
-        if not is_get_or_list_method(method_name):
-            continue
-
-        doc = inspect.getdoc(method)
-        if doc is None:
-            continue
-
-        signature = inspect.signature(method)
-
-        @functools.wraps(method)
-        def tool_wrapper(*args, __method=method, **kwargs):
-            result = __method(*args, **kwargs)
-            return to_jsonable(result)
-
-        tool_wrapper.__name__ = method_name
-        tool_wrapper.__doc__ = doc
-        tool_wrapper.__signature__ = signature
-
-        mcp.tool(name=method_name)(tool_wrapper)
-
-def is_get_or_list_method(method_name: str):
-    return method_name.startswith("get_") or method_name.startswith("list_")
-
-def to_jsonable(value):
-    if value is None:
-        return None
-
-    if isinstance(value, (str, int, float, bool)):
-        return value
-
-    if isinstance(value, list):
-        return [to_jsonable(item) for item in value]
-
-    if isinstance(value, tuple):
-        return [to_jsonable(item) for item in value]
-
-    if isinstance(value, dict):
-        return {
-            str(key): to_jsonable(item)
-            for key, item in value.items()
-        }
-
-    if hasattr(value, "to_dict"):
-        return to_jsonable(value.to_dict())
-
-    if hasattr(value, "__dict__"):
-        return to_jsonable(value.__dict__)
-
-    return str(value)