evalmc 0.0.1__py3-none-any.whl

evalmc/__main__.py

@@ -0,0 +1,4 @@
+from .main import main
+
+if __name__ == "__main__":
+    main()

evalmc/check_challenge.py

@@ -0,0 +1,124 @@
+import argparse
+
+import requests
+
+
+def check_task_phase(api_url, task, phase):
+    """Check if a specific task-phase exists and get its details."""
+    try:
+        response = requests.get(f"{api_url}/task-phase/{task}/{phase}")
+        if response.status_code == 200:
+            data = response.json()
+            return {
+                "exists": True,
+                "id": data.get("id"),
+                "task": data.get("task"),
+                "phase": data.get("phase"),
+                "num_scores": data.get("num_scores"),
+                "score_names": data.get("score_names"),
+                "display_name": data.get("display_name"),
+                "visible": data.get("visible"),
+                "created_at": data.get("created_at"),
+            }
+        else:
+            return {"exists": False, "error": f"HTTP {response.status_code}"}
+    except requests.exceptions.RequestException as e:
+        return {"exists": False, "error": str(e)}
+
+
+def list_visible_task_phases(api_url):
+    """Fetch visible task-phases from the API."""
+    try:
+        response = requests.get(f"{api_url}/task-phases", params={"visible": True})
+        if response.status_code == 200:
+            return response.json()
+        return []
+    except requests.exceptions.RequestException:
+        return []
+
+
+def check_leaderboard(api_url, task, phase):
+    """Check if there's a leaderboard for a specific task-phase."""
+    try:
+        response = requests.get(f"{api_url}/leaderboard/{task}/{phase}")
+        if response.status_code == 200:
+            data = response.json()
+            return {
+                "exists": True,
+                "submission_count": len(data.get("leaderboard", [])),
+                "leaderboard": data.get("leaderboard", []),
+            }
+        else:
+            return {"exists": False, "error": f"HTTP {response.status_code}"}
+    except requests.exceptions.RequestException as e:
+        return {"exists": False, "error": str(e)}
+
+
+def get_competition_stats(api_url):
+    """Get overall competition statistics."""
+    stats = {}
+
+    try:
+        response = requests.get(f"{api_url}/users")
+        if response.status_code == 200:
+            data = response.json()
+            stats["total_teams"] = data.get("total_count", 0)
+            stats["teams"] = data.get("users", [])
+        else:
+            stats["teams_error"] = f"HTTP {response.status_code}"
+    except requests.exceptions.RequestException as e:
+        stats["teams_error"] = str(e)
+
+    return stats
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Check the active phases and tasks for the MyoChallenge competition"
+    )
+    ap.add_argument(
+        "--api",
+        required=True,
+        help="API endpoint URL (e.g., https://api.mc.myosuite.org)",
+    )
+    args = ap.parse_args()
+
+    print("🏆 MyoChallenge Competition Status Check")
+    print("=" * 60)
+
+    try:
+        info = requests.get(f"{args.api}/challenge-info", timeout=10)
+        if info.status_code == 200:
+            data = info.json()
+            print(f"✅ {data.get('title', 'MyoChallenge')} (year {data.get('year')})")
+        else:
+            print("⚠️ Could not load challenge-info")
+    except requests.exceptions.RequestException as e:
+        print(f"❌ Cannot connect to API server: {e}")
+        return
+
+    print()
+
+    visible = list_visible_task_phases(args.api)
+    print("📋 Visible Task-Phase Configurations:")
+    print("-" * 40)
+
+    if not visible:
+        print("❌ No visible task-phases configured")
+        return
+
+    total_score_metrics = 0
+    for tp in visible:
+        total_score_metrics += tp.get("num_scores", 0)
+        label = tp.get("display_name") or f"{tp['task']} phase {tp['phase']}"
+        print(f"✅ {label}: {tp['num_scores']} score metrics")
+        print(f"    Score Names: {', '.join(tp.get('score_names', []))}")
+        print()
+
+    print(
+        f"📊 Summary: {len(visible)} visible task-phases with {total_score_metrics} total score metrics"
+    )
+
+
+if __name__ == "__main__":
+    main()

evalmc/check_registered.py

@@ -0,0 +1,50 @@
+import argparse
+
+import requests
+
+# Example: python check_registered.py --api https://api.mc.myosuite.org --email foo@bar.com
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Check if a team is registered for the MyoChallenge competition"
+    )
+    ap.add_argument(
+        "--api",
+        required=True,
+        help="API endpoint URL (e.g., https://api.mc.myosuite.org)",
+    )
+    ap.add_argument("--team", required=True, help="Team name to check")
+    args = ap.parse_args()
+
+    try:
+        r = requests.post(f"{args.api}/check", json={"team_name": args.team})
+        r.raise_for_status()
+        result = r.json()
+
+        if result["exists"]:
+            print("\n✅ Team found!")
+            if result.get("id"):
+                print(f"ID: {result['id']}")
+            print(f"Team Name: {result['team_name']}")
+            print(f"Email: {result['email']}")
+            if result.get("registered_at"):
+                print(f"Registered: {result['registered_at']}")
+            print()
+        else:
+            print(f"\n❌ Team '{args.team}' is not registered.\n")
+
+    except requests.exceptions.RequestException as e:
+        if hasattr(e, "response") and e.response is not None:
+            try:
+                error_detail = e.response.json().get("detail", str(e))
+            except ValueError:
+                error_detail = e.response.text or str(e)
+            print(f"\n❌ Check failed: {error_detail}")
+        else:
+            print(f"\n❌ Check failed: {str(e)}")
+        exit(1)
+
+
+if __name__ == "__main__":
+    main()

evalmc/host.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""
+MyoChallenge Host - Configure API host URL
+"""
+
+import sys
+from pathlib import Path
+
+import click
+
+# Configuration file paths
+CONFIG_DIR = Path.home() / ".myochallenge"
+HOST_URL_FILE = CONFIG_DIR / "host_url"
+
+# Ensure config directory exists
+CONFIG_DIR.mkdir(exist_ok=True)
+
+
+@click.command()
+@click.option(
+    "-sh", "--set-host", help="Set the Host URL of the MyoChallenge API Instance."
+)
+def host(set_host):
+    """
+    View and configure the Host URL.
+    """
+    if set_host is not None:
+        # Basic URL validation
+        if set_host.startswith(("http://", "https://")):
+            try:
+                HOST_URL_FILE.write_text(set_host)
+                click.echo(
+                    click.style(
+                        f"{set_host} is set as the host url.",
+                        bold=True,
+                    )
+                )
+            except (OSError, IOError) as e:
+                click.echo(f"Error saving host URL: {e}")
+                sys.exit(1)
+        else:
+            click.echo(
+                click.style(
+                    "Sorry, please enter a valid url.\n"
+                    "Example: https://api.mc.myosuite.org or https://myochallenge.com",
+                    bold=True,
+                )
+            )
+            sys.exit(1)
+    else:
+        if not HOST_URL_FILE.exists():
+            click.echo(
+                click.style(
+                    "You haven't configured a Host URL for the CLI.\n"
+                    "The CLI would be using https://api.mc.myosuite.org as the default url.",
+                    bold=True,
+                )
+            )
+        else:
+            try:
+                data = HOST_URL_FILE.read_text().strip()
+                click.echo(
+                    click.style(
+                        f"{data} is the Host URL of MyoChallenge.",
+                        bold=True,
+                    )
+                )
+            except (OSError, IOError) as e:
+                click.echo(f"Error reading host URL: {e}")
+                sys.exit(1)
+
+
+if __name__ == "__main__":
+    host()  # type: ignore

evalmc/login.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""
+MyoChallenge Login - Interactive authentication and credential storage
+"""
+
+import json
+import sys
+from pathlib import Path
+
+import click
+import requests
+
+# Configuration file paths
+CONFIG_DIR = Path.home() / ".myochallenge"
+HOST_URL_FILE = CONFIG_DIR / "host_url"
+AUTH_TOKEN_FILE = CONFIG_DIR / "auth_token.json"
+
+# Ensure config directory exists
+CONFIG_DIR.mkdir(exist_ok=True)
+
+
+def get_default_host():
+    """Get the default host URL from config file."""
+    if HOST_URL_FILE.exists():
+        try:
+            return HOST_URL_FILE.read_text().strip()
+        except (OSError, IOError):
+            pass
+    return "https://api.mc.myosuite.org"
+
+
+def get_stored_auth():
+    """Get stored authentication credentials."""
+    if AUTH_TOKEN_FILE.exists():
+        try:
+            with open(AUTH_TOKEN_FILE, "r", encoding="utf-8") as f:
+                return json.load(f)
+        except (OSError, IOError, json.JSONDecodeError):
+            pass
+    return None
+
+
+def authenticate_and_save(email, auth_token, host_url):
+    """Authenticate with the API and save credentials."""
+    try:
+        response = requests.post(
+            f"{host_url}/login",
+            json={"email": email, "auth_token": auth_token},
+            timeout=10,
+        )
+        response.raise_for_status()
+        auth_data = response.json()
+
+        if auth_data.get("success"):
+            # Store credentials
+            credentials = {
+                "email": email,
+                "auth_token": auth_token,
+                "team_name": auth_data.get("team_name"),
+                "user_id": auth_data.get("user_id"),
+            }
+
+            try:
+                with open(AUTH_TOKEN_FILE, "w", encoding="utf-8") as f:
+                    json.dump(credentials, f)
+                click.echo(click.style("\nLogged in successfully!", bold=True))
+                click.echo(f"Team: {auth_data.get('team_name')}")
+                click.echo(f"User ID: {auth_data.get('user_id')}")
+                return True
+            except (OSError, IOError) as e:
+                click.echo(f"Error saving credentials: {e}")
+                sys.exit(1)
+        else:
+            error_message = auth_data.get("message", "Unknown error")
+            click.echo(
+                click.style(
+                    f"Login failed: {error_message}",
+                    fg="red",
+                )
+            )
+            return False
+    except requests.exceptions.RequestException as e:
+        click.echo(click.style(f"Connection error: {e}", fg="red"))
+        return False
+
+
+@click.command()
+@click.option(
+    "--relogin", is_flag=True, help="Force re-login even if credentials exist"
+)
+def login(relogin):
+    """
+    Login to MyoChallenge and save credentials.
+
+    By default, if credentials already exist, they will be used.
+    Use --relogin to force entering new credentials.
+    """
+    host_url = get_default_host()
+
+    # Check if credentials already exist and relogin is not forced
+    if not relogin:
+        stored_auth = get_stored_auth()
+        if stored_auth:
+            click.echo("🔐 Found existing credentials. Testing authentication...")
+
+            # Test existing credentials
+            if authenticate_and_save(
+                stored_auth["email"], stored_auth["auth_token"], host_url
+            ):
+                click.echo("✅ Using existing credentials - authentication successful!")
+                return
+            else:
+                click.echo("❌ Existing credentials failed. Please re-login.")
+                # Fall through to prompt for new credentials
+
+    # Prompt for new credentials
+    email = click.prompt("Email", type=str)
+    auth_token = click.prompt("Auth Token (Password)", type=str, hide_input=True)
+
+    authenticate_and_save(email, auth_token, host_url)
+
+
+if __name__ == "__main__":
+    login()

evalmc/main.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+"""
+MyoChallenge CLI - Main entry point for all commands
+"""
+
+import click
+
+from .host import host
+from .login import login
+from .set_token import set_token
+from .status import status
+from .submission import push
+from .submit import submit
+from .submit_onnx import submit_onnx
+
+
+@click.group()
+@click.version_option()
+def main():
+    """
+    MyoChallenge submission CLI.
+
+    Two submission paths are available:
+
+    \b
+    evalmc submit policy.zip -p 1 -t tabletennis   # zip upload (recommended)
+    evalmc submit-onnx model.onnx -p 1 -t elbowflexion  # ONNX upload
+    evalmc push   my-image:tag -p 1 -t tabletennis  # Docker/ECR image push
+    """
+    pass
+
+
+main.add_command(host)
+main.add_command(login)
+main.add_command(set_token)
+main.add_command(push)
+main.add_command(submit)
+main.add_command(submit_onnx)
+main.add_command(status)

evalmc/push_helpers.py

@@ -0,0 +1,136 @@
+"""Helpers for evalmc push — validation and user-facing error explanations."""
+
+from __future__ import annotations
+
+from urllib.parse import urlparse
+
+POLICY_PATHS = (
+    "/policy/policy.py",
+    "/policy/model.onnx",
+    "/app/policy.py",
+    "/app/model.onnx",
+)
+
+
+def is_local_api(api_base: str) -> bool:
+    host = (urlparse(api_base).hostname or "").lower()
+    return host in ("localhost", "127.0.0.1", "0.0.0.0")
+
+
+def http_error_detail(response) -> str:
+    try:
+        body = response.json()
+        if isinstance(body, dict):
+            return body.get("detail") or body.get("message") or body.get("error") or str(body)
+    except Exception:
+        pass
+    text = (response.text or "").strip()
+    return text[:500] if text else f"HTTP {response.status_code}"
+
+
+def image_not_found_help(image: str, build_hint: str | None = None) -> str:
+    lines = [
+        f"Docker image '{image}' is not available on this machine.",
+        "",
+        "The image must exist locally before you can push/submit it.",
+        "",
+        "Try:",
+        f"  docker images | grep {image.split(':')[0]}",
+    ]
+    if build_hint:
+        lines.extend(["", "Build example:", f"  {build_hint}"])
+    else:
+        lines.extend(
+            [
+                "",
+                "Build example (elbow flexion):",
+                "  cd mcEvalProd/evaluation/elbowflexion/agent",
+                "  docker build -t my-elbow-agent:latest .",
+            ]
+        )
+    lines.extend(
+        [
+            "",
+            "For elbow flexion, zip/ONNX submission is usually simpler:",
+            "  evalmc submit-onnx model.onnx -t elbowflexion -p 1",
+        ]
+    )
+    return "\n".join(lines)
+
+
+def upload_mode_summary(upload_type: str, api_base: str, repository_uri: str | None) -> str:
+    if upload_type == "local":
+        if is_local_api(api_base):
+            return (
+                "Server upload mode: local (dev)\n"
+                "  The eval worker on the same machine must have the tagged image in Docker."
+            )
+        return (
+            "Server upload mode: local storage (misconfigured for remote Docker push)\n"
+            "  The API will record a local Docker tag, but the eval worker runs on the\n"
+            "  server — it cannot see images that exist only on your laptop.\n"
+            "  Evaluation will almost certainly fail with status 'failed'."
+        )
+    if upload_type == "ecr_manual":
+        return (
+            f"Server upload mode: manual ECR\n"
+            f"  Repository: {repository_uri or '(unknown)'}\n"
+            "  You must docker push to ECR yourself before evaluation can run."
+        )
+    if upload_type == "ecr":
+        return (
+            f"Server upload mode: ECR (automatic)\n"
+            f"  Repository: {repository_uri or '(unknown)'}\n"
+            "  The CLI will build, push to ECR, and record the submission."
+        )
+    return f"Server upload mode: {upload_type}"
+
+
+def local_push_remote_warning(api_base: str, local_tag: str) -> str:
+    return (
+        f"⚠️  Remote API ({api_base}) is in local-storage mode.\n"
+        f"\n"
+        f"Your image will only be tagged locally as: {local_tag}\n"
+        f"The production eval worker cannot pull or see that tag from your machine.\n"
+        f"\n"
+        f"Expected outcome: submission status 'submitted' → 'failed' (no leaderboard entry).\n"
+        f"\n"
+        f"Recommended alternatives for elbow flexion:\n"
+        f"  evalmc submit-onnx model.onnx -t elbowflexion -p 1\n"
+        f"  evalmc submit policy.zip -t elbowflexion -p 1\n"
+        f"\n"
+        f"For Docker on production, the server needs USE_LOCAL_STORAGE=false and ECR push.\n"
+        f"Contact the organizers if ECR upload is required."
+    )
+
+
+def policy_missing_help(image: str) -> str:
+    paths = ", ".join(POLICY_PATHS)
+    return (
+        f"Agent image '{image}' does not contain a policy at any of:\n"
+        f"  {paths}\n"
+        f"\n"
+        f"Harness-based tasks (e.g. elbowflexion) require policy files inside the image.\n"
+        f"\n"
+        f"Example Dockerfile:\n"
+        f"  WORKDIR /policy\n"
+        f"  COPY policy.py policy_utils.py task_config.json .\n"
+        f"\n"
+        f"Verify locally:\n"
+        f"  docker run --rm {image} ls -la /policy"
+    )
+
+
+def post_submit_local_remote_note(submission_id, local_tag: str, api_base: str) -> str:
+    return (
+        f"\n⚠️  Submission {submission_id} recorded, but evaluation may fail on the server.\n"
+        f"   Image URI stored: {local_tag}\n"
+        f"   API host: {api_base}\n"
+        f"\n"
+        f"Check status:\n"
+        f"  evalmc status\n"
+        f"  curl -s {api_base}/submissions/team/YOUR_TEAM | python3 -m json.tool\n"
+        f"\n"
+        f"If status becomes 'failed', the server could not run your local Docker tag.\n"
+        f"Use evalmc submit-onnx or evalmc submit (zip) instead."
+    )

evalmc/register.py

@@ -0,0 +1,47 @@
+import argparse
+
+import requests
+
+# Example: python register.py --api https://api.mc.myosuite.org --email foo@bar.com --team "Cool Team"
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Register a new team for the MyoChallenge competition"
+    )
+    ap.add_argument(
+        "--api",
+        required=True,
+        help="API endpoint URL (e.g., https://api.mc.myosuite.org)",
+    )
+    ap.add_argument("--email", required=True, help="Team contact email")
+    ap.add_argument("--team", required=True, help="Team name (3-100 characters)")
+    args = ap.parse_args()
+
+    try:
+        r = requests.post(
+            f"{args.api}/register", json={"email": args.email, "team_name": args.team}
+        )
+        r.raise_for_status()
+        result = r.json()
+
+        print("\nRegistration successful! 🎉")
+        print(f"Team Name: {result['team_name']}")
+        print(f"Email: {result['email']}")
+        print("\n⚠️  IMPORTANT: Save your auth token, you'll need it to submit results!")
+        print(f"Auth Token: {result['auth_token']}\n")
+
+    except requests.exceptions.RequestException as e:
+        if hasattr(e, "response") and e.response is not None:
+            try:
+                error_detail = e.response.json().get("detail", str(e))
+            except ValueError:
+                error_detail = e.response.text or str(e)
+            print(f"\n❌ Registration failed: {error_detail}")
+        else:
+            print(f"\n❌ Registration failed: {str(e)}")
+        exit(1)
+
+
+if __name__ == "__main__":
+    main()

evalmc/set_token.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""
+MyoChallenge Set Token - Directly configure authentication token
+"""
+
+import json
+import sys
+from pathlib import Path
+
+import click
+
+# Configuration file paths
+CONFIG_DIR = Path.home() / ".myochallenge"
+AUTH_TOKEN_FILE = CONFIG_DIR / "auth_token.json"
+
+# Ensure config directory exists
+CONFIG_DIR.mkdir(exist_ok=True)
+
+
+@click.command()
+@click.argument("auth_token")
+def set_token(auth_token):
+    """
+    Configure MyoChallenge authentication token directly.
+
+    AUTH_TOKEN should be your team's authentication token (password).
+    """
+    try:
+        # Store the token with minimal information
+        credentials = {
+            "auth_token": auth_token,
+            "email": "manual_set",  # Placeholder for manual token
+            "team_name": "manual_set",  # Placeholder for manual token
+            "user_id": None,
+        }
+
+        with open(AUTH_TOKEN_FILE, "w") as f:
+            json.dump(credentials, f)
+
+        click.echo(
+            click.style(
+                "Success: Authentication token is successfully set.",
+                bold=True,
+                fg="green",
+            )
+        )
+        click.echo("💡 Note: This token will be used for API requests")
+        click.echo("💡 Use 'python status.py' to verify configuration")
+
+    except (OSError, IOError) as e:
+        click.echo(f"❌ Error saving token: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    set_token()  # type: ignore