evalmc 0.0.1__py3-none-any.whl

evalmc/status.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""
+MyoChallenge Status - Show configuration status
+"""
+
+import json
+from pathlib import Path
+
+import click
+
+# Configuration file paths
+CONFIG_DIR = Path.home() / ".myochallenge"
+HOST_URL_FILE = CONFIG_DIR / "host_url"
+AUTH_TOKEN_FILE = CONFIG_DIR / "auth_token.json"
+
+# Ensure config directory exists
+CONFIG_DIR.mkdir(exist_ok=True)
+
+
+@click.command()
+def status():
+    """
+    Show current configuration status.
+    """
+    click.echo(click.style("MyoChallenge CLI Configuration", bold=True))
+    click.echo("=" * 40)
+
+    # Host status
+    if HOST_URL_FILE.exists():
+        try:
+            host_url = HOST_URL_FILE.read_text().strip()
+            click.echo(f"🌐 Host URL: {host_url}")
+        except (OSError, IOError) as e:
+            click.echo(f"❌ Error reading host URL: {e}")
+    else:
+        click.echo(
+            "🌐 Host URL: Not configured (using default: https://api.mc.myosuite.org)"
+        )
+
+    # Auth status
+    if AUTH_TOKEN_FILE.exists():
+        try:
+            with open(AUTH_TOKEN_FILE, "r") as f:
+                auth_data = json.load(f)
+            if auth_data.get("email") == "manual_set":
+                click.echo("🔐 Authentication: Token manually set")
+                click.echo("💡 Use 'python login.py' to set full credentials")
+            else:
+                click.echo(f"🔐 Authenticated as: {auth_data.get('email', 'Unknown')}")
+                click.echo(f"👥 Team: {auth_data.get('team_name', 'Unknown')}")
+                click.echo(f"🆔 User ID: {auth_data.get('user_id', 'Unknown')}")
+        except (OSError, IOError, json.JSONDecodeError) as e:
+            click.echo(f"❌ Error reading auth data: {e}")
+    else:
+        click.echo("🔐 Authentication: Not configured")
+
+    click.echo("\n💡 Use 'python host.py --set-host URL' to configure host")
+    click.echo("💡 Use 'python login.py' to authenticate")
+    click.echo("💡 Use 'python set_token.py TOKEN' to set token directly")
+
+
+if __name__ == "__main__":
+    status()

evalmc/submission.py

@@ -0,0 +1,558 @@
+#!/usr/bin/env python3
+"""
+MyoChallenge Submission Script - ECR-based Docker image submission
+"""
+
+import subprocess
+import sys
+import uuid
+from pathlib import Path
+
+import click
+import docker
+import requests
+from docker.errors import ImageNotFound
+
+from .login import get_default_host, get_stored_auth
+from .push_helpers import (
+    POLICY_PATHS,
+    http_error_detail,
+    image_not_found_help,
+    is_local_api,
+    local_push_remote_warning,
+    policy_missing_help,
+    post_submit_local_remote_note,
+    upload_mode_summary,
+)
+
+# Configuration file paths
+CONFIG_DIR = Path.home() / ".myochallenge"
+HOST_URL_FILE = CONFIG_DIR / "host_url"
+AUTH_TOKEN_FILE = CONFIG_DIR / "auth_token.json"
+
+# Ensure config directory exists
+CONFIG_DIR.mkdir(exist_ok=True)
+
+
+# Configuration
+LOCAL_DOCKER_REGISTRY_URI = "localhost:5000"
+AWS_REGION = "us-east-1"
+ENVIRONMENT = "DEVELOPMENT"  # Change to "DEVELOPMENT" for local testing
+
+ELBOW_AGENT_BUILD_HINT = (
+    "cd mcEvalProd/evaluation/elbowflexion/agent && docker build -t my-elbow-agent:latest ."
+)
+
+
+def _fail(message: str, hint: str | None = None) -> None:
+    click.echo(f"❌ {message}", err=True)
+    if hint:
+        click.echo(f"\n{hint}", err=True)
+    sys.exit(1)
+
+
+def _validate_agent_image(docker_client, image: str) -> None:
+    """Ensure the image ships policy.py or model.onnx under /policy or /app."""
+    container = None
+    try:
+        container = docker_client.containers.create(
+            image, entrypoint="sleep", command=["infinity"]
+        )
+        container.start()
+        for path in POLICY_PATHS:
+            try:
+                container.get_archive(path)
+                click.echo(f"✅ Policy artifact found in image: {path}")
+                return
+            except docker.errors.NotFound:
+                continue
+        _fail(
+            "Agent image failed policy contract check",
+            policy_missing_help(image),
+        )
+    except docker.errors.APIError as e:
+        _fail(f"Could not inspect agent image '{image}': {e}")
+    finally:
+        if container is not None:
+            try:
+                container.stop(timeout=5)
+            except Exception:
+                pass
+            try:
+                container.remove(force=True)
+            except Exception:
+                pass
+
+
+def _run_subprocess(cmd: list[str], step: str) -> None:
+    click.echo(f"📋 {step}: {' '.join(cmd)}")
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    if result.returncode != 0:
+        output = (result.stderr or result.stdout or "").strip()
+        _fail(
+            f"{step} failed (exit code {result.returncode})",
+            output or "No command output captured.",
+        )
+
+
+@click.command()
+@click.argument("IMAGE", nargs=1)
+@click.option(
+    "-p",
+    "--phase",
+    help="challenge phase number (e.g., 1, 2, 3)",
+    required=True,
+    type=int,
+)
+@click.option(
+    "-t",
+    "--task",
+    help="challenge task name (e.g.,tabletennis, soccer)",
+    required=True,
+)
+@click.option(
+    "-e",
+    "--email",
+    help="team email address for authentication",
+    required=False,
+)
+@click.option(
+    "-a",
+    "--auth-token",
+    help="team authentication token (password)",
+    required=False,
+)
+@click.option(
+    "-u",
+    "--url",
+    help="Docker Registry URI where image will be pushed (for dev/test)",
+    required=False,
+    default=LOCAL_DOCKER_REGISTRY_URI,
+)
+@click.option("--public", is_flag=True, help="Make submission public")
+@click.option("--private", is_flag=True, help="Make submission private")
+@click.option(
+    "--api-base",
+    help="API base URL",
+    default="https://api.mc.myosuite.org",
+    show_default=True,
+)
+@click.option(
+    "--skip-policy-check",
+    is_flag=True,
+    help="Skip verifying /policy/policy.py or model.onnx inside the image",
+)
+@click.option(
+    "--yes",
+    "-y",
+    is_flag=True,
+    help="Skip confirmation when remote API uses local Docker tagging (likely to fail eval)",
+)
+def push(image, phase, task, email, auth_token, url, public, private, api_base, skip_policy_check, yes):
+    """
+    Submit a Docker image to the MyoChallenge competition.
+
+    IMAGE should be in the format 'name:tag' (e.g., 'myochallenge-test:latest')
+    """
+    # Validate image format
+    if len(image.split(":")) != 2:
+        click.echo(
+            "\n❌ Error: Please enter the tag name with image.\n\n"
+            "For example: `python submission.py ubuntu:latest -p 1 -t manipulation "
+            "-e team@example.com -a token123`",
+            err=True,
+        )
+        sys.exit(1)
+
+    # Validate public/private flags
+    if public and private:
+        click.echo(
+            "\n❌ Error: Submission can't be public and private.\n"
+            "Please select either --public or --private",
+            err=True,
+        )
+        sys.exit(1)
+
+    # Set default visibility
+    if not public and not private:
+        private = True  # Default to private
+
+    api_base = get_default_host()
+
+    click.echo(f"🚀 Starting submission for {image} to {task} phase {phase}")
+    click.echo(f"🌐 API host: {api_base}")
+    click.echo(f"🔒 Visibility: {'Public' if public else 'Private'}")
+
+    # Initialize Docker client
+    try:
+        docker_client = docker.from_env()
+        click.echo("✅ Connected to Docker daemon")
+    except Exception as e:
+        click.echo(f"❌ Failed to connect to Docker: {e}", err=True)
+        sys.exit(1)
+
+    # Check if image exists
+    try:
+        docker_image = docker_client.images.get(image)
+        image_size = docker_image.attrs.get("VirtualSize", 0)
+        image_size_gb = image_size / (1024**3)
+        click.echo(f"✅ Found image: {image} (Size: {image_size_gb:.2f} GB)")
+    except ImageNotFound:
+        hint = image_not_found_help(
+            image,
+            ELBOW_AGENT_BUILD_HINT if task == "elbowflexion" else None,
+        )
+        _fail(f"Image '{image}' not found locally", hint)
+
+    if not skip_policy_check:
+        click.echo("🔍 Checking agent image policy contract...")
+        _validate_agent_image(docker_client, image)
+
+    # Get authentication credentials
+    if email is None or auth_token is None:
+        stored_auth = get_stored_auth()
+
+        if stored_auth:
+            email = stored_auth["email"]
+            auth_token = stored_auth["auth_token"]
+            click.echo(f"📧 Using stored credentials for: {email}")
+        else:
+            _fail(
+                "No credentials provided and none stored",
+                "Run 'evalmc login' or pass --email and --auth-token.",
+            )
+
+    # Authenticate with the API
+    click.echo("🔐 Authenticating with MyoChallenge API...")
+    try:
+        auth_response = requests.post(
+            f"{api_base}/login",
+            json={"email": email, "auth_token": auth_token},
+            timeout=10,
+        )
+        auth_response.raise_for_status()
+        auth_data = auth_response.json()
+        if not auth_data.get("success"):
+            _fail(
+                f"Authentication failed: {auth_data.get('message', 'Unknown error')}",
+                "Use your CLI submission token (not web password). Run: evalmc login --relogin",
+            )
+
+        user_id = auth_data["user_id"]
+        team_name = auth_data["team_name"]
+        click.echo(
+            f"✅ Authenticated successfully as team: {team_name} (ID: {user_id})"
+        )
+
+    except requests.exceptions.HTTPError as e:
+        detail = http_error_detail(e.response) if e.response is not None else str(e)
+        _fail(f"Authentication request failed: {detail}")
+    except requests.exceptions.RequestException as e:
+        _fail(f"Authentication request failed: {e}")
+
+    # Collect submission metadata (not used in ECR flow)
+    # submission_metadata = {
+    #     "is_public": json.dumps(public),
+    #     "method_name": "",
+    #     "method_description": "",
+    #     "project_url": "",
+    #     "publication_url": "",
+    # }
+
+    # Generate unique tag for submission
+    submission_tag = str(uuid.uuid4())
+    click.echo(f"🏷️  Generated submission tag: {submission_tag}")
+
+    # Get ECR upload information
+    click.echo("🔑 Getting ECR upload information...")
+    try:
+        ecr_response = requests.get(
+            f"{api_base}/ecr/upload-info/{team_name}/{task}/{phase}",
+            timeout=10,
+        )
+        ecr_response.raise_for_status()
+        ecr_data = ecr_response.json()
+
+        upload_type = ecr_data.get("upload_type", "unknown")
+        if upload_type == "error":
+            _fail(
+                "Server could not provide upload instructions",
+                ecr_data.get("error") or str(ecr_data),
+            )
+
+        click.echo(upload_mode_summary(upload_type, api_base, ecr_data.get("repository_uri")))
+
+        if upload_type == "local":
+            # Local storage mode — tag image locally so the eval worker can resolve it
+            local_tag = f"{team_name.lower()}-{task}-p{phase}:latest"
+            if not is_local_api(api_base) and not yes:
+                click.echo(local_push_remote_warning(api_base, local_tag))
+                if not click.confirm("Continue with local Docker tag anyway?", default=False):
+                    _fail(
+                        "Submission cancelled",
+                        "Use evalmc submit-onnx or evalmc submit (zip), or re-run with --yes.",
+                    )
+            elif not is_local_api(api_base):
+                click.echo(local_push_remote_warning(api_base, local_tag))
+
+            click.echo("📦 Tagging image for local eval worker...")
+            try:
+                docker_image.tag(local_tag)
+                click.echo(f"🏷️  Tagged image as {local_tag}")
+            except Exception as e:
+                _fail(f"Failed to tag image as {local_tag}: {e}")
+
+            if is_local_api(api_base):
+                click.echo(
+                    "ℹ️  Start the eval worker on this machine so it can find the tagged image:\n"
+                    "   cd mcEvalProd && python -m app.services.evaluate --watch "
+                    f"--task {task} --phase {phase}"
+                )
+            else:
+                click.echo(
+                    click.style(
+                        "⚠️  Eval worker on the remote server will NOT have this Docker tag.",
+                        fg="yellow",
+                        bold=True,
+                    )
+                )
+
+            image_uri = local_tag
+            click.echo(f"📤 Recording local Docker submission: {image_uri}")
+
+            submit_response = requests.post(
+                f"{api_base}/ecr/submit",
+                json={
+                    "team_name": team_name,
+                    "task": task,
+                    "phase": phase,
+                    "image_uri": image_uri,
+                },
+                timeout=10,
+            )
+            try:
+                submit_response.raise_for_status()
+            except requests.exceptions.HTTPError as e:
+                _fail(
+                    "Failed to record submission",
+                    http_error_detail(e.response) if e.response is not None else str(e),
+                )
+            submit_data = submit_response.json()
+
+            if submit_data.get("status") == "success":
+                submission_id = submit_data.get("submission_id")
+                click.echo("✅ Local submission recorded successfully!")
+                click.echo(f"   📋 Submission ID: {submission_id}")
+                click.echo(f"   🐳 Image URI: {image_uri}")
+                click.echo("   📊 Status: submitted")
+                if not is_local_api(api_base):
+                    click.echo(post_submit_local_remote_note(submission_id, local_tag, api_base))
+            else:
+                _fail(
+                    "Failed to record submission",
+                    submit_data.get("message") or str(submit_data),
+                )
+            return
+
+        elif upload_type == "ecr_manual":
+            click.echo("📋 Manual ECR setup required:")
+            for instruction in ecr_data.get("instructions", []):
+                click.echo(f"   {instruction}")
+
+            # Ask user if they want to proceed with manual ECR push
+            if not click.confirm("Do you want to proceed with manual ECR push?"):
+                click.echo("❌ Submission cancelled")
+                sys.exit(0)
+
+            # For manual ECR, we'll just record the submission
+            image_uri = f"{ecr_data.get('repository_uri')}:{team_name}-{task}-{phase}"
+            click.echo(f"📤 Recording submission for image: {image_uri}")
+
+            # Record the submission using the ECR submit endpoint
+            submit_response = requests.post(
+                f"{api_base}/ecr/submit",
+                json={
+                    "team_name": team_name,
+                    "task": task,
+                    "phase": phase,
+                    "image_uri": image_uri,
+                },
+                timeout=10,
+            )
+            submit_response.raise_for_status()
+            submit_data = submit_response.json()
+
+            if submit_data.get("status") == "success":
+                click.echo("✅ ECR submission recorded successfully!")
+                click.echo(f"   📋 Submission ID: {submit_data.get('submission_id')}")
+                click.echo(f"   🐳 Image URI: {image_uri}")
+                click.echo("   📊 Status: submitted")
+                click.echo(
+                    "\nℹ️  You must docker push the image to ECR before evaluation can run.\n"
+                    "   If the image is not in ECR, status will become 'failed'."
+                )
+            else:
+                _fail(
+                    "Failed to record submission",
+                    submit_data.get("message") or str(submit_data),
+                )
+
+        elif upload_type == "ecr":
+            click.echo("🔐 Using automatic ECR credentials...")
+
+            # Get ECR credentials from the response
+            username = ecr_data.get("username")
+            password = ecr_data.get("password")
+            proxy_endpoint = ecr_data.get("proxy_endpoint")
+            repo_uri = ecr_data.get("repository_uri")
+
+            if not all([username, password, proxy_endpoint, repo_uri]):
+                _fail(
+                    "Missing ECR credentials from server",
+                    f"Server returned: {ecr_data}",
+                )
+
+            import time
+
+            submission_id = str(uuid.uuid4())[
+                :8
+            ]  # Use first 8 chars of UUID for uniqueness
+            timestamp = int(time.time())
+            image_tag = f"{team_name}-{task}-{phase}-{submission_id}-{timestamp}"
+            full_image_uri = f"{repo_uri}:{image_tag}"
+            click.echo(f"🏷️  Tagging image as: {full_image_uri}")
+            click.echo(f"🔑 Unique submission ID: {submission_id}")
+            click.echo(f"⏰ Timestamp: {timestamp}")
+
+            try:
+                import json
+                import shutil
+                import tempfile
+
+                temp_dir = tempfile.mkdtemp()
+                dockerfile_path = f"{temp_dir}/Dockerfile"
+
+                metadata = {
+                    "submission_id": submission_id,
+                    "timestamp": timestamp,
+                    "team_name": team_name,
+                    "task": task,
+                    "phase": phase,
+                    "submission_time": time.strftime("%Y-%m-%d %H:%M:%S"),
+                }
+
+                with open(dockerfile_path, "w") as f:
+                    f.write(f"FROM {image}\n")
+                    f.write(f"LABEL submission_metadata='{json.dumps(metadata)}'\n")
+                    f.write(
+                        f"RUN echo 'Submission: {submission_id} at {timestamp}' > /submission_info.txt\n"
+                    )
+
+                click.echo("🔨 Building unique submission image...")
+                _run_subprocess(
+                    ["docker", "build", "-t", full_image_uri, temp_dir],
+                    "Docker build",
+                )
+                shutil.rmtree(temp_dir)
+
+                click.echo("🔐 Logging into ECR...")
+                _run_subprocess(
+                    [
+                        "docker",
+                        "login",
+                        "-u",
+                        username,
+                        "-p",
+                        password,
+                        proxy_endpoint,
+                    ],
+                    "Docker login to ECR",
+                )
+
+                click.echo("📤 Pushing image to ECR...")
+                _run_subprocess(["docker", "push", full_image_uri], "Docker push to ECR")
+                click.echo("✅ Image pushed to ECR successfully!")
+
+                # Record the submission with the full ECR URI
+                image_uri = full_image_uri
+                click.echo(f"📤 Recording submission for image: {image_uri}")
+
+                # Record the submission using the ECR submit endpoint
+                submit_response = requests.post(
+                    f"{api_base}/ecr/submit",
+                    json={
+                        "team_name": team_name,
+                        "task": task,
+                        "phase": phase,
+                        "image_uri": image_uri,
+                    },
+                    timeout=10,
+                )
+                submit_response.raise_for_status()
+                submit_data = submit_response.json()
+
+                if submit_data.get("status") == "success":
+                    click.echo("✅ ECR submission recorded successfully!")
+                    click.echo(
+                        f"   📋 Submission ID: {submit_data.get('submission_id')}"
+                    )
+                    click.echo(f"   🐳 Image URI: {image_uri}")
+                    click.echo("   📊 Status: submitted")
+                else:
+                    _fail(
+                        "Failed to record submission",
+                        submit_data.get("message") or str(submit_data),
+                    )
+
+            except Exception as e:
+                _fail(f"Failed to push to ECR: {e}")
+
+        else:
+            _fail(
+                f"Unknown server upload mode: {upload_type!r}",
+                f"Server response: {ecr_data}",
+            )
+
+    except requests.exceptions.HTTPError as e:
+        detail = http_error_detail(e.response) if e.response is not None else str(e)
+        _fail(f"Failed to get ECR upload info: {detail}")
+    except requests.exceptions.RequestException as e:
+        _fail(f"Failed to get ECR upload info: {e}")
+
+    # Verify submission was recorded
+    click.echo("📝 Verifying submission record...")
+    try:
+        submissions_response = requests.get(
+            f"{api_base}/submissions/team/{team_name}",
+            headers={"X-User-Email": email, "X-Auth-Token": auth_token},
+            timeout=10,
+        )
+        submissions_response.raise_for_status()
+        submissions_data = submissions_response.json()
+
+        # Find our submission by task and phase
+        latest_submission = None
+        for submission in submissions_data.get("submissions", []):
+            if submission.get("task") == task and submission.get("phase") == phase:
+                latest_submission = submission
+                break
+
+        if latest_submission:
+            click.echo("✅ Submission verified successfully!")
+            click.echo(f"   📋 Submission ID: {latest_submission.get('id')}")
+            click.echo(f"   🐳 ECR URI: {latest_submission.get('ecr_uri')}")
+            click.echo(f"   📅 Created: {latest_submission.get('created_at')}")
+            click.echo(f"   📊 Status: {latest_submission.get('status', 'submitted')}")
+        else:
+            click.echo("⚠️  Submission may not have been recorded properly")
+
+    except requests.exceptions.RequestException as e:
+        click.echo(f"⚠️  Could not verify submission record: {e}")
+
+    click.echo("\n🎉 Submission completed successfully!")
+    click.echo(f"📧 You will receive updates at: {email}")
+    click.echo(f"🔗 View your submissions at: {api_base}/dashboard")
+
+
+if __name__ == "__main__":
+    push()