algorithm-nexus 0.2.1__py3-none-any.whl

algorithm_nexus/commands/venv_manager.py

@@ -0,0 +1,158 @@
+# Copyright IBM Corp. 2026
+# SPDX-License-Identifier: Apache-2.0
+
+"""Virtual environment management for benchmark validation."""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+try:
+    from rich.console import Console
+except ImportError:
+    print(
+        "Error: CLI dependencies are not installed.\n"
+        "Please install them with: pip install algorithm-nexus[cli]",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+console = Console()
+
+
+def create_temp_venv(system_site_packages: bool = True) -> Path:
+    """Create a temporary virtual environment using uv.
+
+    Args:
+        system_site_packages: Whether to give access to system site packages (default: True)
+
+    Returns:
+        Path to the created virtual environment
+
+    Raises:
+        subprocess.CalledProcessError: If venv creation fails
+        RuntimeError: If uv is not available
+    """
+
+    # Create temporary directory
+    temp_dir = tempfile.mkdtemp(prefix="nexus-validate-")
+    venv_path = Path(temp_dir) / "venv"
+
+    try:
+        console.print(f"  Creating virtual environment with uv: {venv_path}")
+        cmd = ["uv", "venv", str(venv_path)]
+        if system_site_packages:
+            cmd.append("--system-site-packages")
+        subprocess.run(  # noqa: S603
+            cmd,
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+
+        return venv_path
+
+    except subprocess.CalledProcessError as e:
+        # Clean up on failure
+        shutil.rmtree(temp_dir, ignore_errors=True)
+        console.print(f"[red]Error creating venv:[/red] {e.stderr}")
+        raise
+
+
+def install_packages(
+    venv_path: Path,
+    requirements: list[str],
+    verbose: bool = False,
+) -> bool:
+    """Install packages in the virtual environment using uv.
+
+    Args:
+        venv_path: Path to the virtual environment
+        requirements: List of package requirement specifiers
+        verbose: Whether to show installation output
+
+    Returns:
+        True if installation succeeded, False otherwise
+
+    Raises:
+        RuntimeError: If uv is not available
+    """
+    if not requirements:
+        return True
+
+    python_path = venv_path / "bin" / "python"
+
+    try:
+        console.print(f"  Installing packages with uv: {', '.join(requirements)}")
+        subprocess.run(  # noqa: S603
+            # Forcing installing ado-core for extra safety. However,
+            # ado-core is most probably already in the requirements of the benchmark
+            # package.
+            [  # noqa: S607
+                "uv",
+                "pip",
+                "install",
+                "--python",
+                str(python_path),
+                "ado-core",
+                *requirements,
+            ],
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+
+        return True
+
+    except subprocess.CalledProcessError as e:
+        console.print(f"[red]Error installing packages:[/red] {e.stderr}")
+        return False
+
+
+def run_in_venv(
+    venv_path: Path,
+    command: list[str],
+    capture_output: bool = True,
+) -> subprocess.CompletedProcess:
+    """Execute a command in the virtual environment context.
+
+    Args:
+        venv_path: Path to the virtual environment
+        command: Command to execute (without python prefix)
+        capture_output: Whether to capture stdout/stderr
+
+    Returns:
+        CompletedProcess instance with execution results
+    """
+    python_path = venv_path / "bin" / "python"
+    full_command = [str(python_path), *command]
+
+    return subprocess.run(  # noqa: S603
+        full_command,
+        capture_output=capture_output,
+        text=True,
+        check=False,
+    )
+
+
+def cleanup_venv(venv_path: Path) -> None:
+    """Remove the temporary virtual environment.
+
+    Args:
+        venv_path: Path to the virtual environment to remove
+    """
+    try:
+        # Remove the parent temp directory (which contains the venv)
+        temp_dir = venv_path.parent
+        shutil.rmtree(temp_dir, ignore_errors=True)
+    except Exception as e:
+        console.print(
+            f"[yellow]Warning:[/yellow] Failed to clean up venv at {venv_path}: {e}"
+        )
+
+
+# Made with Bob

algorithm_nexus/models.py

@@ -0,0 +1,408 @@
+# Copyright IBM Corp. 2026
+# SPDX-License-Identifier: Apache-2.0
+
+"""Pydantic models for Nexus package YAML validation."""
+
+from __future__ import annotations
+
+import re
+import sys
+from typing import Annotated, Literal
+
+from pydantic import AfterValidator, computed_field
+
+try:
+    from ado.schema.property import Property
+    from ado.schema.property_value import PropertyValue
+    from ado.schema.reference import ExperimentReference
+    from pydantic import BaseModel, ConfigDict, Field
+except ImportError:
+    print(
+        "Error: CLI dependencies are not installed.\n"
+        "Please install them with: pip install algorithm-nexus[cli]",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+
+def validate_hf_model_id(v: str) -> str:
+    """Validate HuggingFace model ID format and constraints.
+    https://huggingface.co/docs/hub/en/security-sso-okta-scim#step-5-assign-users-or-groups
+
+    Rules:
+    - Only alphanumeric characters, dashes, dots, and underscores are accepted
+    - Double dashes (--) are forbidden
+    - Cannot start or end with a dash
+    - Digit-only names are not accepted (must contain at least one letter)
+    - Format: org-name/model-name where both follow the same rules except length
+      - Maximum length is 42 for org-name and 96 for model-name, minimum length is 2 for both
+    """
+    # Combined regex pattern that validates the entire model ID:
+    # - org-name: 2-42 chars, alphanumeric + dashes + dots + underscores, no double dashes, must have letter
+    # - model-name: 2-96 chars, same rules as org-name
+    # - Separated by exactly one slash
+    # Split validation into org and model parts to ensure each has at least one letter
+    pattern = re.compile(
+        r"^"
+        r"(?=(?:[a-zA-Z0-9._-]*[a-zA-Z][a-zA-Z0-9._-]*)/)"  # org must contain at least one letter
+        r"(?!.*--)"  # no double dashes in org
+        r"[a-zA-Z0-9][a-zA-Z0-9._-]{0,40}[a-zA-Z0-9]"  # org-name: 2-42 chars
+        r"/"  # separator
+        r"(?=(?:[a-zA-Z0-9._-]*[a-zA-Z][a-zA-Z0-9._-]*)$)"  # model must contain at least one letter
+        r"(?!.*--)"  # no double dashes in model
+        r"[a-zA-Z0-9][a-zA-Z0-9._-]{0,94}[a-zA-Z0-9]"  # model-name: 2-96 chars
+        r"$"
+    )
+
+    if not pattern.match(v):
+        msg = "Model ID must be in format 'org-name/model-name' where org-name is 2-42 characters and model-name is 2-96 characters. Both must start and end with alphanumeric, contain only alphanumeric, dashes, dots, and underscores, not have double dashes, and contain at least one letter"
+        raise ValueError(msg)
+
+    return v
+
+
+class BenchmarkPackage(BaseModel):
+    """Benchmark package registration in nexus.yaml.
+
+    Registers a benchmark package and the experiments it exposes.
+    Each package must follow the ADO custom experiment format.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    requirement_specifier: Annotated[
+        str,
+        Field(
+            min_length=1,
+            description="Python package requirement target for the benchmark package. May be a Python package name, a URL to a Python package or source repository, or a local path to a Python package within ./packages in the Nexus repository root.",
+        ),
+    ]
+    experiments: Annotated[
+        list[str],
+        Field(
+            min_length=1,
+            description="Experiment identifiers exposed by that benchmark package and made available to models in the Nexus package.",
+        ),
+    ]
+
+
+class NexusPackageInfo(BaseModel):
+    """Package-level configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    name: Annotated[str, Field(min_length=1, description="Python package name")]
+    benchmark_packages: Annotated[
+        list[BenchmarkPackage] | None,
+        Field(
+            description="List of benchmark packages available to models in this package",
+        ),
+    ] = None
+
+
+class VLLMPlugins(BaseModel):
+    """vLLM plugins configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    general: Annotated[
+        str | None,
+        Field(description="General vLLM plugin that loads the model class"),
+    ] = None
+    io_processors: Annotated[
+        list[str] | None,
+        Field(
+            min_length=1,
+            description="List of vLLM IO processor plugins supported by this model",
+        ),
+    ] = None
+
+
+class VLLMConfig(BaseModel):
+    """vLLM serving configuration.
+
+    Should only be defined for models that require additional vLLM plugins
+    and belong to a Nexus Package targeting the product or candidate distribution variants.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    enabled: Annotated[
+        Literal[True],
+        Field(description="Whether vLLM serving is enabled for this model"),
+    ]
+    plugins: Annotated[
+        VLLMPlugins | None,
+        Field(description="vLLM plugins configuration"),
+    ] = None
+
+
+class ModelInfo(BaseModel):
+    """Model-level configuration."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    id: Annotated[
+        str,
+        Field(
+            min_length=5,
+            max_length=139,
+            description="Hugging Face model repository identifier",
+        ),
+        AfterValidator(validate_hf_model_id),
+    ]
+
+    owner: Annotated[
+        str | None,
+        Field(
+            # Validates the owner field against the GitHub username rules:
+            # https://docs.github.com/en/enterprise-cloud@latest/admin/managing-iam/iam-configuration-reference/username-considerations-for-external-authentication
+            # - Only contains dashes and alphanumeric characters
+            # - Does not start or end with a dash
+            # - Does not contain consecutive dashes
+            # - Has a maximum length of 39 characters
+            pattern=r"^[a-zA-Z0-9]([a-zA-Z0-9]|-[a-zA-Z0-9]){0,38}$",
+            description="Model owner GitHub identifier. If omitted, ownership defaults to the Nexus package owner.",
+        ),
+    ] = None
+
+    vllm: Annotated[
+        VLLMConfig | None,
+        Field(
+            description="vLLM serving configuration. Only required for models that need additional vLLM plugins and belong to a Nexus Package targeting the product or candidate distribution variants.",
+        ),
+    ] = None
+
+
+class AlgorithmNexusModelConfig(BaseModel):
+    """Root model.yaml structure."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    model: Annotated[ModelInfo, Field(description="Model configuration")]
+
+
+class AlgorithmNexusPackageConfig(BaseModel):
+    """Root nexus.yaml structure."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    package: Annotated[
+        NexusPackageInfo, Field(description="Package-level configuration")
+    ]
+
+
+class ValidationResult(BaseModel):
+    """Result of validating a benchmark instance."""
+
+    success: Annotated[bool, Field(description="Whether validation succeeded")]
+    instance_path: Annotated[str, Field(description="Path to the benchmark instance")]
+    errors: Annotated[list[str], Field(description="List of validation errors")]
+    warnings: Annotated[list[str], Field(description="List of validation warnings")]
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def status(self) -> str:
+        """Computed status based on success field."""
+        return "success" if self.success else "failed"
+
+
+class BenchmarkExecutionResult(BaseModel):
+    """Result of executing a benchmark instance."""
+
+    instance_path: Annotated[str, Field(description="Path to the benchmark instance")]
+    status: Annotated[
+        Literal["success", "failed", "started", "unknown"],
+        Field(description="Execution status"),
+    ] = "unknown"
+    message: Annotated[
+        str, Field(description="Status message or error description")
+    ] = ""
+    space_id: Annotated[str | None, Field(description="Created discovery space ID")] = (
+        None
+    )
+    operation_id: Annotated[str | None, Field(description="Created operation ID")] = (
+        None
+    )
+    ray_job_id: Annotated[
+        str | None, Field(description="Ray job ID for remote execution")
+    ] = None
+
+
+class ValidationReport(BaseModel):
+    """Aggregated result of validating all benchmark instances."""
+
+    instances: Annotated[
+        list[dict], Field(description="Per-instance validation result dicts")
+    ]
+    successful: Annotated[int, Field(description="Number of instances that passed")]
+    failed: Annotated[int, Field(description="Number of instances that failed")]
+    total: Annotated[
+        int,
+        Field(
+            description="Total number of instances discovered (may exceed successful + failed under --fail-fast)"
+        ),
+    ]
+
+
+# ---------------------------------------------------------------------------
+# Logical Benchmark models
+# ---------------------------------------------------------------------------
+
+
+class FieldMapping(BaseModel):
+    """1-to-1 mapping of a benchmark property to an experiment property."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    benchmark: Annotated[
+        Property,
+        Field(description="Canonical benchmark property."),
+    ]
+    experiment: Annotated[Property, Field(description="Experiment property.")]
+
+
+class CategoricalValueMapping(BaseModel):
+    """Maps a categorical benchmark property value to a set of experiment property predicates."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    categoricalValue: Annotated[
+        PropertyValue,
+        Field(description="The benchmark categorical value being mapped."),
+    ]
+    predicate: Annotated[
+        list[Property],
+        Field(
+            min_length=1,
+            description="Experiment property constraints that identify this categorical value.",
+        ),
+    ]
+
+
+class MetricIdentifier(BaseModel):
+    """A reference to a canonical benchmark metric by its identifier string."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    identifier: Annotated[
+        str,
+        Field(min_length=1, description="Metric identifier."),
+    ]
+
+
+class MetricMapping(BaseModel):
+    """Maps a canonical benchmark metric name to an experiment metric name."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    benchmark: Annotated[
+        MetricIdentifier,
+        Field(description="Canonical benchmark metric identifier."),
+    ]
+
+    experiment: Annotated[
+        MetricIdentifier,
+        Field(description="Experiment metric identifier."),
+    ]
+
+
+class LogicalBenchmarkDefinition(BaseModel):
+    """Logical benchmark definition — the abstract problem description."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    benchmarkIdentifier: Annotated[
+        str,
+        Field(
+            min_length=1, description="Canonical identifier for this logical benchmark."
+        ),
+    ]
+    description: Annotated[
+        str,
+        Field(
+            min_length=1,
+            description="Human-readable description of the abstract problem being evaluated.",
+        ),
+    ]
+    target: Annotated[
+        Property,
+        Field(
+            description="The property that identifies the quantity being benchmarked."
+        ),
+    ]
+    properties: Annotated[
+        list[Property],
+        Field(
+            min_length=1,
+            description="The properties on which this benchmark is evaluated.",
+        ),
+    ]
+    metrics: Annotated[
+        list[str] | None,
+        Field(description="Canonical metric names for this benchmark."),
+    ] = None
+    owner: Annotated[
+        str | None,
+        Field(
+            description="Team or individual responsible for maintaining this definition."
+        ),
+    ] = None
+
+
+class BenchmarkBinding(BaseModel):
+    """Binding that maps an experiment's properties and metrics to a logical benchmark."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    benchmarkIdentifier: Annotated[
+        str,
+        Field(
+            min_length=1,
+            description="The id of the logical benchmark this experiment targets.",
+        ),
+    ]
+    experiment: Annotated[
+        ExperimentReference,
+        Field(description="The ado ExperimentReference object."),
+    ]
+    targetMapping: Annotated[
+        str,
+        Field(
+            min_length=1,
+            description="The experiment property name that carries the benchmark target.",
+        ),
+    ]
+    staticFilters: Annotated[
+        list[PropertyValue] | None,
+        Field(
+            description="Static experiment property filters implicit in this logical benchmark."
+        ),
+    ] = None
+    propertyMapping: Annotated[
+        list[FieldMapping | CategoricalValueMapping] | None,
+        Field(
+            description="Maps experiment internal properties to canonical benchmark properties."
+        ),
+    ] = None
+    metricMapping: Annotated[
+        list[MetricMapping] | None,
+        Field(
+            description="Translates per-experiment metric names to canonical benchmark metric names."
+        ),
+    ] = None
+
+
+class LogicalBenchmarkConfig(BaseModel):
+    """Root model for a logical benchmark YAML file (logicalBenchmark + optional bindings)."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    logicalBenchmark: Annotated[
+        LogicalBenchmarkDefinition,
+        Field(description="The logical benchmark definition."),
+    ]
+    bindings: Annotated[
+        list[BenchmarkBinding] | None,
+        Field(description="Benchmark bindings declared in this file."),
+    ] = None