wafer-cli 0.2.3__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/PKG-INFO +1 -1
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/pyproject.toml +1 -1
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/cli.py +479 -18
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/evaluate.py +760 -268
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/gpu_run.py +5 -1
- wafer_cli-0.2.4/wafer/problems.py +357 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/wevin_cli.py +22 -2
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer_cli.egg-info/PKG-INFO +1 -1
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer_cli.egg-info/SOURCES.txt +1 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/README.md +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/setup.cfg +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/tests/test_analytics.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/tests/test_billing.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/tests/test_cli_coverage.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/tests/test_cli_parity_integration.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/tests/test_config_integration.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/tests/test_file_operations_integration.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/tests/test_isa_cli.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/tests/test_rocprof_compute_integration.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/tests/test_ssh_integration.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/tests/test_wevin_cli.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/tests/test_workflow_integration.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/GUIDE.md +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/__init__.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/analytics.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/api_client.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/auth.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/autotuner.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/billing.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/config.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/corpus.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/global_config.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/inference.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/ncu_analyze.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/nsys_analyze.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/rocprof_compute.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/rocprof_sdk.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/rocprof_systems.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/skills/wafer-guide/SKILL.md +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/targets.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/templates/__init__.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/templates/ask_docs.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/templates/optimize_kernel.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/templates/trace_analyze.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/tracelens.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer/workspaces.py +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer_cli.egg-info/dependency_links.txt +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer_cli.egg-info/entry_points.txt +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer_cli.egg-info/requires.txt +0 -0
- {wafer_cli-0.2.3 → wafer_cli-0.2.4}/wafer_cli.egg-info/top_level.txt +0 -0
|
@@ -30,6 +30,14 @@ import typer
|
|
|
30
30
|
|
|
31
31
|
from .config import WaferConfig, WaferEnvironment
|
|
32
32
|
from .inference import infer_upload_files, resolve_environment
|
|
33
|
+
from .problems import (
|
|
34
|
+
download_problems,
|
|
35
|
+
get_problem_path,
|
|
36
|
+
get_problems_path,
|
|
37
|
+
)
|
|
38
|
+
from .problems import (
|
|
39
|
+
list_problems as list_problems_fn,
|
|
40
|
+
)
|
|
33
41
|
|
|
34
42
|
app = typer.Typer(
|
|
35
43
|
help="GPU development toolkit for LLM coding agents",
|
|
@@ -95,7 +103,7 @@ def main_callback(ctx: typer.Context) -> None:
|
|
|
95
103
|
global _command_outcome
|
|
96
104
|
# Mark as failure if SystemExit with non-zero code, or any other exception
|
|
97
105
|
if exc_type is SystemExit:
|
|
98
|
-
exit_code = exc_value.code if hasattr(exc_value,
|
|
106
|
+
exit_code = exc_value.code if hasattr(exc_value, "code") else 1
|
|
99
107
|
if exit_code != 0 and exit_code is not None:
|
|
100
108
|
_command_outcome = "failure"
|
|
101
109
|
else:
|
|
@@ -200,6 +208,13 @@ kernelbench_app = typer.Typer(
|
|
|
200
208
|
)
|
|
201
209
|
evaluate_app.add_typer(kernelbench_app, name="kernelbench")
|
|
202
210
|
|
|
211
|
+
# Nested subcommand for gpumode format
|
|
212
|
+
gpumode_app = typer.Typer(
|
|
213
|
+
help="Evaluate kernels in GPUMode format (custom_kernel/ref_kernel functions)",
|
|
214
|
+
invoke_without_command=True,
|
|
215
|
+
)
|
|
216
|
+
evaluate_app.add_typer(gpumode_app, name="gpumode")
|
|
217
|
+
|
|
203
218
|
# =============================================================================
|
|
204
219
|
# Dev commands (internal, used by web app proxy)
|
|
205
220
|
# =============================================================================
|
|
@@ -396,6 +411,124 @@ def skill_status() -> None:
|
|
|
396
411
|
typer.echo(f"{tool_name}: Not installed")
|
|
397
412
|
|
|
398
413
|
|
|
414
|
+
# =============================================================================
|
|
415
|
+
# Provider auth management (wafer auth ...)
|
|
416
|
+
# =============================================================================
|
|
417
|
+
|
|
418
|
+
provider_auth_app = typer.Typer(help="Manage API keys for cloud GPU providers")
|
|
419
|
+
app.add_typer(provider_auth_app, name="auth")
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
@provider_auth_app.command("login")
|
|
423
|
+
def provider_auth_login(
|
|
424
|
+
provider: str = typer.Argument(
|
|
425
|
+
...,
|
|
426
|
+
help="Provider name: runpod, digitalocean, or modal",
|
|
427
|
+
),
|
|
428
|
+
api_key: str | None = typer.Option(
|
|
429
|
+
None,
|
|
430
|
+
"--api-key",
|
|
431
|
+
"-k",
|
|
432
|
+
help="API key (if not provided, reads from stdin)",
|
|
433
|
+
),
|
|
434
|
+
) -> None:
|
|
435
|
+
"""Save API key for a cloud GPU provider.
|
|
436
|
+
|
|
437
|
+
Stores the key in ~/.wafer/auth.json. Environment variables
|
|
438
|
+
(e.g., WAFER_RUNPOD_API_KEY) take precedence over stored keys.
|
|
439
|
+
|
|
440
|
+
Examples:
|
|
441
|
+
wafer auth login runpod --api-key rp_xxx
|
|
442
|
+
wafer auth login digitalocean --api-key dop_v1_xxx
|
|
443
|
+
echo $API_KEY | wafer auth login runpod
|
|
444
|
+
"""
|
|
445
|
+
import sys
|
|
446
|
+
|
|
447
|
+
from wafer_core.auth import PROVIDERS, save_api_key
|
|
448
|
+
|
|
449
|
+
# Validate provider
|
|
450
|
+
if provider not in PROVIDERS:
|
|
451
|
+
typer.echo(f"Error: Unknown provider '{provider}'", err=True)
|
|
452
|
+
typer.echo(f"Valid providers: {', '.join(PROVIDERS.keys())}", err=True)
|
|
453
|
+
raise typer.Exit(1)
|
|
454
|
+
|
|
455
|
+
# Get API key from option or stdin
|
|
456
|
+
if api_key is None:
|
|
457
|
+
if sys.stdin.isatty():
|
|
458
|
+
typer.echo(f"Enter API key for {PROVIDERS[provider]['display_name']}:")
|
|
459
|
+
api_key = typer.prompt("API key", hide_input=True)
|
|
460
|
+
else:
|
|
461
|
+
api_key = sys.stdin.read().strip()
|
|
462
|
+
|
|
463
|
+
if not api_key:
|
|
464
|
+
typer.echo("Error: No API key provided", err=True)
|
|
465
|
+
raise typer.Exit(1)
|
|
466
|
+
|
|
467
|
+
# Save the key
|
|
468
|
+
save_api_key(provider, api_key)
|
|
469
|
+
typer.echo(f"API key saved for {PROVIDERS[provider]['display_name']}")
|
|
470
|
+
typer.echo(f"Stored in: ~/.wafer/auth.json")
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
@provider_auth_app.command("logout")
|
|
474
|
+
def provider_auth_logout(
|
|
475
|
+
provider: str = typer.Argument(
|
|
476
|
+
...,
|
|
477
|
+
help="Provider name: runpod, digitalocean, or modal",
|
|
478
|
+
),
|
|
479
|
+
) -> None:
|
|
480
|
+
"""Remove stored API key for a cloud GPU provider.
|
|
481
|
+
|
|
482
|
+
Examples:
|
|
483
|
+
wafer auth logout runpod
|
|
484
|
+
wafer auth logout digitalocean
|
|
485
|
+
"""
|
|
486
|
+
from wafer_core.auth import PROVIDERS, remove_api_key
|
|
487
|
+
|
|
488
|
+
# Validate provider
|
|
489
|
+
if provider not in PROVIDERS:
|
|
490
|
+
typer.echo(f"Error: Unknown provider '{provider}'", err=True)
|
|
491
|
+
typer.echo(f"Valid providers: {', '.join(PROVIDERS.keys())}", err=True)
|
|
492
|
+
raise typer.Exit(1)
|
|
493
|
+
|
|
494
|
+
if remove_api_key(provider):
|
|
495
|
+
typer.echo(f"API key removed for {PROVIDERS[provider]['display_name']}")
|
|
496
|
+
else:
|
|
497
|
+
typer.echo(f"No stored API key found for {PROVIDERS[provider]['display_name']}")
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
@provider_auth_app.command("status")
|
|
501
|
+
def provider_auth_status() -> None:
|
|
502
|
+
"""Show authentication status for all cloud GPU providers.
|
|
503
|
+
|
|
504
|
+
Displays which providers have API keys configured and where
|
|
505
|
+
the keys are coming from (environment variable or auth.json).
|
|
506
|
+
|
|
507
|
+
Example:
|
|
508
|
+
wafer auth status
|
|
509
|
+
"""
|
|
510
|
+
from wafer_core.auth import get_all_auth_status
|
|
511
|
+
|
|
512
|
+
statuses = get_all_auth_status()
|
|
513
|
+
|
|
514
|
+
typer.echo("Cloud GPU Provider Authentication Status")
|
|
515
|
+
typer.echo("=" * 45)
|
|
516
|
+
|
|
517
|
+
for status in statuses:
|
|
518
|
+
if status.is_authenticated:
|
|
519
|
+
source_str = f"({status.source})" if status.source else ""
|
|
520
|
+
typer.echo(
|
|
521
|
+
f" {status.display_name}: ✓ {status.key_preview} {source_str}"
|
|
522
|
+
)
|
|
523
|
+
else:
|
|
524
|
+
typer.echo(f" {status.display_name}: ✗ Not configured")
|
|
525
|
+
typer.echo(f" Run: wafer auth login {status.provider}")
|
|
526
|
+
typer.echo(f" Or set: {status.key_url}")
|
|
527
|
+
|
|
528
|
+
typer.echo("")
|
|
529
|
+
typer.echo("Note: Environment variables take precedence over stored keys.")
|
|
530
|
+
|
|
531
|
+
|
|
399
532
|
@app.command(hidden=True)
|
|
400
533
|
def run(
|
|
401
534
|
command: str = typer.Argument(..., help="Command to run in Docker container"),
|
|
@@ -1289,13 +1422,25 @@ def evaluate( # noqa: PLR0913
|
|
|
1289
1422
|
--benchmark --defensive
|
|
1290
1423
|
|
|
1291
1424
|
Subcommands:
|
|
1292
|
-
|
|
1425
|
+
gpumode Use GPUMode format (functional) - RECOMMENDED
|
|
1293
1426
|
kernelbench Use KernelBench format (ModelNew class)
|
|
1427
|
+
make-template Generate template files for this format (deprecated)
|
|
1294
1428
|
"""
|
|
1295
1429
|
# If a subcommand is being invoked, skip the main evaluation logic
|
|
1296
1430
|
if ctx.invoked_subcommand is not None:
|
|
1297
1431
|
return
|
|
1298
1432
|
|
|
1433
|
+
# Deprecation warning for bare evaluate
|
|
1434
|
+
typer.echo(
|
|
1435
|
+
"⚠️ Deprecation warning: 'wafer evaluate' will be removed in a future version.",
|
|
1436
|
+
err=True,
|
|
1437
|
+
)
|
|
1438
|
+
typer.echo(
|
|
1439
|
+
" Use 'wafer evaluate gpumode' instead for the functional format.",
|
|
1440
|
+
err=True,
|
|
1441
|
+
)
|
|
1442
|
+
typer.echo("", err=True)
|
|
1443
|
+
|
|
1299
1444
|
# Validate required args when running evaluation (not subcommands)
|
|
1300
1445
|
missing_args = []
|
|
1301
1446
|
if implementation is None:
|
|
@@ -1310,12 +1455,12 @@ def evaluate( # noqa: PLR0913
|
|
|
1310
1455
|
typer.echo(f" Required: {', '.join(missing_args)}", err=True)
|
|
1311
1456
|
typer.echo("", err=True)
|
|
1312
1457
|
typer.echo(
|
|
1313
|
-
"Usage: wafer evaluate --impl KERNEL.py --reference REF.py --test-cases TESTS.json",
|
|
1458
|
+
"Usage: wafer evaluate gpumode --impl KERNEL.py --reference REF.py --test-cases TESTS.json",
|
|
1314
1459
|
err=True,
|
|
1315
1460
|
)
|
|
1316
1461
|
typer.echo("", err=True)
|
|
1317
|
-
typer.echo("Run 'wafer evaluate --help' for full options.", err=True)
|
|
1318
|
-
typer.echo("Run 'wafer evaluate
|
|
1462
|
+
typer.echo("Run 'wafer evaluate gpumode --help' for full options.", err=True)
|
|
1463
|
+
typer.echo("Run 'wafer evaluate gpumode download' to download problem sets.", err=True)
|
|
1319
1464
|
raise typer.Exit(1)
|
|
1320
1465
|
|
|
1321
1466
|
from .evaluate import EvaluateArgs, run_evaluate
|
|
@@ -1503,8 +1648,59 @@ def evaluate_make_template(
|
|
|
1503
1648
|
# KernelBench format evaluation
|
|
1504
1649
|
# =============================================================================
|
|
1505
1650
|
|
|
1506
|
-
|
|
1507
|
-
|
|
1651
|
+
|
|
1652
|
+
def _get_kernelbench_root() -> Path | None:
|
|
1653
|
+
"""Get KernelBench problems root, preferring downloaded location."""
|
|
1654
|
+
# First check downloaded location
|
|
1655
|
+
downloaded = get_problems_path("kernelbench")
|
|
1656
|
+
if downloaded is not None:
|
|
1657
|
+
kb_root = downloaded / "KernelBench"
|
|
1658
|
+
if kb_root.exists():
|
|
1659
|
+
return kb_root
|
|
1660
|
+
return downloaded
|
|
1661
|
+
|
|
1662
|
+
# Fall back to legacy location (for development)
|
|
1663
|
+
legacy = Path(__file__).parent.parent.parent.parent / "research" / "KernelBench" / "KernelBench"
|
|
1664
|
+
if legacy.exists():
|
|
1665
|
+
return legacy
|
|
1666
|
+
|
|
1667
|
+
return None
|
|
1668
|
+
|
|
1669
|
+
|
|
1670
|
+
@kernelbench_app.command("download")
|
|
1671
|
+
def kernelbench_download(
|
|
1672
|
+
force: bool = typer.Option(False, "--force", "-f", help="Re-download even if exists"),
|
|
1673
|
+
) -> None:
|
|
1674
|
+
"""Download KernelBench problems from GitHub.
|
|
1675
|
+
|
|
1676
|
+
Downloads the problem set to ~/.cache/wafer/problems/kernelbench/
|
|
1677
|
+
|
|
1678
|
+
Examples:
|
|
1679
|
+
wafer evaluate kernelbench download
|
|
1680
|
+
wafer evaluate kernelbench download --force # Re-download
|
|
1681
|
+
"""
|
|
1682
|
+
try:
|
|
1683
|
+
path = download_problems("kernelbench", force=force, verbose=True)
|
|
1684
|
+
typer.echo("")
|
|
1685
|
+
typer.echo(f"Problems available at: {path}")
|
|
1686
|
+
typer.echo("Run 'wafer evaluate kernelbench list-problems' to see available problems.")
|
|
1687
|
+
except Exception as e:
|
|
1688
|
+
typer.echo(f"Error downloading problems: {e}", err=True)
|
|
1689
|
+
raise typer.Exit(1) from None
|
|
1690
|
+
|
|
1691
|
+
|
|
1692
|
+
@kernelbench_app.command("list-problems")
|
|
1693
|
+
def kernelbench_list_problems() -> None:
|
|
1694
|
+
"""List available KernelBench problems.
|
|
1695
|
+
|
|
1696
|
+
Examples:
|
|
1697
|
+
wafer evaluate kernelbench list-problems
|
|
1698
|
+
"""
|
|
1699
|
+
try:
|
|
1700
|
+
list_problems_fn("kernelbench", verbose=True)
|
|
1701
|
+
except ValueError as e:
|
|
1702
|
+
typer.echo(str(e), err=True)
|
|
1703
|
+
raise typer.Exit(1) from None
|
|
1508
1704
|
|
|
1509
1705
|
|
|
1510
1706
|
@kernelbench_app.callback(invoke_without_command=True)
|
|
@@ -1530,6 +1726,10 @@ def kernelbench_evaluate( # noqa: PLR0913
|
|
|
1530
1726
|
),
|
|
1531
1727
|
benchmark: bool = typer.Option(False, "--benchmark", help="Run performance benchmarks"),
|
|
1532
1728
|
profile: bool = typer.Option(False, "--profile", help="Enable profiling"),
|
|
1729
|
+
inputs: Path | None = typer.Option(
|
|
1730
|
+
None, "--inputs", help="Custom inputs file to override get_inputs()"
|
|
1731
|
+
),
|
|
1732
|
+
seed: int = typer.Option(42, "--seed", help="Random seed for weight initialization"),
|
|
1533
1733
|
defensive: bool = typer.Option(
|
|
1534
1734
|
False, "--defensive", help="Enable defensive timing to detect evaluation hacking"
|
|
1535
1735
|
),
|
|
@@ -1594,6 +1794,8 @@ def kernelbench_evaluate( # noqa: PLR0913
|
|
|
1594
1794
|
target_name=target or "",
|
|
1595
1795
|
benchmark=benchmark,
|
|
1596
1796
|
profile=profile,
|
|
1797
|
+
inputs=inputs,
|
|
1798
|
+
seed=seed,
|
|
1597
1799
|
defensive=defensive,
|
|
1598
1800
|
sync_artifacts=sync_artifacts,
|
|
1599
1801
|
gpu_id=gpu_id,
|
|
@@ -1655,6 +1857,13 @@ def kernelbench_make_template(
|
|
|
1655
1857
|
# Overwrite existing
|
|
1656
1858
|
wafer evaluate kernelbench make-template level1/1 --force
|
|
1657
1859
|
"""
|
|
1860
|
+
# Get problems root (downloaded or legacy)
|
|
1861
|
+
kb_root = _get_kernelbench_root()
|
|
1862
|
+
if kb_root is None:
|
|
1863
|
+
typer.echo("Error: KernelBench problems not found.", err=True)
|
|
1864
|
+
typer.echo("Run 'wafer evaluate kernelbench download' to download problems.", err=True)
|
|
1865
|
+
raise typer.Exit(1)
|
|
1866
|
+
|
|
1658
1867
|
# Parse problem ID
|
|
1659
1868
|
parts = problem.split("/")
|
|
1660
1869
|
if len(parts) != 2:
|
|
@@ -1666,10 +1875,10 @@ def kernelbench_make_template(
|
|
|
1666
1875
|
level_str = f"level{level_str}"
|
|
1667
1876
|
|
|
1668
1877
|
# Find the problem file
|
|
1669
|
-
problem_dir =
|
|
1878
|
+
problem_dir = kb_root / level_str
|
|
1670
1879
|
if not problem_dir.exists():
|
|
1671
1880
|
typer.echo(f"Error: KernelBench level directory not found: {problem_dir}", err=True)
|
|
1672
|
-
typer.echo(
|
|
1881
|
+
typer.echo("Run 'wafer evaluate kernelbench download' to download problems.", err=True)
|
|
1673
1882
|
raise typer.Exit(1)
|
|
1674
1883
|
|
|
1675
1884
|
# Find matching problem file
|
|
@@ -1736,6 +1945,252 @@ def kernelbench_make_template(
|
|
|
1736
1945
|
typer.echo(f" wafer evaluate kernelbench --impl my_kernel.py --reference {output}")
|
|
1737
1946
|
|
|
1738
1947
|
|
|
1948
|
+
# =============================================================================
|
|
1949
|
+
# GPUMode format evaluation
|
|
1950
|
+
# =============================================================================
|
|
1951
|
+
|
|
1952
|
+
|
|
1953
|
+
@gpumode_app.command("download")
|
|
1954
|
+
def gpumode_download(
|
|
1955
|
+
force: bool = typer.Option(False, "--force", "-f", help="Re-download even if exists"),
|
|
1956
|
+
) -> None:
|
|
1957
|
+
"""Download GPUMode reference kernels from GitHub.
|
|
1958
|
+
|
|
1959
|
+
Downloads the problem set to ~/.cache/wafer/problems/gpumode/
|
|
1960
|
+
|
|
1961
|
+
Examples:
|
|
1962
|
+
wafer evaluate gpumode download
|
|
1963
|
+
wafer evaluate gpumode download --force # Re-download
|
|
1964
|
+
"""
|
|
1965
|
+
try:
|
|
1966
|
+
path = download_problems("gpumode", force=force, verbose=True)
|
|
1967
|
+
typer.echo("")
|
|
1968
|
+
typer.echo(f"Problems available at: {path}")
|
|
1969
|
+
typer.echo("Run 'wafer evaluate gpumode list-problems' to see available problems.")
|
|
1970
|
+
except Exception as e:
|
|
1971
|
+
typer.echo(f"Error downloading problems: {e}", err=True)
|
|
1972
|
+
raise typer.Exit(1) from None
|
|
1973
|
+
|
|
1974
|
+
|
|
1975
|
+
@gpumode_app.command("list-problems")
|
|
1976
|
+
def gpumode_list_problems() -> None:
|
|
1977
|
+
"""List available GPUMode problems.
|
|
1978
|
+
|
|
1979
|
+
Examples:
|
|
1980
|
+
wafer evaluate gpumode list-problems
|
|
1981
|
+
"""
|
|
1982
|
+
try:
|
|
1983
|
+
list_problems_fn("gpumode", verbose=True)
|
|
1984
|
+
except ValueError as e:
|
|
1985
|
+
typer.echo(str(e), err=True)
|
|
1986
|
+
raise typer.Exit(1) from None
|
|
1987
|
+
|
|
1988
|
+
|
|
1989
|
+
@gpumode_app.command("make-template")
|
|
1990
|
+
def gpumode_make_template(
|
|
1991
|
+
problem: str = typer.Option(
|
|
1992
|
+
...,
|
|
1993
|
+
"--problem",
|
|
1994
|
+
"-p",
|
|
1995
|
+
help="Problem ID (e.g., 'pmpp/vectoradd_py' or 'amd/fp8-mm')",
|
|
1996
|
+
),
|
|
1997
|
+
output: Path = typer.Option(
|
|
1998
|
+
None, "--output", "-o", help="Output directory (default: ./<problem_name>/)"
|
|
1999
|
+
),
|
|
2000
|
+
force: bool = typer.Option(False, "--force", "-f", help="Overwrite existing files"),
|
|
2001
|
+
) -> None:
|
|
2002
|
+
"""Extract a GPUMode problem as template files.
|
|
2003
|
+
|
|
2004
|
+
Creates a directory with reference.py, task.yml, and other problem files.
|
|
2005
|
+
You then create kernel.py with your custom_kernel implementation.
|
|
2006
|
+
|
|
2007
|
+
Examples:
|
|
2008
|
+
# Extract pmpp vectoradd problem
|
|
2009
|
+
wafer evaluate gpumode make-template --problem pmpp/vectoradd_py
|
|
2010
|
+
|
|
2011
|
+
# Extract to specific directory
|
|
2012
|
+
wafer evaluate gpumode make-template --problem pmpp/vectoradd_py --output ./my-kernel/
|
|
2013
|
+
"""
|
|
2014
|
+
import shutil
|
|
2015
|
+
|
|
2016
|
+
# Get problem path
|
|
2017
|
+
problem_path = get_problem_path("gpumode", problem)
|
|
2018
|
+
if problem_path is None:
|
|
2019
|
+
# Check if problems are downloaded
|
|
2020
|
+
if get_problems_path("gpumode") is None:
|
|
2021
|
+
typer.echo("Error: GPUMode problems not downloaded.", err=True)
|
|
2022
|
+
typer.echo("Run 'wafer evaluate gpumode download' first.", err=True)
|
|
2023
|
+
else:
|
|
2024
|
+
typer.echo(f"Error: Problem '{problem}' not found.", err=True)
|
|
2025
|
+
typer.echo(
|
|
2026
|
+
"Run 'wafer evaluate gpumode list-problems' to see available problems.", err=True
|
|
2027
|
+
)
|
|
2028
|
+
raise typer.Exit(1)
|
|
2029
|
+
|
|
2030
|
+
# Determine output path
|
|
2031
|
+
if output is None:
|
|
2032
|
+
output = Path.cwd() / problem.replace("/", "_")
|
|
2033
|
+
|
|
2034
|
+
output = output.resolve()
|
|
2035
|
+
|
|
2036
|
+
# Check if exists
|
|
2037
|
+
if output.exists() and not force:
|
|
2038
|
+
typer.echo(f"Error: {output} already exists. Use --force to overwrite.", err=True)
|
|
2039
|
+
raise typer.Exit(1)
|
|
2040
|
+
|
|
2041
|
+
# Copy the problem directory
|
|
2042
|
+
if output.exists():
|
|
2043
|
+
shutil.rmtree(output)
|
|
2044
|
+
shutil.copytree(problem_path, output)
|
|
2045
|
+
|
|
2046
|
+
typer.echo(f"Created {output}/")
|
|
2047
|
+
typer.echo("")
|
|
2048
|
+
typer.echo("Contents:")
|
|
2049
|
+
for f in sorted(output.iterdir()):
|
|
2050
|
+
if not f.name.startswith("."):
|
|
2051
|
+
typer.echo(f" {f.name}")
|
|
2052
|
+
typer.echo("")
|
|
2053
|
+
typer.echo("Next steps:")
|
|
2054
|
+
typer.echo(" 1. Read reference.py to understand the kernel interface")
|
|
2055
|
+
typer.echo(" 2. Create kernel.py with your custom_kernel implementation:")
|
|
2056
|
+
typer.echo("")
|
|
2057
|
+
typer.echo(" def custom_kernel(data):")
|
|
2058
|
+
typer.echo(" # Your optimized implementation")
|
|
2059
|
+
typer.echo(" ...")
|
|
2060
|
+
typer.echo("")
|
|
2061
|
+
typer.echo(" 3. Run evaluation:")
|
|
2062
|
+
typer.echo(
|
|
2063
|
+
f" wafer evaluate gpumode --impl {output}/kernel.py --reference {output}/reference.py \\"
|
|
2064
|
+
)
|
|
2065
|
+
typer.echo(f" --test-cases {output}/test_cases.json --target <target>")
|
|
2066
|
+
|
|
2067
|
+
|
|
2068
|
+
@gpumode_app.callback(invoke_without_command=True)
|
|
2069
|
+
def gpumode_evaluate( # noqa: PLR0913
|
|
2070
|
+
ctx: typer.Context,
|
|
2071
|
+
implementation: Path | None = typer.Option(
|
|
2072
|
+
None, "--impl", "-i", help="Path to implementation kernel file"
|
|
2073
|
+
),
|
|
2074
|
+
reference: Path | None = typer.Option(
|
|
2075
|
+
None, "--reference", help="Path to reference kernel file"
|
|
2076
|
+
),
|
|
2077
|
+
test_cases: Path | None = typer.Option(
|
|
2078
|
+
None, "--test-cases", help="Path to test cases JSON file"
|
|
2079
|
+
),
|
|
2080
|
+
target: str | None = typer.Option(
|
|
2081
|
+
None,
|
|
2082
|
+
"--target",
|
|
2083
|
+
"-t",
|
|
2084
|
+
help="GPU target name. See 'wafer config targets list' for available targets.",
|
|
2085
|
+
autocompletion=complete_target_name,
|
|
2086
|
+
),
|
|
2087
|
+
benchmark: bool = typer.Option(False, "--benchmark", help="Run performance benchmarks"),
|
|
2088
|
+
profile: bool = typer.Option(False, "--profile", help="Enable profiling"),
|
|
2089
|
+
defensive: bool = typer.Option(
|
|
2090
|
+
False, "--defensive", help="Enable defensive timing to detect evaluation hacking"
|
|
2091
|
+
),
|
|
2092
|
+
sync_artifacts: bool = typer.Option(
|
|
2093
|
+
True, "--sync-artifacts/--no-sync-artifacts", help="Download artifacts"
|
|
2094
|
+
),
|
|
2095
|
+
gpu_id: int | None = typer.Option(None, "--gpu-id", help="Override GPU ID"),
|
|
2096
|
+
) -> None:
|
|
2097
|
+
"""Run kernel evaluation in GPUMode format (functional).
|
|
2098
|
+
|
|
2099
|
+
This format expects:
|
|
2100
|
+
- Implementation: Python file with `custom_kernel(inputs)` function
|
|
2101
|
+
- Reference: Python file with `ref_kernel(inputs)` and `generate_input(**kwargs)` functions
|
|
2102
|
+
- Test cases: JSON file with test parameters
|
|
2103
|
+
|
|
2104
|
+
Examples:
|
|
2105
|
+
# Basic correctness check
|
|
2106
|
+
wafer evaluate gpumode --impl kernel.py --reference ref.py --test-cases tests.json
|
|
2107
|
+
|
|
2108
|
+
# With benchmarking
|
|
2109
|
+
wafer evaluate gpumode --impl kernel.py --reference ref.py --test-cases tests.json \\
|
|
2110
|
+
--target vultr-b200 --benchmark
|
|
2111
|
+
|
|
2112
|
+
Subcommands:
|
|
2113
|
+
download Download GPUMode problems from GitHub
|
|
2114
|
+
list-problems List available problems
|
|
2115
|
+
make-template Extract a problem as template files
|
|
2116
|
+
"""
|
|
2117
|
+
# If a subcommand is being invoked, skip the main evaluation logic
|
|
2118
|
+
if ctx.invoked_subcommand is not None:
|
|
2119
|
+
return
|
|
2120
|
+
|
|
2121
|
+
# Validate required args when running evaluation (not subcommands)
|
|
2122
|
+
missing_args = []
|
|
2123
|
+
if implementation is None:
|
|
2124
|
+
missing_args.append("--impl/-i")
|
|
2125
|
+
if reference is None:
|
|
2126
|
+
missing_args.append("--reference")
|
|
2127
|
+
if test_cases is None:
|
|
2128
|
+
missing_args.append("--test-cases")
|
|
2129
|
+
|
|
2130
|
+
if missing_args:
|
|
2131
|
+
typer.echo("Error: Missing required arguments", err=True)
|
|
2132
|
+
typer.echo(f" Required: {', '.join(missing_args)}", err=True)
|
|
2133
|
+
typer.echo("", err=True)
|
|
2134
|
+
typer.echo(
|
|
2135
|
+
"Usage: wafer evaluate gpumode --impl KERNEL.py --reference REF.py --test-cases TESTS.json",
|
|
2136
|
+
err=True,
|
|
2137
|
+
)
|
|
2138
|
+
typer.echo("", err=True)
|
|
2139
|
+
typer.echo("Run 'wafer evaluate gpumode --help' for full options.", err=True)
|
|
2140
|
+
typer.echo("Run 'wafer evaluate gpumode download' to download problem sets.", err=True)
|
|
2141
|
+
raise typer.Exit(1)
|
|
2142
|
+
|
|
2143
|
+
# Reuse the existing evaluate logic (same format)
|
|
2144
|
+
from .evaluate import EvaluateArgs, run_evaluate
|
|
2145
|
+
|
|
2146
|
+
args = EvaluateArgs(
|
|
2147
|
+
implementation=implementation,
|
|
2148
|
+
reference=reference,
|
|
2149
|
+
test_cases=test_cases,
|
|
2150
|
+
target_name=target or "",
|
|
2151
|
+
benchmark=benchmark,
|
|
2152
|
+
profile=profile,
|
|
2153
|
+
defensive=defensive,
|
|
2154
|
+
sync_artifacts=sync_artifacts,
|
|
2155
|
+
gpu_id=gpu_id,
|
|
2156
|
+
)
|
|
2157
|
+
|
|
2158
|
+
try:
|
|
2159
|
+
import trio_asyncio
|
|
2160
|
+
|
|
2161
|
+
result = trio_asyncio.run(run_evaluate, args)
|
|
2162
|
+
except KeyboardInterrupt:
|
|
2163
|
+
typer.echo("\nInterrupted by user", err=True)
|
|
2164
|
+
raise typer.Exit(130) from None
|
|
2165
|
+
except Exception as e:
|
|
2166
|
+
if hasattr(e, "exceptions") and e.exceptions:
|
|
2167
|
+
for exc in e.exceptions:
|
|
2168
|
+
typer.echo(f"Error: {type(exc).__name__}: {exc}", err=True)
|
|
2169
|
+
else:
|
|
2170
|
+
typer.echo(f"Error: {e}", err=True)
|
|
2171
|
+
raise typer.Exit(1) from None
|
|
2172
|
+
|
|
2173
|
+
# Print results
|
|
2174
|
+
if result.success:
|
|
2175
|
+
typer.echo("")
|
|
2176
|
+
typer.echo("=" * 60)
|
|
2177
|
+
status = "PASS" if result.all_correct else "FAIL"
|
|
2178
|
+
typer.echo(f"Result: {status}")
|
|
2179
|
+
score_pct = f"{result.correctness_score:.1%}"
|
|
2180
|
+
typer.echo(f"Correctness: {result.passed_tests}/{result.total_tests} ({score_pct})")
|
|
2181
|
+
if result.geomean_speedup > 0:
|
|
2182
|
+
typer.echo(f"Speedup: {result.geomean_speedup:.2f}x")
|
|
2183
|
+
if result.artifact_path:
|
|
2184
|
+
typer.echo(f"Artifacts: {result.artifact_path}")
|
|
2185
|
+
typer.echo("=" * 60)
|
|
2186
|
+
|
|
2187
|
+
if not result.all_correct:
|
|
2188
|
+
raise typer.Exit(1)
|
|
2189
|
+
else:
|
|
2190
|
+
typer.echo(f"Error: {result.error_message}", err=True)
|
|
2191
|
+
raise typer.Exit(1)
|
|
2192
|
+
|
|
2193
|
+
|
|
1739
2194
|
# =============================================================================
|
|
1740
2195
|
# Push and Remote-Run commands
|
|
1741
2196
|
# =============================================================================
|
|
@@ -1867,7 +2322,7 @@ def _run_direct_mode(
|
|
|
1867
2322
|
typer.echo(f"Uploading {upload_dir.name}...")
|
|
1868
2323
|
try:
|
|
1869
2324
|
push_result = push_direct(upload_dir, target)
|
|
1870
|
-
workspace_name = push_result.
|
|
2325
|
+
workspace_name = push_result.workspace_name
|
|
1871
2326
|
typer.echo(f"Uploaded {len(push_result.files_uploaded)} files")
|
|
1872
2327
|
except Exception as e:
|
|
1873
2328
|
typer.echo(f"Error uploading: {e}", err=True)
|
|
@@ -2040,7 +2495,10 @@ def login(
|
|
|
2040
2495
|
None, "--token", "-t", help="Access token (skip browser OAuth)"
|
|
2041
2496
|
),
|
|
2042
2497
|
port: int | None = typer.Option(
|
|
2043
|
-
None,
|
|
2498
|
+
None,
|
|
2499
|
+
"--port",
|
|
2500
|
+
"-p",
|
|
2501
|
+
help="Port for OAuth callback server (default: 8765 for SSH, random for local)",
|
|
2044
2502
|
),
|
|
2045
2503
|
) -> None:
|
|
2046
2504
|
"""Authenticate CLI with wafer-api via GitHub OAuth.
|
|
@@ -2142,9 +2600,8 @@ def login(
|
|
|
2142
2600
|
@app.command("logout")
|
|
2143
2601
|
def logout() -> None:
|
|
2144
2602
|
"""Remove stored credentials."""
|
|
2145
|
-
from .auth import clear_credentials
|
|
2146
|
-
|
|
2147
2603
|
from . import analytics
|
|
2604
|
+
from .auth import clear_credentials
|
|
2148
2605
|
|
|
2149
2606
|
# Track logout event first (while credentials still exist for user identification)
|
|
2150
2607
|
# Note: track_logout() handles the case where user is not logged in
|
|
@@ -3115,7 +3572,9 @@ def billing_usage(
|
|
|
3115
3572
|
@billing_app.command("topup")
|
|
3116
3573
|
def billing_topup(
|
|
3117
3574
|
amount: int = typer.Argument(25, help="Amount in dollars ($10-$500)"),
|
|
3118
|
-
no_browser: bool = typer.Option(
|
|
3575
|
+
no_browser: bool = typer.Option(
|
|
3576
|
+
False, "--no-browser", help="Print URL instead of opening browser"
|
|
3577
|
+
),
|
|
3119
3578
|
) -> None:
|
|
3120
3579
|
"""Add credits to your account.
|
|
3121
3580
|
|
|
@@ -3161,7 +3620,9 @@ def billing_topup(
|
|
|
3161
3620
|
|
|
3162
3621
|
@billing_app.command("portal")
|
|
3163
3622
|
def billing_portal(
|
|
3164
|
-
no_browser: bool = typer.Option(
|
|
3623
|
+
no_browser: bool = typer.Option(
|
|
3624
|
+
False, "--no-browser", help="Print URL instead of opening browser"
|
|
3625
|
+
),
|
|
3165
3626
|
) -> None:
|
|
3166
3627
|
"""Open Stripe billing portal.
|
|
3167
3628
|
|
|
@@ -4437,8 +4898,8 @@ def _setup_wafer_core_env() -> None:
|
|
|
4437
4898
|
- WAFER_API_URL: If already set, uses that instead of config
|
|
4438
4899
|
- WAFER_AUTH_TOKEN: If already set, uses that instead of cached token
|
|
4439
4900
|
"""
|
|
4440
|
-
from .global_config import get_api_url
|
|
4441
4901
|
from .auth import get_valid_token
|
|
4902
|
+
from .global_config import get_api_url
|
|
4442
4903
|
|
|
4443
4904
|
# Set API URL (get_api_url already respects WAFER_API_URL env var)
|
|
4444
4905
|
os.environ["WAFER_API_URL"] = get_api_url()
|
|
@@ -4742,8 +5203,8 @@ def capture_command( # noqa: PLR0915
|
|
|
4742
5203
|
import os
|
|
4743
5204
|
import tomllib
|
|
4744
5205
|
|
|
4745
|
-
from .global_config import get_api_url
|
|
4746
5206
|
from .auth import get_valid_token
|
|
5207
|
+
from .global_config import get_api_url
|
|
4747
5208
|
|
|
4748
5209
|
# Set environment variables for wafer-core BEFORE importing it
|
|
4749
5210
|
# wafer-core backend.py reads WAFER_API_URL and WAFER_AUTH_TOKEN from env
|
|
@@ -4947,8 +5408,8 @@ def capture_list_command(
|
|
|
4947
5408
|
"""
|
|
4948
5409
|
import os
|
|
4949
5410
|
|
|
4950
|
-
from .global_config import get_api_url
|
|
4951
5411
|
from .auth import get_valid_token
|
|
5412
|
+
from .global_config import get_api_url
|
|
4952
5413
|
|
|
4953
5414
|
# Set environment variables for wafer-core BEFORE importing it
|
|
4954
5415
|
os.environ["WAFER_API_URL"] = get_api_url()
|