tt-check 0.1.0__tar.gz

tt_check-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,28 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+  push:
+    tags:
+      - "v*"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+      - name: Build distributions
+        run: |
+          python -m pip install --upgrade build
+          python -m build
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

tt_check-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+.DS_Store
+*.py[cod]
+__pycache__/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.venv/
+build/
+dist/
+*.egg-info/

tt_check-0.1.0/.python-version

@@ -0,0 +1 @@
+3.10

tt_check-0.1.0/LICENSE

@@ -0,0 +1,176 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction,
+and distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by
+the copyright owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all
+other entities that control, are controlled by, or are under common
+control with that entity. For the purposes of this definition,
+"control" means (i) the power, direct or indirect, to cause the
+direction or management of such entity, whether by contract or
+otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity
+exercising permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications,
+including but not limited to software source code, documentation
+source, and configuration files.
+
+"Object" form shall mean any form resulting from mechanical
+transformation or translation of a Source form, including but
+not limited to compiled object code, generated documentation,
+and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or
+Object form, made available under the License, as indicated by a
+copyright notice that is included in or attached to the work
+(an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object
+form, that is based on (or derived from) the Work and for which the
+editorial revisions, annotations, elaborations, or other modifications
+represent, as a whole, an original work of authorship. For the purposes
+of this License, Derivative Works shall not include works that remain
+separable from, or merely link (or bind by name) to the interfaces of,
+the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including
+the original version of the Work and any modifications or additions
+to that Work or Derivative Works thereof, that is intentionally
+submitted to Licensor for inclusion in the Work by the copyright owner
+or by an individual or Legal Entity authorized to submit on behalf of
+the copyright owner. For the purposes of this definition, "submitted"
+means any form of electronic, verbal, or written communication sent
+to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems,
+and issue tracking systems that are managed by, or on behalf of, the
+Licensor for the purpose of discussing and improving the Work, but
+excluding communication that is conspicuously marked or otherwise
+designated in writing by the copyright owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity
+on behalf of whom a Contribution has been received by Licensor and
+subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+this License, each Contributor hereby grants to You a perpetual,
+worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+copyright license to reproduce, prepare Derivative Works of,
+publicly display, publicly perform, sublicense, and distribute the
+Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+this License, each Contributor hereby grants to You a perpetual,
+worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+(except as stated in this section) patent license to make, have made,
+use, offer to sell, sell, import, and otherwise transfer the Work,
+where such license applies only to those patent claims licensable
+by such Contributor that are necessarily infringed by their
+Contribution(s) alone or by combination of their Contribution(s)
+with the Work to which such Contribution(s) was submitted. If You
+institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work
+or a Contribution incorporated within the Work constitutes direct
+or contributory patent infringement, then any patent licenses
+granted to You under this License for that Work shall terminate
+as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+Work or Derivative Works thereof in any medium, with or without
+modifications, and in Source or Object form, provided that You
+meet the following conditions:
+
+(a) You must give any other recipients of the Work or Derivative
+Works a copy of this License; and
+
+(b) You must cause any modified files to carry prominent notices
+stating that You changed the files; and
+
+(c) You must retain, in the Source form of any Derivative Works
+that You distribute, all copyright, patent, trademark, and
+attribution notices from the Source form of the Work, excluding
+those notices that do not pertain to any part of the Derivative
+Works; and
+
+(d) If the Work includes a "NOTICE" text file as part of its
+distribution, then any Derivative Works that You distribute must
+include a readable copy of the attribution notices contained
+within such NOTICE file, excluding those notices that do not
+pertain to any part of the Derivative Works, in at least one
+of the following places: within a NOTICE text file distributed
+as part of the Derivative Works; within the Source form or
+documentation, if provided along with the Derivative Works; or,
+within a display generated by the Derivative Works, if and
+wherever such third-party notices normally appear. The contents
+of the NOTICE file are for informational purposes only and
+do not modify the License. You may add Your own attribution
+notices within Derivative Works that You distribute, alongside
+or as an addendum to the NOTICE text from the Work, provided
+that such additional attribution notices cannot be construed
+as modifying the License.
+
+You may add Your own copyright statement to Your modifications and
+may provide additional or different license terms and conditions
+for use, reproduction, or distribution of Your modifications, or
+for any such Derivative Works as a whole, provided Your use,
+reproduction, and distribution of the Work otherwise complies with
+the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+any Contribution intentionally submitted for inclusion in the Work
+by You to the Licensor shall be under the terms and conditions of
+this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify
+the terms of any separate license agreement you may have executed
+with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+names, trademarks, service marks, or product names of the Licensor,
+except as required for reasonable and customary use in describing the
+origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+agreed to in writing, Licensor provides the Work (and each
+Contributor provides its Contributions) on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied, including, without limitation, any warranties or conditions
+of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+PARTICULAR PURPOSE. You are solely responsible for determining the
+appropriateness of using or redistributing the Work and assume any
+risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+whether in tort (including negligence), contract, or otherwise,
+unless required by applicable law (such as deliberate and grossly
+negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special,
+incidental, or consequential damages of any character arising as a
+result of this License or out of the use or inability to use the
+Work (including but not limited to damages for loss of goodwill,
+work stoppage, computer failure or malfunction, or any and all
+other commercial damages or losses), even if such Contributor
+has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+the Work or Derivative Works thereof, You may choose to offer,
+and charge a fee for, acceptance of support, warranty, indemnity,
+or other liability obligations and/or rights consistent with this
+License. However, in accepting such obligations, You may act only
+on Your own behalf and on Your sole responsibility, not on behalf
+of any other Contributor, and only if You agree to indemnify,
+defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason
+of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS

tt_check-0.1.0/PKG-INFO

@@ -0,0 +1,87 @@
+Metadata-Version: 2.4
+Name: tt-check
+Version: 0.1.0
+Summary: Small TTNN readiness check for Tenstorrent systems
+Project-URL: Homepage, https://github.com/yieldthought/tt-check
+Project-URL: Repository, https://github.com/yieldthought/tt-check
+Project-URL: Issues, https://github.com/yieldthought/tt-check/issues
+Author: Tenstorrent
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: hardware,readiness,tenstorrent,tt-smi,ttnn
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: !=3.11.*,<3.13,>=3.10
+Requires-Dist: torch; platform_system == 'Linux'
+Requires-Dist: tt-smi>=3.1.1; platform_system == 'Linux'
+Requires-Dist: ttnn; platform_system == 'Linux'
+Description-Content-Type: text/markdown
+
+# tt-check
+
+`tt-check` is a small command-line readiness test for a Tenstorrent system with
+`tt-smi` and the public `ttnn` Python package installed.
+
+It performs the following checks:
+
+1. Resets devices with `tt-smi -r`.
+2. Collects system information from `tt-smi -s --snapshot_no_tty`.
+3. Opens a TTNN device.
+4. Runs a simulated tensor-parallel three-weight gated MLP 100 times in both
+   prefill and decode shapes:
+   - BF16 activations
+   - BFP8 weights
+   - 1024 activation width per device shard
+   - prefill rows: 1024
+   - decode rows: 1
+5. Warms each MLP shape once, captures one dynamic TTNN trace per shape, then
+   executes each trace for the requested run count.
+6. Compares each trace replay against a PyTorch reference with PCC >= 0.99 and
+   requires TTNN output tensors to be identical across all replays.
+
+## Install
+
+Preferred:
+
+```bash
+uv tool install --python 3.10 .
+```
+
+Or run directly from a checkout:
+
+```bash
+uv run tt-check
+```
+
+`uv` uses the PyTorch CPU wheel index for this project; the check only needs
+Torch for reference math. The project pins a Python version compatible with the
+current public `ttnn` wheels.
+
+Pip also works:
+
+```bash
+python3 -m pip install .
+```
+
+## Run
+
+```bash
+tt-check
+```
+
+The command exits `0` if all checks pass. It exits `1` and writes the failure to
+stderr otherwise.
+
+Useful options:
+
+```bash
+tt-check --device-id 0 --runs 100 --pcc-threshold 0.99
+tt-check --prefill-rows 1024 --decode-rows 1 --activation-width-per-device 1024
+```

tt_check-0.1.0/README.md

@@ -0,0 +1,61 @@
+# tt-check
+
+`tt-check` is a small command-line readiness test for a Tenstorrent system with
+`tt-smi` and the public `ttnn` Python package installed.
+
+It performs the following checks:
+
+1. Resets devices with `tt-smi -r`.
+2. Collects system information from `tt-smi -s --snapshot_no_tty`.
+3. Opens a TTNN device.
+4. Runs a simulated tensor-parallel three-weight gated MLP 100 times in both
+   prefill and decode shapes:
+   - BF16 activations
+   - BFP8 weights
+   - 1024 activation width per device shard
+   - prefill rows: 1024
+   - decode rows: 1
+5. Warms each MLP shape once, captures one dynamic TTNN trace per shape, then
+   executes each trace for the requested run count.
+6. Compares each trace replay against a PyTorch reference with PCC >= 0.99 and
+   requires TTNN output tensors to be identical across all replays.
+
+## Install
+
+Preferred:
+
+```bash
+uv tool install --python 3.10 .
+```
+
+Or run directly from a checkout:
+
+```bash
+uv run tt-check
+```
+
+`uv` uses the PyTorch CPU wheel index for this project; the check only needs
+Torch for reference math. The project pins a Python version compatible with the
+current public `ttnn` wheels.
+
+Pip also works:
+
+```bash
+python3 -m pip install .
+```
+
+## Run
+
+```bash
+tt-check
+```
+
+The command exits `0` if all checks pass. It exits `1` and writes the failure to
+stderr otherwise.
+
+Useful options:
+
+```bash
+tt-check --device-id 0 --runs 100 --pcc-threshold 0.99
+tt-check --prefill-rows 1024 --decode-rows 1 --activation-width-per-device 1024
+```

tt_check-0.1.0/pyproject.toml

@@ -0,0 +1,52 @@
+[build-system]
+requires = ["hatchling>=1.25"]
+build-backend = "hatchling.build"
+
+[project]
+name = "tt-check"
+version = "0.1.0"
+description = "Small TTNN readiness check for Tenstorrent systems"
+readme = "README.md"
+requires-python = ">=3.10,<3.13,!=3.11.*"
+license = "Apache-2.0"
+authors = [
+  { name = "Tenstorrent" },
+]
+keywords = ["tenstorrent", "ttnn", "tt-smi", "readiness", "hardware"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Environment :: Console",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: Apache Software License",
+  "Operating System :: POSIX :: Linux",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.12",
+  "Topic :: Software Development :: Testing",
+]
+dependencies = [
+  "torch; platform_system == 'Linux'",
+  "tt-smi>=3.1.1; platform_system == 'Linux'",
+  "ttnn; platform_system == 'Linux'",
+]
+
+[project.urls]
+Homepage = "https://github.com/yieldthought/tt-check"
+Repository = "https://github.com/yieldthought/tt-check"
+Issues = "https://github.com/yieldthought/tt-check/issues"
+
+[project.scripts]
+tt-check = "tt_check.cli:main"
+
+[tool.uv.sources]
+torch = [
+  { index = "pytorch-cpu", marker = "platform_system == 'Linux'" },
+]
+
+[[tool.uv.index]]
+name = "pytorch-cpu"
+url = "https://download.pytorch.org/whl/cpu"
+explicit = true
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/tt_check"]

tt_check-0.1.0/src/tt_check/__init__.py

@@ -0,0 +1,5 @@
+"""TTNN readiness check package."""
+
+__all__ = ["__version__"]
+
+__version__ = "0.1.0"

tt_check-0.1.0/src/tt_check/cli.py

@@ -0,0 +1,680 @@
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+class CheckError(RuntimeError):
+    """Expected readiness-check failure."""
+
+
+@dataclass(frozen=True)
+class CommandResult:
+    stdout: str
+    stderr: str
+
+
+def main(argv: list[str] | None = None) -> int:
+    raw_argv = list(sys.argv[1:] if argv is None else argv)
+    args = _parse_args(raw_argv)
+    if args._worker_json:
+        return _main_worker(args)
+    return _main_parent(raw_argv, args)
+
+
+def _main_worker(args: argparse.Namespace) -> int:
+    try:
+        result = run_check(args)
+        Path(args._worker_json).write_text(json.dumps(result, indent=2, sort_keys=True), encoding="utf-8")
+    except CheckError as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        return 1
+    except KeyboardInterrupt:
+        print("ERROR: interrupted", file=sys.stderr)
+        return 1
+    except Exception as exc:
+        print(f"ERROR: unexpected failure: {type(exc).__name__}: {exc}", file=sys.stderr)
+        return 1
+    return 0
+
+
+def _main_parent(raw_argv: list[str], args: argparse.Namespace) -> int:
+    with tempfile.TemporaryDirectory(prefix="tt-check-") as tmp_dir:
+        result_path = Path(tmp_dir) / "result.json"
+        stdout_path = Path(tmp_dir) / "stdout.txt"
+        stderr_path = Path(tmp_dir) / "stderr.txt"
+        worker_argv = [*raw_argv, "--_worker-json", str(result_path)]
+        env = os.environ.copy()
+        env.setdefault("TT_LOGGER_LEVEL", "FATAL")
+        env.setdefault("LOGURU_LEVEL", "ERROR")
+
+        with stdout_path.open("w", encoding="utf-8") as stdout_file, stderr_path.open(
+            "w", encoding="utf-8"
+        ) as stderr_file:
+            process = subprocess.Popen(
+                [sys.executable, "-m", "tt_check.cli", *worker_argv],
+                stdout=stdout_file,
+                stderr=stderr_file,
+                text=True,
+                env=env,
+            )
+            try:
+                _wait_with_progress(process, enabled=not args.json)
+            except KeyboardInterrupt:
+                process.terminate()
+                try:
+                    process.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    process.kill()
+                print("ERROR: interrupted", file=sys.stderr)
+                return 1
+
+        stdout = _read_text(stdout_path)
+        stderr = _read_text(stderr_path)
+        if process.returncode != 0:
+            print(_format_failure(stdout, stderr), file=sys.stderr)
+            return 1
+        if not result_path.exists():
+            print(_format_failure(stdout, stderr, fallback="ERROR: tt-check did not produce a result"), file=sys.stderr)
+            return 1
+
+        try:
+            result = json.loads(result_path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError as exc:
+            print(f"ERROR: invalid result JSON: {exc}", file=sys.stderr)
+            return 1
+
+    if args.json:
+        print(json.dumps(result, indent=2, sort_keys=True))
+    else:
+        print(_format_human_result(result))
+    return 0
+
+
+def _parse_args(argv: list[str] | None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Run a small TTNN readiness check.")
+    parser.add_argument("--device-id", type=int, default=0, help="TTNN device id to open.")
+    parser.add_argument("--runs", type=int, default=100, help="Number of identical MLP runs per mode.")
+    parser.add_argument("--pcc-threshold", type=float, default=0.99, help="Minimum PCC against PyTorch reference.")
+    parser.add_argument(
+        "--activation-width-per-device",
+        type=int,
+        default=1024,
+        help="Activation shard width used for the simulated tensor-parallel MLP.",
+    )
+    parser.add_argument("--prefill-rows", type=int, default=1024, help="Input rows for prefill mode.")
+    parser.add_argument("--decode-rows", type=int, default=1, help="Input rows for decode mode.")
+    parser.add_argument(
+        "--intermediate-multiplier",
+        type=int,
+        default=4,
+        help="MLP intermediate width multiplier relative to the activation width.",
+    )
+    parser.add_argument("--seed", type=int, default=0, help="Random seed for deterministic inputs and weights.")
+    parser.add_argument("--tt-smi-timeout", type=float, default=120.0, help="Timeout for each tt-smi command.")
+    parser.add_argument("--json", action="store_true", help="Print final result as JSON.")
+    parser.add_argument("--_worker-json", dest="_worker_json", default=None, help=argparse.SUPPRESS)
+    return parser.parse_args(argv)
+
+
+def run_check(args: argparse.Namespace) -> dict[str, Any]:
+    _validate_args(args)
+    _require_executable("tt-smi")
+
+    reset_result = _run_command(["tt-smi", "-r"], timeout=args.tt_smi_timeout)
+    snapshot = collect_tt_smi_snapshot(timeout=args.tt_smi_timeout)
+    system_info = summarize_system(snapshot)
+
+    mlp_results = run_ttnn_mlp_check(
+        device_id=args.device_id,
+        runs=args.runs,
+        pcc_threshold=args.pcc_threshold,
+        activation_width=args.activation_width_per_device,
+        prefill_rows=args.prefill_rows,
+        decode_rows=args.decode_rows,
+        intermediate_multiplier=args.intermediate_multiplier,
+        seed=args.seed,
+    )
+
+    return {
+        "status": "pass",
+        "reset": {"stdout": reset_result.stdout.strip(), "stderr": reset_result.stderr.strip()},
+        "system": system_info,
+        "mlp": mlp_results,
+    }
+
+
+def _validate_args(args: argparse.Namespace) -> None:
+    if args.runs < 1:
+        raise CheckError("--runs must be >= 1")
+    if not 0.0 <= args.pcc_threshold <= 1.0:
+        raise CheckError("--pcc-threshold must be between 0 and 1")
+    for name in ("activation_width_per_device", "prefill_rows", "decode_rows", "intermediate_multiplier"):
+        if getattr(args, name) < 1:
+            raise CheckError(f"--{name.replace('_', '-')} must be >= 1")
+    if args.activation_width_per_device % 32 != 0:
+        raise CheckError("--activation-width-per-device must be divisible by 32 for tiled matmuls")
+    if (args.activation_width_per_device * args.intermediate_multiplier) % 32 != 0:
+        raise CheckError("intermediate width must be divisible by 32 for tiled matmuls")
+
+
+def _require_executable(name: str) -> None:
+    if shutil.which(name) is None:
+        raise CheckError(f"{name} not found on PATH")
+
+
+def _run_command(command: list[str], *, timeout: float) -> CommandResult:
+    try:
+        result = subprocess.run(command, capture_output=True, text=True, timeout=timeout, check=False)
+    except FileNotFoundError:
+        raise CheckError(f"{command[0]} not found on PATH") from None
+    except subprocess.TimeoutExpired as exc:
+        raise CheckError(f"{' '.join(command)} timed out after {timeout:g}s") from exc
+    if result.returncode != 0:
+        details = _command_output_summary(result.stdout, result.stderr)
+        raise CheckError(f"{' '.join(command)} failed with exit code {result.returncode}{details}")
+    return CommandResult(stdout=result.stdout, stderr=result.stderr)
+
+
+def _command_output_summary(stdout: str, stderr: str) -> str:
+    combined = "\n".join(part.strip() for part in (stdout, stderr) if part.strip())
+    if not combined:
+        return ""
+    return f": {combined[-2000:]}"
+
+
+def _wait_with_progress(process: subprocess.Popen[str], *, enabled: bool) -> None:
+    start = time.monotonic()
+    drew_progress = False
+    while process.poll() is None:
+        elapsed = time.monotonic() - start
+        if enabled and sys.stdout.isatty() and elapsed >= 10:
+            drew_progress = True
+            sys.stdout.write(f"\rtt-check {_moving_bar(elapsed)} {elapsed:4.0f}s")
+            sys.stdout.flush()
+        time.sleep(0.2)
+    if drew_progress:
+        sys.stdout.write("\r" + " " * 48 + "\r")
+        sys.stdout.flush()
+
+
+def _moving_bar(elapsed: float, *, width: int = 24) -> str:
+    position = int(elapsed * 8) % (width * 2)
+    if position >= width:
+        position = width * 2 - position - 1
+    cells = ["-"] * width
+    for offset in range(5):
+        index = position - offset
+        if 0 <= index < width:
+            cells[index] = "#"
+    return "[" + "".join(cells) + "]"
+
+
+def _read_text(path: Path) -> str:
+    try:
+        return path.read_text(encoding="utf-8", errors="replace")
+    except FileNotFoundError:
+        return ""
+
+
+def _format_failure(stdout: str, stderr: str, *, fallback: str = "ERROR: tt-check failed") -> str:
+    combined = "\n".join(part.strip() for part in (stderr, stdout) if part.strip())
+    if not combined:
+        return fallback
+
+    lines = [line.rstrip() for line in combined.splitlines() if line.strip()]
+    error_lines = [line for line in lines if line.startswith("ERROR:")]
+    if error_lines:
+        return error_lines[-1]
+
+    tail = "\n".join(lines[-20:])
+    return f"{fallback}\n\nLast diagnostics:\n{tail}"
+
+
+def collect_tt_smi_snapshot(*, timeout: float) -> dict[str, Any]:
+    for command in (["tt-smi", "-s", "--snapshot_no_tty"], ["tt-smi", "-s"]):
+        try:
+            result = _run_command(command, timeout=timeout)
+            return _parse_json_from_output(result.stdout)
+        except CheckError:
+            if command[-1] != "-s":
+                continue
+            raise
+    raise CheckError("unable to collect tt-smi snapshot")
+
+
+def _parse_json_from_output(output: str) -> dict[str, Any]:
+    text = output.strip()
+    start = text.find("{")
+    end = text.rfind("}")
+    if start < 0 or end < start:
+        raise CheckError("tt-smi snapshot did not contain JSON")
+    try:
+        parsed = json.loads(text[start : end + 1])
+    except json.JSONDecodeError as exc:
+        raise CheckError(f"failed to parse tt-smi JSON snapshot: {exc}") from exc
+    if not isinstance(parsed, dict):
+        raise CheckError("tt-smi snapshot JSON root was not an object")
+    return parsed
+
+
+def summarize_system(snapshot: dict[str, Any]) -> dict[str, Any]:
+    devices = _devices_from_snapshot(snapshot)
+    if not devices:
+        raise CheckError("tt-smi snapshot reported no devices")
+
+    architectures = sorted(_compact(_architecture_for_device(d) for d in devices))
+    board_types = sorted(
+        _compact(
+            _value_from_paths(d, ("board_type", "type"))
+            or _value_from_paths(d.get("board_info", {}), ("board_type", "type"))
+            for d in devices
+        )
+    )
+    device_series = sorted(
+        _compact(
+            _value_from_paths(d, ("device_series", "series", "board_name", "product_name"))
+            or _value_from_paths(d.get("board_info", {}), ("device_series", "series", "board_name", "product_name"))
+            for d in devices
+        )
+    )
+    board_numbers = sorted(_compact(_board_identity(d) for d in devices))
+    coordinates = sorted(
+        _compact(
+            _value_from_paths(d, ("coords", "coordinate", "coordinates"))
+            or _value_from_paths(d.get("board_info", {}), ("coords", "coordinate", "coordinates"))
+            for d in devices
+        )
+    )
+
+    return {
+        "architecture": architectures or ["unknown"],
+        "board_types": board_types or ["unknown"],
+        "device_series": device_series or board_types or ["unknown"],
+        "device_count": len(devices),
+        "card_count": len(board_numbers) if board_numbers else len(devices),
+        "board_numbers": board_numbers,
+        "device_coordinates": coordinates,
+        "mesh_topology": _infer_mesh_topology(snapshot, len(devices), len(board_numbers) if board_numbers else len(devices)),
+    }
+
+
+def _devices_from_snapshot(snapshot: dict[str, Any]) -> list[dict[str, Any]]:
+    raw = snapshot.get("device_info", [])
+    if isinstance(raw, dict):
+        values = raw.values()
+    elif isinstance(raw, list):
+        values = raw
+    else:
+        values = []
+    return [item for item in values if isinstance(item, dict)]
+
+
+def _value_from_paths(data: Any, names: Iterable[str]) -> str | None:
+    if not isinstance(data, dict):
+        return None
+    lowered = {str(key).lower(): value for key, value in data.items()}
+    for name in names:
+        value = lowered.get(name.lower())
+        if value is not None:
+            text = str(value).strip()
+            if text and text.upper() != "N/A":
+                return text
+    return None
+
+
+def _board_identity(device: dict[str, Any]) -> str | None:
+    names = (
+        "board_number",
+        "board_id",
+        "board_serial",
+        "serial_number",
+        "serial",
+        "card_id",
+        "card_serial",
+    )
+    return _value_from_paths(device, names) or _value_from_paths(device.get("board_info", {}), names)
+
+
+def _architecture_for_device(device: dict[str, Any]) -> str | None:
+    explicit = _value_from_paths(device, ("arch", "architecture", "device_arch"))
+    if explicit:
+        return explicit
+
+    text = " ".join(
+        str(value).lower()
+        for key, value in _walk_key_values(device)
+        if any(token in key.lower() for token in ("arch", "board_type", "device_series", "product", "name"))
+    )
+    if any(token in text for token in ("wormhole", "n150", "n300", "nebula")):
+        return "wormhole_b0"
+    if any(token in text for token in ("blackhole", "p100", "p150", "p300", "p150a", "p150b")):
+        return "blackhole"
+    return None
+
+
+def _compact(values: Iterable[str | None]) -> set[str]:
+    return {value for value in values if value}
+
+
+def _infer_mesh_topology(snapshot: dict[str, Any], device_count: int, card_count: int) -> str:
+    reported = []
+    for key, value in _walk_key_values(snapshot):
+        lowered = key.lower()
+        if any(token in lowered for token in ("mesh", "topology", "cluster")):
+            text = str(value).strip()
+            if text and text.upper() != "N/A" and len(text) < 160:
+                reported.append(f"{key}={text}")
+    if reported:
+        return "; ".join(sorted(set(reported))[:6])
+
+    series_text = " ".join(
+        str(value).lower()
+        for key, value in _walk_key_values(snapshot)
+        if key.lower() in {"device_series", "series", "board_type", "board_name", "product_name"}
+    )
+    if any(token in series_text for token in ("quietbox", "loudbox", " t3000", "qb", "lb")):
+        return "mesh system inferred from board series"
+    if card_count >= 32 or device_count >= 32:
+        return "galaxy-scale system inferred from device/card count"
+    if card_count in {4, 8}:
+        return f"{card_count}-card mesh inferred from card count; explicit topology not reported by tt-smi snapshot"
+    if card_count <= 1:
+        return "single-card/non-mesh inferred from card count"
+    return "unknown; not reported by tt-smi snapshot"
+
+
+def _walk_key_values(value: Any, prefix: str = "") -> Iterable[tuple[str, Any]]:
+    if isinstance(value, dict):
+        for key, nested in value.items():
+            name = f"{prefix}.{key}" if prefix else str(key)
+            yield from _walk_key_values(nested, name)
+    elif isinstance(value, list):
+        for index, nested in enumerate(value):
+            yield from _walk_key_values(nested, f"{prefix}[{index}]")
+    else:
+        yield prefix, value
+
+
+def run_ttnn_mlp_check(
+    *,
+    device_id: int,
+    runs: int,
+    pcc_threshold: float,
+    activation_width: int,
+    prefill_rows: int,
+    decode_rows: int,
+    intermediate_multiplier: int,
+    seed: int,
+) -> list[dict[str, Any]]:
+    try:
+        import torch
+        import ttnn
+    except ModuleNotFoundError as exc:
+        raise CheckError(f"missing Python dependency: {exc.name}") from exc
+
+    torch.manual_seed(seed)
+    device = None
+    try:
+        device = ttnn.open_device(device_id=device_id, trace_region_size=0)
+        results = []
+        for mode, rows in (("prefill", prefill_rows), ("decode", decode_rows)):
+            results.append(
+                _run_mlp_mode(
+                    torch=torch,
+                    ttnn=ttnn,
+                    device=device,
+                    mode=mode,
+                    rows=rows,
+                    activation_width=activation_width,
+                    intermediate_width=activation_width * intermediate_multiplier,
+                    runs=runs,
+                    pcc_threshold=pcc_threshold,
+                )
+            )
+        return results
+    finally:
+        if device is not None:
+            try:
+                ttnn.close_device(device)
+            except Exception as exc:
+                raise CheckError(f"ttnn.close_device failed: {type(exc).__name__}: {exc}") from exc
+
+
+def _run_mlp_mode(
+    *,
+    torch: Any,
+    ttnn: Any,
+    device: Any,
+    mode: str,
+    rows: int,
+    activation_width: int,
+    intermediate_width: int,
+    runs: int,
+    pcc_threshold: float,
+) -> dict[str, Any]:
+    shape = (1, 1, rows, activation_width)
+    x = (torch.randn(shape, dtype=torch.float32) * 0.5).to(torch.bfloat16)
+    w1 = torch.randn((activation_width, intermediate_width), dtype=torch.float32) / math.sqrt(activation_width)
+    w3 = torch.randn((activation_width, intermediate_width), dtype=torch.float32) / math.sqrt(activation_width)
+    w2 = torch.randn((intermediate_width, activation_width), dtype=torch.float32) / math.sqrt(intermediate_width)
+
+    tt_x = ttnn.from_torch(x, dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT, device=device)
+    tt_w1 = ttnn.from_torch(w1, dtype=ttnn.bfloat8_b, layout=ttnn.TILE_LAYOUT, device=device)
+    tt_w3 = ttnn.from_torch(w3, dtype=ttnn.bfloat8_b, layout=ttnn.TILE_LAYOUT, device=device)
+    tt_w2 = ttnn.from_torch(w2, dtype=ttnn.bfloat8_b, layout=ttnn.TILE_LAYOUT, device=device)
+
+    try:
+        qw1 = ttnn.to_torch(tt_w1).to(torch.float32)
+        qw3 = ttnn.to_torch(tt_w3).to(torch.float32)
+        qw2 = ttnn.to_torch(tt_w2).to(torch.float32)
+        reference = torch.matmul(torch.nn.functional.silu(torch.matmul(x.to(torch.float32), qw1)) * torch.matmul(x.to(torch.float32), qw3), qw2)
+
+        mode_start = time.perf_counter()
+        pcc_values = []
+
+        warmup_start = time.perf_counter()
+        warmup_output, _ = _ttnn_mlp_forward(ttnn, tt_x, tt_w1, tt_w3, tt_w2)
+        warmup_torch = ttnn.to_torch(warmup_output).to(torch.float32)
+        _deallocate(ttnn, warmup_output)
+        warmup_elapsed = time.perf_counter() - warmup_start
+        first_output = warmup_torch
+        pcc_values.append(_validate_mlp_output(torch, reference, warmup_torch, pcc_threshold, mode, "warmup", None))
+
+        _synchronize_device(ttnn, device)
+        trace_id = None
+        trace_output = None
+        trace_intermediates = ()
+        try:
+            capture_start = time.perf_counter()
+            trace_id = ttnn.begin_trace_capture(device, cq_id=0)
+            trace_output, trace_intermediates = _ttnn_mlp_forward(
+                ttnn, tt_x, tt_w1, tt_w3, tt_w2, keep_intermediates=True
+            )
+            ttnn.end_trace_capture(device, trace_id, cq_id=0)
+            capture_elapsed = time.perf_counter() - capture_start
+            _synchronize_device(ttnn, device)
+
+            capture_torch = ttnn.to_torch(trace_output).to(torch.float32)
+            pcc_values.append(
+                _validate_mlp_output(torch, reference, capture_torch, pcc_threshold, mode, "capture", first_output)
+            )
+
+            replay_start = time.perf_counter()
+            for run_index in range(runs):
+                ttnn.execute_trace(device, trace_id, cq_id=0, blocking=True)
+                output_torch = ttnn.to_torch(trace_output).to(torch.float32)
+                pcc_values.append(
+                    _validate_mlp_output(
+                        torch, reference, output_torch, pcc_threshold, mode, f"trace replay {run_index}", first_output
+                    )
+                )
+            replay_elapsed = time.perf_counter() - replay_start
+        finally:
+            if trace_id is not None:
+                _release_trace(ttnn, device, trace_id)
+            for tensor in (trace_output, *trace_intermediates):
+                _deallocate(ttnn, tensor)
+
+        unique_pcc_values = {f"{pcc:.12g}" for pcc in pcc_values}
+        if len(unique_pcc_values) != 1:
+            raise CheckError(f"{mode}: PCC values were not identical across runs: {sorted(unique_pcc_values)}")
+
+        return {
+            "mode": mode,
+            "shape": list(shape),
+            "runs": runs,
+            "execution": "trace",
+            "warmup_runs": 1,
+            "captured_runs": 1,
+            "pcc": pcc_values[0],
+            "pcc_threshold": pcc_threshold,
+            "outputs_identical": True,
+            "timing": {
+                "warmup_s": warmup_elapsed,
+                "capture_s": capture_elapsed,
+                "replay_s": replay_elapsed,
+                "total_s": time.perf_counter() - mode_start,
+            },
+        }
+    finally:
+        for tensor in (tt_x, tt_w1, tt_w3, tt_w2):
+            _deallocate(ttnn, tensor)
+
+
+def _ttnn_mlp_forward(
+    ttnn: Any, x: Any, w1: Any, w3: Any, w2: Any, *, keep_intermediates: bool = False
+) -> tuple[Any, tuple[Any, ...]]:
+    w1_out = ttnn.linear(x, w1, dtype=ttnn.bfloat16)
+    w3_out = ttnn.linear(x, w3, dtype=ttnn.bfloat16)
+    activated = ttnn.silu(w1_out)
+    hidden = ttnn.mul(activated, w3_out, dtype=ttnn.bfloat16)
+    output = ttnn.linear(hidden, w2, dtype=ttnn.bfloat16)
+
+    intermediates = (hidden, activated, w3_out, w1_out)
+    if keep_intermediates:
+        return output, intermediates
+    for tensor in intermediates:
+        _deallocate(ttnn, tensor)
+    return output, ()
+
+
+def _validate_mlp_output(
+    torch: Any,
+    reference: Any,
+    actual: Any,
+    pcc_threshold: float,
+    mode: str,
+    run_label: str,
+    expected_identical: Any | None,
+) -> float:
+    if not torch.isfinite(actual).all():
+        raise CheckError(f"{mode} {run_label}: TTNN output contains non-finite values")
+
+    pcc = _pearson_corrcoef(torch, reference, actual)
+    if pcc < pcc_threshold:
+        raise CheckError(f"{mode} {run_label}: PCC {pcc:.8f} < {pcc_threshold:.8f}")
+
+    if expected_identical is not None and not torch.equal(expected_identical, actual):
+        max_abs = torch.max(torch.abs(expected_identical - actual)).item()
+        differing = torch.count_nonzero(expected_identical != actual).item()
+        raise CheckError(
+            f"{mode} {run_label}: output differs from warmup "
+            f"({differing} elements differ, max_abs_diff={max_abs:.8g})"
+        )
+    return pcc
+
+
+def _synchronize_device(ttnn: Any, device: Any) -> None:
+    try:
+        ttnn.synchronize_device(device)
+    except AttributeError:
+        return
+
+
+def _release_trace(ttnn: Any, device: Any, trace_id: Any) -> None:
+    try:
+        ttnn.release_trace(device, trace_id)
+    except Exception:
+        pass
+
+
+def _deallocate(ttnn: Any, tensor: Any) -> None:
+    if tensor is None:
+        return
+    try:
+        ttnn.deallocate(tensor)
+    except Exception:
+        pass
+
+
+def _pearson_corrcoef(torch: Any, expected: Any, actual: Any) -> float:
+    expected_flat = expected.reshape(-1).to(torch.float64)
+    actual_flat = actual.reshape(-1).to(torch.float64)
+    if expected_flat.numel() != actual_flat.numel():
+        raise CheckError(f"shape mismatch: expected {tuple(expected.shape)}, actual {tuple(actual.shape)}")
+
+    expected_centered = expected_flat - torch.mean(expected_flat)
+    actual_centered = actual_flat - torch.mean(actual_flat)
+    denominator = torch.linalg.norm(expected_centered) * torch.linalg.norm(actual_centered)
+    if denominator.item() == 0.0:
+        return 1.0 if torch.equal(expected_flat, actual_flat) else 0.0
+    return (torch.dot(expected_centered, actual_centered) / denominator).item()
+
+
+def _format_human_result(result: dict[str, Any]) -> str:
+    system = result["system"]
+    mlp_by_mode = {item["mode"]: item for item in result["mlp"]}
+    prefill = mlp_by_mode["prefill"]
+    decode = mlp_by_mode["decode"]
+    runs = max(item["runs"] for item in result["mlp"])
+    cards = system["card_count"]
+    lines = [
+        "tt-check passed",
+        "",
+        "reset   ok",
+        f"system  {_join_short(system['architecture'])} | {_join_short(system['device_series'])} | {cards} {_plural(cards, 'card')} | {_short_topology(system['mesh_topology'])}",
+        f"mlp     prefill pcc {prefill['pcc']:.8f} | decode pcc {decode['pcc']:.8f} | trace x{runs}",
+        "ready.",
+    ]
+    return "\n".join(lines)
+
+
+def _join_short(values: list[str]) -> str:
+    return ",".join(values) if values else "unknown"
+
+
+def _plural(count: int, noun: str) -> str:
+    return noun if count == 1 else f"{noun}s"
+
+
+def _short_topology(topology: str) -> str:
+    lowered = topology.lower()
+    if lowered.startswith("single-card"):
+        return "single-card"
+    if "galaxy" in lowered:
+        return "galaxy"
+    if "4-card mesh" in lowered:
+        return "4-card mesh"
+    if "8-card mesh" in lowered:
+        return "8-card mesh"
+    if "mesh system" in lowered:
+        return "mesh"
+    if "unknown" in lowered:
+        return "topology unknown"
+    return topology
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tt_check-0.1.0/tests/test_cli.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import argparse
+import unittest
+
+from tt_check import cli
+
+
+class CliHelpersTest(unittest.TestCase):
+    def test_validate_runs(self) -> None:
+        args = argparse.Namespace(
+            runs=0,
+            pcc_threshold=0.99,
+            activation_width_per_device=1024,
+            prefill_rows=1024,
+            decode_rows=1,
+            intermediate_multiplier=4,
+        )
+
+        with self.assertRaisesRegex(cli.CheckError, "--runs must be >= 1"):
+            cli._validate_args(args)
+
+    def test_system_summary_from_blackhole_snapshot(self) -> None:
+        snapshot = {
+            "device_info": [
+                {
+                    "arch": "blackhole",
+                    "board_type": "p150b",
+                    "board_number": "0000041100000000",
+                }
+            ]
+        }
+
+        summary = cli.summarize_system(snapshot)
+
+        self.assertEqual(summary["architecture"], ["blackhole"])
+        self.assertEqual(summary["board_types"], ["p150b"])
+        self.assertEqual(summary["card_count"], 1)
+        self.assertIn("single-card", summary["mesh_topology"])
+
+    def test_human_result_mentions_trace(self) -> None:
+        result = {
+            "system": {
+                "architecture": ["blackhole"],
+                "device_series": ["p150b"],
+                "card_count": 1,
+                "mesh_topology": "single-card/non-mesh inferred from card count",
+            },
+            "mlp": [
+                {"mode": "prefill", "runs": 100, "pcc": 0.9998160161456406},
+                {"mode": "decode", "runs": 100, "pcc": 0.999871423156057},
+            ],
+        }
+
+        text = cli._format_human_result(result)
+
+        self.assertIn("trace x100", text)
+        self.assertIn("ready.", text)
+
+
+if __name__ == "__main__":
+    unittest.main()