tan-cli 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tan/__init__.py +0 -0
- tan/__main__.py +120 -0
- tan/cli.py +939 -0
- tan/command_table.py +265 -0
- tan/commands/__init__.py +8 -0
- tan/commands/bootstrap_cmd.py +4856 -0
- tan/commands/bootstrap_patches.py +164 -0
- tan/commands/bootstrap_toolchain_fetch.py +155 -0
- tan/commands/bootstrap_toolchain_pin.py +36 -0
- tan/commands/build/__init__.py +1 -0
- tan/commands/build/configure_inputs.py +277 -0
- tan/commands/build/execute.py +1852 -0
- tan/commands/build/host_python.py +59 -0
- tan/commands/build/link_stale.py +78 -0
- tan/commands/build/manifest.py +428 -0
- tan/commands/build/materialise.py +81 -0
- tan/commands/build/token_substitution.py +381 -0
- tan/commands/build/toolchain.py +482 -0
- tan/commands/build/user_defines_wipe.py +115 -0
- tan/commands/build/workspace_patches.py +65 -0
- tan/commands/build_cmd.py +2236 -0
- tan/commands/build_output.py +331 -0
- tan/commands/clean_cmd.py +1183 -0
- tan/commands/completion_cmd.py +786 -0
- tan/commands/debug_config_cmd.py +2425 -0
- tan/commands/diff_cmd.py +940 -0
- tan/commands/doctor_cmd.py +4767 -0
- tan/commands/examples_cmd.py +644 -0
- tan/commands/explain_cmd.py +1319 -0
- tan/commands/faultdecode_cmd.py +931 -0
- tan/commands/flash_atoc_guard.py +240 -0
- tan/commands/flash_cmd.py +5731 -0
- tan/commands/flash_mram_guard.py +150 -0
- tan/commands/flash_ram.py +578 -0
- tan/commands/flash_raw.py +583 -0
- tan/commands/generate_cmd.py +1456 -0
- tan/commands/image_cmd.py +753 -0
- tan/commands/init_cmd.py +1740 -0
- tan/commands/inspect_cmd.py +377 -0
- tan/commands/kconfig_cmd.py +734 -0
- tan/commands/model_cmd.py +1514 -0
- tan/commands/model_device_cmd.py +351 -0
- tan/commands/model_device_live.py +234 -0
- tan/commands/model_host_cmd.py +430 -0
- tan/commands/model_zoo_cmd.py +363 -0
- tan/commands/monitor_actions.py +125 -0
- tan/commands/monitor_cmd.py +626 -0
- tan/commands/monitor_session.py +505 -0
- tan/commands/new_som_cmd.py +1522 -0
- tan/commands/pinmux_cmd.py +568 -0
- tan/commands/presets_cmd.py +1123 -0
- tan/commands/probe_cmd.py +421 -0
- tan/commands/reset_cmd.py +354 -0
- tan/commands/reset_confirm.py +168 -0
- tan/commands/run_cmd.py +637 -0
- tan/commands/scaffold_cmd.py +490 -0
- tan/commands/sdk_cmd.py +1723 -0
- tan/commands/size_cmd.py +1020 -0
- tan/commands/support_bundle_cmd.py +1073 -0
- tan/commands/trace_cmd.py +341 -0
- tan/commands/validate_cmd.py +2009 -0
- tan/commands/west_forward_cmd.py +736 -0
- tan/commands/workspace_patch_check.py +296 -0
- tan/core/__init__.py +1 -0
- tan/core/alp_conf_pregen.py +100 -0
- tan/core/artifact_provenance.py +240 -0
- tan/core/atoc_guard.py +276 -0
- tan/core/atoc_guard_messages.py +232 -0
- tan/core/atoc_replacement.py +118 -0
- tan/core/atomic_write.py +178 -0
- tan/core/board_context.py +351 -0
- tan/core/board_diagnostic.py +111 -0
- tan/core/board_files.py +221 -0
- tan/core/board_validator.py +542 -0
- tan/core/board_validator_compat.py +196 -0
- tan/core/board_validator_run.py +129 -0
- tan/core/board_validator_schema.py +186 -0
- tan/core/board_validator_skew.py +46 -0
- tan/core/board_yaml_edit.py +295 -0
- tan/core/board_yaml_pos.py +118 -0
- tan/core/bootstrap.py +2572 -0
- tan/core/build_plan.py +625 -0
- tan/core/consent.py +104 -0
- tan/core/console_filter.py +245 -0
- tan/core/debug_launch.py +1803 -0
- tan/core/devicetree_lint.py +168 -0
- tan/core/dir_removal.py +229 -0
- tan/core/doctor_git.py +172 -0
- tan/core/doctor_libraries.py +253 -0
- tan/core/doctor_render.py +185 -0
- tan/core/doctor_scope.py +66 -0
- tan/core/doctor_setools.py +237 -0
- tan/core/doctor_stale.py +180 -0
- tan/core/document_guards.py +723 -0
- tan/core/dp_id.py +97 -0
- tan/core/elf_load.py +63 -0
- tan/core/error_catalog.py +171 -0
- tan/core/example_catalog.py +499 -0
- tan/core/example_facets.py +151 -0
- tan/core/faultdecode.py +521 -0
- tan/core/flash_plan.py +2501 -0
- tan/core/flow_d_report.py +381 -0
- tan/core/flow_d_uptime.py +164 -0
- tan/core/fs_confine.py +130 -0
- tan/core/global_flags.py +230 -0
- tan/core/host_python.py +210 -0
- tan/core/image_bundle.py +247 -0
- tan/core/inert.py +147 -0
- tan/core/jlink_binary.py +128 -0
- tan/core/jlink_probe.py +481 -0
- tan/core/jsonc_splice.py +404 -0
- tan/core/launch_provenance.py +257 -0
- tan/core/link_refusal.py +76 -0
- tan/core/metadata_schema.py +376 -0
- tan/core/model_check.py +227 -0
- tan/core/model_device.py +449 -0
- tan/core/model_doctor.py +198 -0
- tan/core/model_host.py +48 -0
- tan/core/model_list.py +112 -0
- tan/core/model_zoo.py +353 -0
- tan/core/module_template.py +340 -0
- tan/core/mram_link.py +87 -0
- tan/core/new_som_ospi.py +94 -0
- tan/core/os_class.py +159 -0
- tan/core/patch_paths.py +87 -0
- tan/core/pending.py +40 -0
- tan/core/plain_zephyr_plan.py +138 -0
- tan/core/plan_exec.py +373 -0
- tan/core/plan_tokens.py +600 -0
- tan/core/probe.py +155 -0
- tan/core/probe_plan.py +237 -0
- tan/core/proxy.py +168 -0
- tan/core/publish.py +31 -0
- tan/core/python_floor.py +178 -0
- tan/core/ram_run.py +694 -0
- tan/core/ram_watch.py +231 -0
- tan/core/raw_write.py +160 -0
- tan/core/reset_plan.py +56 -0
- tan/core/run.py +161 -0
- tan/core/scaffold.py +2203 -0
- tan/core/scaffold_selftest_identity.py +225 -0
- tan/core/sdk_default_registry.py +565 -0
- tan/core/sdk_discovery.py +996 -0
- tan/core/sdk_removal.py +465 -0
- tan/core/serial_actions.py +175 -0
- tan/core/serial_capture.py +185 -0
- tan/core/serial_url.py +45 -0
- tan/core/setools.py +608 -0
- tan/core/setools_scratch.py +243 -0
- tan/core/shapes.py +194 -0
- tan/core/size.py +610 -0
- tan/core/som_buildability.py +204 -0
- tan/core/som_schema_version.py +62 -0
- tan/core/subprocess_env.py +201 -0
- tan/core/sysbuild_image.py +94 -0
- tan/core/system_manifest.py +827 -0
- tan/core/text_layout.py +73 -0
- tan/core/timestamp.py +182 -0
- tan/core/tool_lookup.py +383 -0
- tan/core/toolchain_pin.py +117 -0
- tan/core/toolchain_provision.py +1224 -0
- tan/core/trusted_path.py +134 -0
- tan/core/uboot_breakin.py +154 -0
- tan/core/uri_reference.py +415 -0
- tan/core/user_defines.py +152 -0
- tan/core/venv.py +332 -0
- tan/core/west_patches.py +143 -0
- tan/core/west_workspace_refusal.py +159 -0
- tan/core/zephyr_env.py +124 -0
- tan/env.py +185 -0
- tan/envelope.py +647 -0
- tan/exit_codes.py +12 -0
- tan/lazy_typer.py +185 -0
- tan/model/__init__.py +2 -0
- tan/model/_gen_fixture.py +132 -0
- tan/model/adapters/__init__.py +116 -0
- tan/model/adapters/cpu.py +26 -0
- tan/model/adapters/deepx.py +94 -0
- tan/model/adapters/drpai.py +242 -0
- tan/model/adapters/ethos_u.py +951 -0
- tan/model/adapters/executorch.py +55 -0
- tan/model/analyze.py +694 -0
- tan/model/build.py +304 -0
- tan/model/check.py +682 -0
- tan/model/manifest.py +625 -0
- tan/model/measure.py +216 -0
- tan/model/npu_ops_validate.py +130 -0
- tan/model/package.py +125 -0
- tan/model/perf.py +635 -0
- tan/model/perf_apply.py +721 -0
- tan/model/prep.py +193 -0
- tan/model/sram_fit.py +502 -0
- tan/model/targets.py +374 -0
- tan/model/tensorio.py +287 -0
- tan/net.py +73 -0
- tan/output_format.py +213 -0
- tan/planner/__init__.py +219 -0
- tan/planner/aperture.py +202 -0
- tan/planner/buildplan.py +1092 -0
- tan/planner/camera_owner.py +192 -0
- tan/planner/cameras.py +132 -0
- tan/planner/carveout.py +755 -0
- tan/planner/cli.py +274 -0
- tan/planner/headers.py +315 -0
- tan/planner/kconfig.py +2435 -0
- tan/planner/kconfig_symbols.py +473 -0
- tan/planner/libraries.py +677 -0
- tan/planner/link_target.py +237 -0
- tan/planner/linux_ownership.py +216 -0
- tan/planner/loader.py +1525 -0
- tan/planner/manifest.py +116 -0
- tan/planner/memory.py +161 -0
- tan/planner/memregion.py +56 -0
- tan/planner/models.py +533 -0
- tan/planner/orchestrator.py +874 -0
- tan/planner/ownership.py +203 -0
- tan/planner/partition.py +1160 -0
- tan/planner/paths.py +32 -0
- tan/planner/plain_slice.py +139 -0
- tan/planner/project_emit/__init__.py +60 -0
- tan/planner/project_emit/bom_netlist.py +375 -0
- tan/planner/project_emit/dts.py +580 -0
- tan/planner/project_emit/hw_info.py +221 -0
- tan/planner/project_emit/native_sim.py +136 -0
- tan/planner/project_emit/west_libs.py +213 -0
- tan/planner/project_loader.py +264 -0
- tan/planner/sdk_capability.py +203 -0
- tan/planner/sdk_compat.py +395 -0
- tan/planner/secure.py +350 -0
- tan/planner/slugs.py +394 -0
- tan/planner/som_metadata.py +325 -0
- tan/planner/strict_loaders.py +131 -0
- tan/planner/template.py +1037 -0
- tan/planner/template_pins.py +535 -0
- tan/planner/template_rewrite.py +714 -0
- tan/planner/topology.py +208 -0
- tan/planner/validate.py +365 -0
- tan/planner/whole_device_alias.py +67 -0
- tan/planner/zephyr_board.py +3510 -0
- tan/planner_cli.py +97 -0
- tan/planner_emit.py +617 -0
- tan/planner_root.py +119 -0
- tan/soc_ref.py +58 -0
- tan/templates/__init__.py +34 -0
- tan/templates/vendored/MANIFEST.md +1862 -0
- tan/templates/vendored/diagnostics/E1M-AEN801/CMakeLists.txt +8 -0
- tan/templates/vendored/diagnostics/E1M-AEN801/README.md +112 -0
- tan/templates/vendored/diagnostics/E1M-AEN801/board.yaml +38 -0
- tan/templates/vendored/diagnostics/E1M-AEN801/boards/native_sim_native_64.conf +11 -0
- tan/templates/vendored/diagnostics/E1M-AEN801/boards/native_sim_native_64.overlay +29 -0
- tan/templates/vendored/diagnostics/E1M-AEN801/prj.conf +5 -0
- tan/templates/vendored/diagnostics/E1M-AEN801/src/main.c +266 -0
- tan/templates/vendored/diagnostics/E1M-AEN801/testcase.yaml +53 -0
- tan/templates/vendored/diagnostics/E1M-V2N101/CMakeLists.txt +8 -0
- tan/templates/vendored/diagnostics/E1M-V2N101/README.md +112 -0
- tan/templates/vendored/diagnostics/E1M-V2N101/board.yaml +38 -0
- tan/templates/vendored/diagnostics/E1M-V2N101/boards/native_sim_native_64.conf +11 -0
- tan/templates/vendored/diagnostics/E1M-V2N101/boards/native_sim_native_64.overlay +29 -0
- tan/templates/vendored/diagnostics/E1M-V2N101/prj.conf +5 -0
- tan/templates/vendored/diagnostics/E1M-V2N101/src/main.c +266 -0
- tan/templates/vendored/diagnostics/E1M-V2N101/testcase.yaml +53 -0
- tan/templates/vendored/edge-ai/E1M-AEN801/CMakeLists.txt +10 -0
- tan/templates/vendored/edge-ai/E1M-AEN801/README.md +75 -0
- tan/templates/vendored/edge-ai/E1M-AEN801/board.yaml +38 -0
- tan/templates/vendored/edge-ai/E1M-AEN801/prj.conf +8 -0
- tan/templates/vendored/edge-ai/E1M-AEN801/src/cold_chain.c +371 -0
- tan/templates/vendored/edge-ai/E1M-AEN801/src/cold_chain.h +178 -0
- tan/templates/vendored/edge-ai/E1M-AEN801/src/main.c +373 -0
- tan/templates/vendored/edge-ai/E1M-AEN801/testcase.yaml +62 -0
- tan/templates/vendored/edge-ai/E1M-V2N101/CMakeLists.txt +10 -0
- tan/templates/vendored/edge-ai/E1M-V2N101/README.md +75 -0
- tan/templates/vendored/edge-ai/E1M-V2N101/board.yaml +37 -0
- tan/templates/vendored/edge-ai/E1M-V2N101/prj.conf +8 -0
- tan/templates/vendored/edge-ai/E1M-V2N101/src/cold_chain.c +371 -0
- tan/templates/vendored/edge-ai/E1M-V2N101/src/cold_chain.h +178 -0
- tan/templates/vendored/edge-ai/E1M-V2N101/src/main.c +373 -0
- tan/templates/vendored/edge-ai/E1M-V2N101/testcase.yaml +62 -0
- tan/templates/vendored/iot/E1M-AEN801/CMakeLists.txt +14 -0
- tan/templates/vendored/iot/E1M-AEN801/README.md +103 -0
- tan/templates/vendored/iot/E1M-AEN801/board.yaml +69 -0
- tan/templates/vendored/iot/E1M-AEN801/prj.conf +15 -0
- tan/templates/vendored/iot/E1M-AEN801/src/cc3501e_bridge.c +196 -0
- tan/templates/vendored/iot/E1M-AEN801/src/cc3501e_bridge.h +173 -0
- tan/templates/vendored/iot/E1M-AEN801/src/main.c +262 -0
- tan/templates/vendored/iot/E1M-AEN801/testcase.yaml +68 -0
- tan/templates/vendored/minimal/E1M-AEN801/CMakeLists.txt +8 -0
- tan/templates/vendored/minimal/E1M-AEN801/README.md +108 -0
- tan/templates/vendored/minimal/E1M-AEN801/board.yaml +32 -0
- tan/templates/vendored/minimal/E1M-AEN801/prj.conf +8 -0
- tan/templates/vendored/minimal/E1M-AEN801/src/main.c +111 -0
- tan/templates/vendored/minimal/E1M-AEN801/testcase.yaml +49 -0
- tan/templates/vendored/minimal/E1M-V2N101/CMakeLists.txt +8 -0
- tan/templates/vendored/minimal/E1M-V2N101/README.md +108 -0
- tan/templates/vendored/minimal/E1M-V2N101/board.yaml +32 -0
- tan/templates/vendored/minimal/E1M-V2N101/prj.conf +8 -0
- tan/templates/vendored/minimal/E1M-V2N101/src/main.c +111 -0
- tan/templates/vendored/minimal/E1M-V2N101/testcase.yaml +49 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/CMakeLists.txt +8 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/README.md +120 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/board.yaml +57 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/boards/native_sim_native_64.overlay +40 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/peer/CMakeLists.txt +13 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/peer/main.c +171 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/peer/prj.conf +17 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/peer/testcase.yaml +23 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/prj.conf +22 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/src/main.c +185 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/testcase.yaml +26 -0
- tan/templates/vendored/sensor/E1M-AEN801/CMakeLists.txt +8 -0
- tan/templates/vendored/sensor/E1M-AEN801/README.md +112 -0
- tan/templates/vendored/sensor/E1M-AEN801/board.yaml +57 -0
- tan/templates/vendored/sensor/E1M-AEN801/boards/native_sim_native_64.conf +10 -0
- tan/templates/vendored/sensor/E1M-AEN801/boards/native_sim_native_64.overlay +28 -0
- tan/templates/vendored/sensor/E1M-AEN801/prj.conf +5 -0
- tan/templates/vendored/sensor/E1M-AEN801/src/main.c +186 -0
- tan/templates/vendored/sensor/E1M-AEN801/testcase.yaml +52 -0
- tan/templates/vendored/sensor/E1M-V2N101/CMakeLists.txt +8 -0
- tan/templates/vendored/sensor/E1M-V2N101/README.md +112 -0
- tan/templates/vendored/sensor/E1M-V2N101/board.yaml +57 -0
- tan/templates/vendored/sensor/E1M-V2N101/boards/native_sim_native_64.conf +10 -0
- tan/templates/vendored/sensor/E1M-V2N101/boards/native_sim_native_64.overlay +28 -0
- tan/templates/vendored/sensor/E1M-V2N101/prj.conf +5 -0
- tan/templates/vendored/sensor/E1M-V2N101/src/main.c +186 -0
- tan/templates/vendored/sensor/E1M-V2N101/testcase.yaml +52 -0
- tan/version.py +102 -0
- tan_cli-0.7.0.dist-info/METADATA +125 -0
- tan_cli-0.7.0.dist-info/RECORD +330 -0
- tan_cli-0.7.0.dist-info/WHEEL +5 -0
- tan_cli-0.7.0.dist-info/entry_points.txt +2 -0
- tan_cli-0.7.0.dist-info/top_level.txt +1 -0
tan/model/perf.py
ADDED
|
@@ -0,0 +1,635 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Manufacturer-precomputed model perf points -- the READER half of tier 2.
|
|
3
|
+
|
|
4
|
+
alp-sdk publishes bench-measured perf points under `metadata/model_perf/`
|
|
5
|
+
(`metadata/schemas/model-perf-v1.schema.json`), exactly as it already publishes
|
|
6
|
+
op-support tables under `metadata/npu_ops/`. This module finds the ONE point
|
|
7
|
+
that describes a given measurement identity, so `tan model check` can answer
|
|
8
|
+
"does this model run on this module, and how fast" for a customer holding
|
|
9
|
+
NEITHER the NPU toolchain NOR the silicon -- the middle fidelity tier between
|
|
10
|
+
the always-offline static screen (`tan.model.analyze`, `basis:
|
|
11
|
+
"static-screen"`) and the customer's own `--exact` compile (`tan.model.check`,
|
|
12
|
+
`basis: "compiled"`).
|
|
13
|
+
|
|
14
|
+
THE AUTHORITATIVE SCHEMA (tan-cli#1115). Read from alp-sdk directly, not
|
|
15
|
+
inferred from one fixture: `metadata/schemas/model-perf-v1.schema.json`
|
|
16
|
+
(`additionalProperties: false` at every level -- a published point can carry
|
|
17
|
+
NO field this schema doesn't name), cross-checked by `scripts/
|
|
18
|
+
validate_metadata.py`'s `_check_model_perf_semantics()`, and demonstrated by
|
|
19
|
+
`tests/fixtures/model_perf/e1m_aen801_ethos_u55_hp.yaml`. A valid point is:
|
|
20
|
+
|
|
21
|
+
schema_version: 1
|
|
22
|
+
sku: E1M-... # top level, not nested
|
|
23
|
+
hw_rev: rN # top level, not nested
|
|
24
|
+
model: {name, src_sha, format} # src_sha is the 64-hex match key
|
|
25
|
+
target: {backend, accel_config, core, compiler_version}
|
|
26
|
+
vela: {system_config, memory_mode} # required only when target.backend
|
|
27
|
+
# == "ethos_u"; absent otherwise
|
|
28
|
+
perf: {req_sram_kib, arena_bytes, latency_ms?: {mean, p50, p95, stdev?, runs}}
|
|
29
|
+
capture: {date, operator, bench_id, notes?}
|
|
30
|
+
|
|
31
|
+
BEFORE THIS RECONCILIATION, `PerfPoint`/`_identity()`/`read_perf_point()`
|
|
32
|
+
recognised a DIFFERENT, never-published shape -- a top-level `stance` field,
|
|
33
|
+
a `measured_on` block, and a `measured` block carrying `npu_ops`/`cpu_ops`
|
|
34
|
+
op-placement counts. None of that exists in alp-sdk's real schema, has never
|
|
35
|
+
existed in its git history (`git log -- metadata/schemas/model-perf-v1.schema.json`
|
|
36
|
+
is one commit, `9b466018` (alp-sdk #1884, Refs #1520), and it always looked
|
|
37
|
+
like the shape above), and `additionalProperties: false` means it never can
|
|
38
|
+
without a new schema version alp-sdk would have to author and this reader
|
|
39
|
+
would have to re-reconcile against. So the old fields are DEAD, not
|
|
40
|
+
renamed-with-a-gap, and are deleted rather than kept unpopulated:
|
|
41
|
+
|
|
42
|
+
* `stance` had no real counterpart at all -- deleted outright. Publication
|
|
43
|
+
under `metadata/model_perf/` (vs. `tests/fixtures/model_perf/`, see the
|
|
44
|
+
fixture rule below) is alp-sdk's own way of saying "this is a real
|
|
45
|
+
measurement"; there is no in-document flag for it.
|
|
46
|
+
* `measured_on.{sku,hw_rev,core,backend,accel_config}` map onto top-level
|
|
47
|
+
`sku`/`hw_rev` and `target.{core,backend,accel_config}`.
|
|
48
|
+
* `measured.{npu_ops,cpu_ops}` are GENUINELY DEAD: no field anywhere in the
|
|
49
|
+
real schema records a per-operator NPU/CPU placement split. A bench point
|
|
50
|
+
can state that a model fits in a given SRAM/arena budget and (once a
|
|
51
|
+
timing harness exists) how fast it ran -- the schema's own `perf`
|
|
52
|
+
description says the first capture campaign "can record fit/SRAM alone
|
|
53
|
+
until a timing harness exists" -- but not what fraction of its operators
|
|
54
|
+
the compiler placed on the NPU. `tan.model.perf_apply` no longer asks a
|
|
55
|
+
bench point for a placement percentage; see that module's docstring for
|
|
56
|
+
what a matched point can and cannot claim as a result.
|
|
57
|
+
* `measured.{arena_bytes,req_sram_kib,runs}` map onto `perf.arena_bytes`,
|
|
58
|
+
`perf.req_sram_kib`, `perf.latency_ms.runs`.
|
|
59
|
+
* `measured.{latency_ms_mean,latency_ms_p95}` map onto
|
|
60
|
+
`perf.latency_ms.{mean,p95}` (which also carries `p50`/`stdev`, not
|
|
61
|
+
represented in the old shape at all).
|
|
62
|
+
* `toolchain.{name,version}` map onto ONE field, `target.compiler_version`
|
|
63
|
+
(a single string, e.g. `"vela 4.1.0"` or `"passthrough"` for a backend
|
|
64
|
+
with no real compile step) -- there is no separate name/version pair on
|
|
65
|
+
the wire. `PerfPoint.compiler_version` carries it verbatim; a caller that
|
|
66
|
+
needs just the leading tool name (to narrow by which toolchain a backend
|
|
67
|
+
is benched with, the way `find_perf_points`' own `toolchain` query field
|
|
68
|
+
always has) gets it via `_toolchain_name()` below, not a second stored
|
|
69
|
+
field -- there is only one string to lie about.
|
|
70
|
+
* `toolchain.{system_config,memory_mode}` map onto `vela.{system_config,
|
|
71
|
+
memory_mode}` -- renamed `PerfPoint.vela_system_config`/
|
|
72
|
+
`vela_memory_mode` to match the wire block's own name (`vela`, not
|
|
73
|
+
`toolchain`: only ever populated for an `ethos_u` point, because only
|
|
74
|
+
vela has this profile concept at all).
|
|
75
|
+
* `capture.reference` doesn't exist; the real citation field is
|
|
76
|
+
`capture.bench_id` ("which physical bench/rig captured this point").
|
|
77
|
+
`PerfPoint.capture_reference` is renamed `capture_bench_id` to match.
|
|
78
|
+
* `notes` (top-level in the old shape) is `capture.notes` in the real one --
|
|
79
|
+
renamed `PerfPoint.notes` to `capture_notes`.
|
|
80
|
+
* `model.{slug,size_bytes,source}` don't exist; the real `model` block is
|
|
81
|
+
`{name, src_sha, format}` -- `PerfPoint.model_slug`/`model_sha256` are
|
|
82
|
+
renamed `model_name`/`model_src_sha`, and `model_size_bytes`/
|
|
83
|
+
`model_source` are deleted outright (no wire field backs them, and
|
|
84
|
+
nothing in tan ever read them).
|
|
85
|
+
|
|
86
|
+
WHAT THIS READER REFUSES TO MATCH IS ITS WHOLE VALUE. A perf point is an exact
|
|
87
|
+
number a customer sizes hardware from precisely because it says "bench", so a
|
|
88
|
+
point describing a NEARBY identity is worse than no point at all:
|
|
89
|
+
|
|
90
|
+
* **No "closest model".** Part of the match key is `model.src_sha` -- the
|
|
91
|
+
exact bytes -- never `model.name`, which is a human label two different
|
|
92
|
+
byte-sequences can share. Re-quantize a model and its old point stops
|
|
93
|
+
applying.
|
|
94
|
+
* **No "same accelerator family".** `ethos-u85-256` is not `ethos-u85-128`
|
|
95
|
+
and not `ethos-u55-256`; `accel_config` is compared as an exact string.
|
|
96
|
+
* **No "same module, different revision".** An r1 and an r2 measurement of
|
|
97
|
+
one model on one target are different measurements (alp-sdk `f724d3e4`),
|
|
98
|
+
so `hw_rev` is a REQUIRED query field: a caller that cannot say which
|
|
99
|
+
revision it holds gets nothing rather than the other revision's number.
|
|
100
|
+
* **No "same die, different core".** An A-cluster and an M-class inference
|
|
101
|
+
of one model differ by orders of magnitude, and on Ethos-U the U55 paired
|
|
102
|
+
to the HP core is not the one paired to the HE core.
|
|
103
|
+
* **No fixture data, ever.** alp-sdk's own validator draws the fixture-vs-
|
|
104
|
+
published line by PATH, not by a document key
|
|
105
|
+
(`scripts/validate_metadata.py`'s `_MODEL_PERF_FIXTURE_MARKER = "_fixture"`,
|
|
106
|
+
checked against `path.parent.name`/`path.name` -- i.e. the SKU-directory
|
|
107
|
+
segment and the filename, the two components that are actually PART of
|
|
108
|
+
the published-tree naming): "a fixture belongs under
|
|
109
|
+
`tests/fixtures/model_perf/`, never in the published `metadata/
|
|
110
|
+
model_perf/` tree." This reader adopts that exact rule rather than
|
|
111
|
+
inventing a document-level one, for the reason alp-sdk's own comment
|
|
112
|
+
gives -- a real point's BODY has no reason to ever say "I am a fixture";
|
|
113
|
+
only where it SITS says so.
|
|
114
|
+
* **No ambiguity resolved by preference.** More than one point left standing
|
|
115
|
+
by the match key is genuine ambiguity: the toolchain PROFILE
|
|
116
|
+
(`vela_system_config`/`vela_memory_mode`) is part of a point's identity
|
|
117
|
+
but deliberately NOT part of the key, because a customer with no
|
|
118
|
+
toolchain cannot state a profile. Picking one would author a preference
|
|
119
|
+
the data does not express -- and the collision that put the profile into
|
|
120
|
+
a point's identity hash (alp-sdk `f724d3e4`) left the DRAM-backed profile
|
|
121
|
+
as the survivor, which is exactly measured and describes the wrong
|
|
122
|
+
machine. `find_perf_points` therefore hands the caller ALL of them and
|
|
123
|
+
lets a caller with a silicon fact decide (`tan.model.check` uses the SoC
|
|
124
|
+
spec's own `npu_toolchain` profile); `find_perf_point` hands back `None`.
|
|
125
|
+
|
|
126
|
+
Every refusal above returns `None`, SILENTLY. Absence of a point means "we have
|
|
127
|
+
not benched this", never "it does not fit" -- the same rule `npu-ops-v1` states
|
|
128
|
+
for a missing op table, and the reason a missing `metadata/model_perf/`
|
|
129
|
+
directory is handled here as the ORDINARY case rather than an error: alp-sdk
|
|
130
|
+
ships that tree EMPTY (`metadata/model_perf/README.md`: "This directory is
|
|
131
|
+
intentionally empty... a perf point comes off real Alp Lab bench silicon or it
|
|
132
|
+
does not exist, and none has been captured yet"), and a consumer that treated
|
|
133
|
+
its absence as exceptional would break `tan model check` on every SDK checkout
|
|
134
|
+
in existence today.
|
|
135
|
+
|
|
136
|
+
Nothing in this module raises. A malformed, truncated or schema-violating point
|
|
137
|
+
is simply not a match: the producing side (alp-sdk's
|
|
138
|
+
`scripts/validate_metadata.py` + `tests/scripts/test_model_perf_metadata.py`)
|
|
139
|
+
is where a bad point is made LOUD, and a consumer that also failed loudly would
|
|
140
|
+
turn a metadata defect in another repository into a broken `tan model check`
|
|
141
|
+
for a customer who cannot fix it.
|
|
142
|
+
|
|
143
|
+
`coverage_from_placement` lives here rather than in `tan.model.check` because
|
|
144
|
+
it is THE one function in tan that may return the word `"fits"` off a real,
|
|
145
|
+
measured per-operator placement -- today that is only a real compile's own
|
|
146
|
+
summary (`tan.model.check._report_from_vela_compile`'s vela "CPU/NPU
|
|
147
|
+
operators = N" line); a bench perf point carries no such summary (see the
|
|
148
|
+
DEAD FIELDS note above) and no longer calls this function at all.
|
|
149
|
+
|
|
150
|
+
THE PATH: `metadata/model_perf/<SKU>/<hash>.yaml`, EXACTLY TWO PATH SEGMENTS
|
|
151
|
+
BELOW ROOT (tan-cli#1115's second, initially-missed half). alp-sdk's own
|
|
152
|
+
`scripts/validate_metadata.py::_collect_model_perf_files()` enforces this
|
|
153
|
+
shape on the PUBLISHING side ("a real point is exactly <SKU>/<hash>.yaml (2
|
|
154
|
+
segments); a file this shallow or this deep is never opened by the schema/
|
|
155
|
+
semantic passes and validates nothing") and refuses anything else outright --
|
|
156
|
+
there is no `<sku>/<target>/` subdirectory the way an earlier, unreconciled
|
|
157
|
+
version of this reader searched. `find_perf_points` below globs
|
|
158
|
+
NON-RECURSIVELY (`Path.glob`, never `Path.rglob`) directly under
|
|
159
|
+
`root/<sku>/`, which is what PINS the segment count: a point nested one
|
|
160
|
+
level deeper (a stray `<sku>/_fixture/<hash>.yaml`, or any other
|
|
161
|
+
subdirectory) is invisible to this reader by construction, not by a count
|
|
162
|
+
that could quietly drift back to matching a wrong depth --
|
|
163
|
+
`tests/model/test_perf.py::test_a_point_nested_one_level_too_deep_is_never_found`
|
|
164
|
+
proves it.
|
|
165
|
+
|
|
166
|
+
ONE PARSER, BOTH FORMATS. `read_perf_point` parses with `yaml.safe_load`, not
|
|
167
|
+
`json.loads`: alp-sdk publishes its tier-2 perf points as YAML, while every
|
|
168
|
+
document `tan`'s own test suite writes for this reader is JSON. Valid JSON is
|
|
169
|
+
valid YAML, so ONE parser reads both, with no format-sniff or extension check
|
|
170
|
+
of its own to drift from what either side actually writes -- a reader pinned
|
|
171
|
+
to `json.loads` in one place while the real documents are YAML is exactly how
|
|
172
|
+
tan-cli#1105 happened (the fixture discovery predicate in `tests/conftest.py`
|
|
173
|
+
globbed for `*.json` against a `*.yaml` file and permanently skipped, hiding
|
|
174
|
+
that this reader could not have parsed it either). `yaml` is imported inside
|
|
175
|
+
`read_perf_point`, not at module scope, because `tan.model.perf` sits on
|
|
176
|
+
`tan.cli`'s eager import graph (via `tan/commands/model_cmd.py`) and PyYAML is
|
|
177
|
+
single-use here (tan-cli#810, enforced by `tests/gates/test_cli_import_is_lean.py`).
|
|
178
|
+
|
|
179
|
+
YAML 1.1 TYPING IS ALREADY GUARDED. `yaml.safe_load` gives an unquoted scalar
|
|
180
|
+
an implicit type -- `2026-08-16` becomes a `datetime.date`, `5.1` becomes a
|
|
181
|
+
`float` -- and `_text()` below is deliberately `str`-strict, so an identity
|
|
182
|
+
field a hand-authored point left unquoted does not raise, it silently drops
|
|
183
|
+
the WHOLE point (the same `None` as a missing field). Deliberately not
|
|
184
|
+
"fixed" by coercing non-`str` scalars back to text: this reader's entire
|
|
185
|
+
design is to refuse rather than guess, and a value YAML re-typed out from
|
|
186
|
+
under the schema's own `"type": "string"` is exactly the ambiguity that
|
|
187
|
+
design refuses on. alp-sdk's own `validate_metadata.py` schema pass already
|
|
188
|
+
rejects an unquoted date/version at the SOURCE, so every point that actually
|
|
189
|
+
reaches a customer's `metadata_root` -- published or the `tests/fixtures/`
|
|
190
|
+
synthetic -- was quoted correctly before it got there; the residual risk is a
|
|
191
|
+
document nobody has validated yet, which is refused, not misread.
|
|
192
|
+
"""
|
|
193
|
+
from __future__ import annotations
|
|
194
|
+
|
|
195
|
+
from dataclasses import dataclass
|
|
196
|
+
from pathlib import Path
|
|
197
|
+
|
|
198
|
+
#: The published tree's directory name under a `metadata/` root. Named once
|
|
199
|
+
#: here so a consumer never has to spell it, and so `has_perf_points` and
|
|
200
|
+
#: `find_perf_point` cannot drift apart on it.
|
|
201
|
+
_PERF_DIRNAME = "model_perf"
|
|
202
|
+
|
|
203
|
+
#: alp-sdk's OWN fixture convention, adopted verbatim (see the module
|
|
204
|
+
#: docstring's fixture-rule note): `scripts/validate_metadata.py`'s
|
|
205
|
+
#: `_MODEL_PERF_FIXTURE_MARKER`. Checked against the SKU-directory segment
|
|
206
|
+
#: and the filename -- the two path components that are actually part of the
|
|
207
|
+
#: published-tree naming -- never the full absolute path, which could contain
|
|
208
|
+
#: this substring by coincidence (a developer's own checkout directory name)
|
|
209
|
+
#: with nothing to do with the tree's own content.
|
|
210
|
+
_FIXTURE_MARKER = "_fixture"
|
|
211
|
+
|
|
212
|
+
#: The only `schema_version` this reader understands. A future v2 alp-sdk
|
|
213
|
+
#: might publish under new field names this reader has never seen -- refused
|
|
214
|
+
#: exactly like any other unrecognised document, rather than half-parsed
|
|
215
|
+
#: under v1's field mapping.
|
|
216
|
+
_SCHEMA_VERSION = 1
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
@dataclass(frozen=True)
|
|
220
|
+
class PerfPoint:
|
|
221
|
+
"""One bench-measured perf point, as published.
|
|
222
|
+
|
|
223
|
+
Identity fields are all required and always populated -- a document
|
|
224
|
+
missing any of them is not a point and never becomes one of these. Every
|
|
225
|
+
OTHER field is OPTIONAL and `None` means "not measured" (or, for
|
|
226
|
+
`vela_system_config`/`vela_memory_mode`, "not applicable to this
|
|
227
|
+
backend"), never zero: the schema omits a figure nobody measured rather
|
|
228
|
+
than zero-filling it, so a zero here is a MEASURED zero.
|
|
229
|
+
"""
|
|
230
|
+
|
|
231
|
+
path: Path # where it was read from, for traceability
|
|
232
|
+
sku: str # top-level `sku`
|
|
233
|
+
hw_rev: str # top-level `hw_rev`, e.g. "r2"
|
|
234
|
+
core: str # target.core, e.g. "m55_hp"
|
|
235
|
+
backend: str # target.backend
|
|
236
|
+
accel_config: str # target.accel_config; "" when N/A
|
|
237
|
+
compiler_version: str # target.compiler_version, e.g. "vela 4.1.0"
|
|
238
|
+
model_name: str # model.name -- a LABEL, never a match key
|
|
239
|
+
model_src_sha: str # model.src_sha -- part of the match key
|
|
240
|
+
model_format: str # model.format -- "tflite" | "onnx"
|
|
241
|
+
capture_date: str # capture.date
|
|
242
|
+
capture_operator: str # capture.operator
|
|
243
|
+
capture_bench_id: str # capture.bench_id -- WHICH bench/rig captured this,
|
|
244
|
+
# e.g. "e1m-aen-evk-01" -- a RIG identifier,
|
|
245
|
+
# never a capture artefact's own citation
|
|
246
|
+
# The MEMORY PROFILE the figures describe. `None` for a backend that has
|
|
247
|
+
# none (only `ethos_u` points ever carry a `vela:` block at all), never
|
|
248
|
+
# for the reason vela itself matters: invoked flagless, `ethos-u-vela`
|
|
249
|
+
# silently picks a DRAM-backed default per accelerator and the arena
|
|
250
|
+
# figures then describe THAT profile rather than the module. A point that
|
|
251
|
+
# did not record the profile is exactly measured and describes an
|
|
252
|
+
# unknown machine -- so a consumer that HAS them must show them, or it
|
|
253
|
+
# re-creates on the reading side the ambiguity the schema closed on the
|
|
254
|
+
# writing side.
|
|
255
|
+
vela_system_config: str | None = None
|
|
256
|
+
vela_memory_mode: str | None = None
|
|
257
|
+
req_sram_kib: int | None = None
|
|
258
|
+
arena_bytes: int | None = None
|
|
259
|
+
latency_ms_mean: float | None = None
|
|
260
|
+
latency_ms_p50: float | None = None
|
|
261
|
+
latency_ms_p95: float | None = None
|
|
262
|
+
latency_ms_stdev: float | None = None
|
|
263
|
+
latency_runs: int | None = None
|
|
264
|
+
capture_notes: str | None = None
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _toolchain_name(compiler_version: str) -> str:
|
|
268
|
+
"""The leading whitespace-delimited token of @compiler_version -- `"vela"`
|
|
269
|
+
out of `"vela 4.1.0"`, `"drp-ai_tvm"` out of `"drp-ai_tvm 3.0.1"`,
|
|
270
|
+
`"DX-COM"` out of `"DX-COM 2.3.0"` (alp-sdk `scripts/alp_model/adapters/
|
|
271
|
+
ethos_u.py::_vela_version`, `drpai.py::_compiler_version`,
|
|
272
|
+
`deepx.py::_dxcom_version` -- the exact producers of this field, measured
|
|
273
|
+
tan-cli#1115 PR review), or the whole string when it carries no space
|
|
274
|
+
(e.g. `"passthrough"`, the schema's own example for a backend with no
|
|
275
|
+
real compile step; also each adapter's own OWN degraded fallback when a
|
|
276
|
+
version could not be probed -- `"vela"`, `"drp-ai_tvm"` unchanged, but
|
|
277
|
+
`deepx.py::_dxcom_version` falls back to the differently-spelled bare
|
|
278
|
+
`"dxcom"` (no hyphen) rather than `"DX-COM"` with one, which this
|
|
279
|
+
function -- a plain leading-token split -- cannot reconcile with the
|
|
280
|
+
normal, versioned form. That fallback is intentionally not chased: a real
|
|
281
|
+
bench capture is expected to run against a working, version-probable
|
|
282
|
+
toolchain install, and `deepx_dxm1`'s failure spelling differs by more
|
|
283
|
+
than case (`.split(None, 1)`, not `.casefold()`, is what would be needed
|
|
284
|
+
to normalise whitespace here, and neither operation makes `"dxcom"` and
|
|
285
|
+
`"DX-COM"` the same token).
|
|
286
|
+
|
|
287
|
+
`.split(None, 1)`, not `.split(" ", 1)`: a literal single-space split
|
|
288
|
+
treats a tab or a run of spaces between the name and the version as part
|
|
289
|
+
of the name, silently refusing a real document over incidental
|
|
290
|
+
whitespace; `None` splits on ANY whitespace run, matching `str.strip()`'s
|
|
291
|
+
own notion of a separator.
|
|
292
|
+
|
|
293
|
+
`compiler_version` is the ONE wire field; there is no separate stored
|
|
294
|
+
toolchain-name attribute to drift from it. Used only to narrow
|
|
295
|
+
`find_perf_points`' own `toolchain` query field (a bare name a caller
|
|
296
|
+
already knows, e.g. `tan.model.perf_apply._PERF_TOOLCHAIN`'s values)
|
|
297
|
+
against a point actually captured by that tool, without inventing a
|
|
298
|
+
second identity field the schema doesn't have. Compared case-foldedly by
|
|
299
|
+
the caller (`find_perf_points`, below) -- alp-sdk's own three real
|
|
300
|
+
producers do not agree on case (`vela`/`drp-ai_tvm` lowercase, `DX-COM`
|
|
301
|
+
upper) and nothing about that spelling is part of a measurement's
|
|
302
|
+
identity, the same reasoning `model.src_sha`'s own hex-case tolerance
|
|
303
|
+
already applies elsewhere in this module."""
|
|
304
|
+
return compiler_version.split(None, 1)[0]
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def coverage_from_placement(npu_ops: int | None, cpu_ops: int | None) -> str | None:
|
|
308
|
+
"""The `npu_coverage` a REAL per-operator placement supports -- and the
|
|
309
|
+
ONLY function in tan that may return `"fits"`.
|
|
310
|
+
|
|
311
|
+
`"fits"` only at 100% NPU placement (no CPU operator, and at least one
|
|
312
|
+
operator placed); `"cpu-only"` at 0%; `"partial"` in between. `None` when
|
|
313
|
+
there is no placement to read at all -- either count absent, or both zero
|
|
314
|
+
(an unreadable summary, not an all-NPU one). A caller handed `None` must
|
|
315
|
+
NOT manufacture a verdict from it: "nothing said it fell back to the CPU"
|
|
316
|
+
is not evidence that it did not.
|
|
317
|
+
|
|
318
|
+
TODAY'S ONLY CALLER is `tan.model.check._report_from_vela_compile`, off a
|
|
319
|
+
real vela compile's own "CPU/NPU operators = N" summary
|
|
320
|
+
(`blob.npu_op_count`/`cpu_op_count`). It is deliberately NOT called from
|
|
321
|
+
`tan.model.perf_apply` any more (tan-cli#1115): alp-sdk's real
|
|
322
|
+
`model-perf-v1` schema carries no per-operator placement counts on a bench
|
|
323
|
+
point at all (see this module's own docstring, "DEAD FIELDS"), so there is
|
|
324
|
+
no bench-side npu_ops/cpu_ops pair to feed this with. A matched bench
|
|
325
|
+
point WITHHOLDS `npu_coverage` to `"undetermined"` unless the report it is
|
|
326
|
+
rebasing already carries a REAL placement from a prior `--exact` compile
|
|
327
|
+
(`basis == "compiled"`) -- it never inherits a static screen's name-level
|
|
328
|
+
estimate as if it were a measurement -- see
|
|
329
|
+
`tan.model.perf_apply.apply_perf_point`/`_perf_point_report`."""
|
|
330
|
+
if npu_ops is None or cpu_ops is None:
|
|
331
|
+
return None
|
|
332
|
+
if npu_ops + cpu_ops == 0:
|
|
333
|
+
return None
|
|
334
|
+
if cpu_ops == 0:
|
|
335
|
+
return "fits"
|
|
336
|
+
return "cpu-only" if npu_ops == 0 else "partial"
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def perf_points_root(metadata_root: Path) -> Path:
|
|
340
|
+
"""`metadata/model_perf/` under @metadata_root. May not exist."""
|
|
341
|
+
return Path(metadata_root) / _PERF_DIRNAME
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def has_perf_points(metadata_root: Path) -> bool:
|
|
345
|
+
"""Does this SDK checkout publish a perf-point tree at all?
|
|
346
|
+
|
|
347
|
+
A cheap `is_dir()` a caller uses to skip work it would otherwise do for
|
|
348
|
+
nothing -- notably hashing the model source, which `find_perf_point` needs
|
|
349
|
+
and which is pure waste on the (currently universal) checkout publishing no
|
|
350
|
+
points. NOT a precondition: `find_perf_point` makes the same check itself
|
|
351
|
+
and answers `None`, so a caller that skips this is correct, just slower.
|
|
352
|
+
"""
|
|
353
|
+
return perf_points_root(metadata_root).is_dir()
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
#: `.yaml` is what alp-sdk actually publishes; `.json` is additionally
|
|
357
|
+
#: discovered because tan's own test suite writes JSON synthetic points
|
|
358
|
+
#: (`tests/model/test_check.py::_write_perf_point`'s own default, and most of
|
|
359
|
+
#: `tests/model/test_perf.py`) -- an ASYMMETRIC exception to the `.yml`
|
|
360
|
+
#: reasoning right below, named honestly rather than silently: `.yml` is
|
|
361
|
+
#: excluded because alp-sdk's own `_collect_model_perf_files` rejects it too
|
|
362
|
+
#: ("extension `.yml` is not `.yaml`"), so a point spelled that way is
|
|
363
|
+
#: invisible on the publishing side already and matching it here would let
|
|
364
|
+
#: tan accept a shape alp-sdk's own gate would reject -- but NOTHING alp-sdk
|
|
365
|
+
#: publishes is ever `.json` either, so `.json` support exists purely to let
|
|
366
|
+
#: tan's OWN synthetic fixtures skip a `yaml.safe_dump` round-trip, not
|
|
367
|
+
#: because a real document could be shaped that way. The consequence is real
|
|
368
|
+
#: and acknowledged, not hidden: most of this file's own test coverage
|
|
369
|
+
#: exercises the `.json` branch, and `test_a_yaml_suffixed_point_is_
|
|
370
|
+
#: discovered_not_just_a_json_one` (this module's own suite) plus
|
|
371
|
+
#: `tests/model/test_check.py::test_a_matched_point_is_found_at_a_yaml_
|
|
372
|
+
#: suffixed_filename_too` are what actually prove the `.yaml` branch this
|
|
373
|
+
#: reader depends on in production.
|
|
374
|
+
_PERF_POINT_GLOBS = ("*.yaml", "*.json")
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _perf_point_files(sku_dir: Path) -> list[Path]:
|
|
378
|
+
"""Every file directly under @sku_dir this reader will attempt, in path
|
|
379
|
+
order. Non-recursive (`Path.glob`, never `Path.rglob`) BY CONSTRUCTION --
|
|
380
|
+
see the module docstring's "THE PATH" note for why that is what pins the
|
|
381
|
+
two-segments-below-root shape rather than merely documenting it."""
|
|
382
|
+
return sorted({p for pattern in _PERF_POINT_GLOBS for p in sku_dir.glob(pattern)})
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _as_dict(value: object) -> dict:
|
|
386
|
+
"""@value if it is a dict, else `{}` -- the safe "read a nested block
|
|
387
|
+
that might be missing or wrong-typed" idiom this module reuses for every
|
|
388
|
+
OPTIONAL block (`vela`, `perf`, `perf.latency_ms`)."""
|
|
389
|
+
return value if isinstance(value, dict) else {}
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _text(doc: dict, key: str) -> str | None:
|
|
393
|
+
value = doc.get(key)
|
|
394
|
+
return value if isinstance(value, str) else None
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _count(doc: dict, key: str) -> int | None:
|
|
398
|
+
value = doc.get(key)
|
|
399
|
+
# `bool` is an `int` subclass and `True` is not a measurement.
|
|
400
|
+
if isinstance(value, bool) or not isinstance(value, int):
|
|
401
|
+
return None
|
|
402
|
+
return value if value >= 0 else None
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def _millis(doc: dict, key: str) -> float | None:
|
|
406
|
+
value = doc.get(key)
|
|
407
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
408
|
+
return None
|
|
409
|
+
# `>= 0`, not `> 0`: a `0` here is a MEASURED zero (`PerfPoint`'s own
|
|
410
|
+
# docstring), the same reading `_count` above already gives a `0` op
|
|
411
|
+
# count. `> 0` silently turned a real "0.0 ms" latency into "not
|
|
412
|
+
# measured" -- indistinguishable from an omitted key on the wire.
|
|
413
|
+
return float(value) if value >= 0 else None
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _identity(doc: dict) -> dict | None:
|
|
417
|
+
"""@doc's identity fields as plain values, or `None` when any is missing
|
|
418
|
+
or of the wrong type.
|
|
419
|
+
|
|
420
|
+
`accel_config` is REQUIRED-AND-MAY-BE-EMPTY and the distinction is
|
|
421
|
+
load-bearing: `""` is a backend that HAS no accel config, while an absent
|
|
422
|
+
key is a point that failed to record one. `_text` separates them -- `""`
|
|
423
|
+
is a `str` and survives the emptiness test below, `None` is the
|
|
424
|
+
missing-key answer and refuses.
|
|
425
|
+
|
|
426
|
+
`capture.{date,operator,bench_id}` are folded in here, not left to
|
|
427
|
+
`read_perf_point` as optional extras: a number nobody can trace back to a
|
|
428
|
+
run is not reproducible, and a perf point is worth exactly what its
|
|
429
|
+
reproducibility is (mirrors the pre-reconciliation reader's own stance,
|
|
430
|
+
carried forward under the real field names)."""
|
|
431
|
+
model, target, capture = (doc.get(k) for k in ("model", "target", "capture"))
|
|
432
|
+
if not all(isinstance(block, dict) for block in (model, target, capture)):
|
|
433
|
+
return None
|
|
434
|
+
out = {
|
|
435
|
+
"sku": _text(doc, "sku"),
|
|
436
|
+
"hw_rev": _text(doc, "hw_rev"),
|
|
437
|
+
"core": _text(target, "core"),
|
|
438
|
+
"backend": _text(target, "backend"),
|
|
439
|
+
"compiler_version": _text(target, "compiler_version"),
|
|
440
|
+
"model_name": _text(model, "name"),
|
|
441
|
+
"model_src_sha": _text(model, "src_sha"),
|
|
442
|
+
"model_format": _text(model, "format"),
|
|
443
|
+
"capture_date": _text(capture, "date"),
|
|
444
|
+
"capture_operator": _text(capture, "operator"),
|
|
445
|
+
"capture_bench_id": _text(capture, "bench_id"),
|
|
446
|
+
}
|
|
447
|
+
accel_config = _text(target, "accel_config")
|
|
448
|
+
if not all(out.values()) or accel_config is None:
|
|
449
|
+
return None
|
|
450
|
+
return {**out, "accel_config": accel_config}
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def read_perf_point(path: Path) -> PerfPoint | None:
|
|
454
|
+
"""Parse ONE perf-point file, or `None` for anything this reader will not
|
|
455
|
+
treat as a measurement.
|
|
456
|
+
|
|
457
|
+
Parsed with `yaml.safe_load` -- see the module docstring's "ONE PARSER,
|
|
458
|
+
BOTH FORMATS" note for why, and why the import is deferred to here rather
|
|
459
|
+
than module scope.
|
|
460
|
+
|
|
461
|
+
`None` covers: unreadable bytes, a document this parser cannot make sense
|
|
462
|
+
of at all, a document that is not a mapping, a path naming alp-sdk's own
|
|
463
|
+
`_fixture` marker, an unrecognised `schema_version`, and any document
|
|
464
|
+
missing an identity field. Never raises -- see the module docstring on why
|
|
465
|
+
a consumer stays quiet where the producing repository's own gate is loud.
|
|
466
|
+
"""
|
|
467
|
+
import yaml # noqa: PLC0415 (tan-cli#810: single-use, kept off `tan --version`)
|
|
468
|
+
|
|
469
|
+
if _FIXTURE_MARKER in path.parent.name or _FIXTURE_MARKER in path.name:
|
|
470
|
+
return None
|
|
471
|
+
try:
|
|
472
|
+
# `ValueError` is required alongside `yaml.YAMLError`, not redundant
|
|
473
|
+
# with it: `path.read_text(encoding="utf-8")` raises
|
|
474
|
+
# `UnicodeDecodeError` (a `ValueError` subclass) on non-UTF-8 bytes,
|
|
475
|
+
# BEFORE the parser ever runs, and `yaml.YAMLError` does not cover it.
|
|
476
|
+
doc = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
477
|
+
except (OSError, ValueError, yaml.YAMLError):
|
|
478
|
+
return None
|
|
479
|
+
if not isinstance(doc, dict) or doc.get("schema_version") != _SCHEMA_VERSION:
|
|
480
|
+
return None
|
|
481
|
+
identity = _identity(doc)
|
|
482
|
+
if identity is None:
|
|
483
|
+
return None
|
|
484
|
+
vela = _as_dict(doc.get("vela"))
|
|
485
|
+
perf = _as_dict(doc.get("perf"))
|
|
486
|
+
latency = _as_dict(perf.get("latency_ms"))
|
|
487
|
+
capture = doc["capture"] # already proven a dict by `_identity`
|
|
488
|
+
return PerfPoint(
|
|
489
|
+
path=path, **identity,
|
|
490
|
+
vela_system_config=_text(vela, "system_config"),
|
|
491
|
+
vela_memory_mode=_text(vela, "memory_mode"),
|
|
492
|
+
req_sram_kib=_count(perf, "req_sram_kib"),
|
|
493
|
+
arena_bytes=_count(perf, "arena_bytes"),
|
|
494
|
+
latency_ms_mean=_millis(latency, "mean"),
|
|
495
|
+
latency_ms_p50=_millis(latency, "p50"),
|
|
496
|
+
latency_ms_p95=_millis(latency, "p95"),
|
|
497
|
+
latency_ms_stdev=_millis(latency, "stdev"),
|
|
498
|
+
latency_runs=_count(latency, "runs"),
|
|
499
|
+
capture_notes=_text(capture, "notes"),
|
|
500
|
+
)
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
# ---------------------------------------------------------------------------
|
|
504
|
+
# THE MATCH RULE (alp-sdk `f724d3e4`), and the three things about it that are
|
|
505
|
+
# DECISIONS rather than mechanics.
|
|
506
|
+
#
|
|
507
|
+
# THE KEY IS EIGHT FIELDS: sku + hw_rev + core + backend + accel_config +
|
|
508
|
+
# model.src_sha + toolchain name (derived from target.compiler_version) +
|
|
509
|
+
# target.compiler_version itself. Every one of them changes the number, which
|
|
510
|
+
# is why every one of them is also part of alp-sdk's own identity hash
|
|
511
|
+
# (`_model_perf_identity_hash()`) -- a component left out of that hash is one
|
|
512
|
+
# on which a second measurement silently overwrites the first.
|
|
513
|
+
#
|
|
514
|
+
# THREE OF THOSE EIGHT ARE OPTIONAL *QUERY* FIELDS, because a caller does not
|
|
515
|
+
# always hold the fact. `hw_rev` is not one of them: a caller that cannot say
|
|
516
|
+
# which module revision it has gets NOTHING, since serving an r2 measurement to
|
|
517
|
+
# an r1 customer is precisely the "describes a different machine" failure. But
|
|
518
|
+
# `core` and `compiler_version` (the exact string, not just the toolchain name)
|
|
519
|
+
# may be left unstated -- there is no local toolchain to read a version off in
|
|
520
|
+
# the tier-2 premise, and nothing ties a declared model to a core on a
|
|
521
|
+
# multi-core module -- and an unstated field is simply not narrowed on BY THIS
|
|
522
|
+
# FUNCTION: within `find_perf_points` alone, it is never guessed, and it never
|
|
523
|
+
# widens a match, it only leaves MORE points standing here.
|
|
524
|
+
#
|
|
525
|
+
# THAT IS NOT THE WHOLE PIPELINE'S GUARANTEE. ONE caller-side check narrows
|
|
526
|
+
# `core` further than this function's own query shape can
|
|
527
|
+
# (`tan.model.perf_apply._topology_core_ids`): a point's core must exist in
|
|
528
|
+
# @sku's OWN `topology:` map at all, UNCONDITIONALLY, for EVERY backend on
|
|
529
|
+
# EVERY SKU, whether or not the accelerator being screened pairs to a core --
|
|
530
|
+
# this is what refuses `m55_hp` on a V2N SKU (a real Ensemble core, not
|
|
531
|
+
# a V2N one) or `cortex_potato` anywhere. `core` narrows to @target's own
|
|
532
|
+
# `paired_core` ONLY through this function's QUERY parameter (above) -- a
|
|
533
|
+
# declared fact about the SAME accelerator being screened -- and nothing
|
|
534
|
+
# wider; where @target declares no pairing of its own, `core` stays unnarrowed
|
|
535
|
+
# past the topology check.
|
|
536
|
+
#
|
|
537
|
+
# THE TOOLCHAIN PROFILE IS IDENTITY BUT NOT KEY. `vela_system_config`/
|
|
538
|
+
# `vela_memory_mode` are part of a point's real identity (both required by
|
|
539
|
+
# alp-sdk whenever `target.backend == "ethos_u"`), but a customer with no
|
|
540
|
+
# toolchain cannot state a profile, so requiring one would make tier 2
|
|
541
|
+
# unusable for exactly the person it exists for. The consequence is that a
|
|
542
|
+
# lookup can legitimately leave MORE THAN ONE point standing -- one model, one
|
|
543
|
+
# module, one core, one toolchain, captured under two different vela profiles
|
|
544
|
+
# -- and those describe DIFFERENT MACHINES. `find_perf_points` returns all of
|
|
545
|
+
# them and picks nothing; `find_perf_point` returns `None`. A caller holding a
|
|
546
|
+
# silicon fact may of course choose (`tan.model.perf_apply` narrows on the
|
|
547
|
+
# profile the SoC spec's own `npu_toolchain` block declares for the part,
|
|
548
|
+
# BEFORE trusting even a single standing point, not only as a tiebreak once
|
|
549
|
+
# two or more survive) -- but that decision belongs where the silicon facts
|
|
550
|
+
# are, not here.
|
|
551
|
+
#
|
|
552
|
+
# THE PATH IS AN INDEX, THE BODY IS THE TRUTH. `<root>/<sku>/` narrows the
|
|
553
|
+
# search cheaply, but every field is then re-checked against the document
|
|
554
|
+
# itself, so a file whose body disagrees with where it sits is not found
|
|
555
|
+
# here. alp-sdk's own validator refuses to publish one, and this reader does
|
|
556
|
+
# not have to trust that it did. Nothing here parses the FILENAME -- alp-sdk's
|
|
557
|
+
# own filename is an opaque content hash of the identity fields
|
|
558
|
+
# (`_model_perf_identity_hash()`), and a consumer that re-derived identity
|
|
559
|
+
# from it would be reading the index instead of the record.
|
|
560
|
+
# ---------------------------------------------------------------------------
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
def find_perf_points(*, sku: str, backend: str, accel_config: str,
|
|
564
|
+
model_sha256: str, toolchain: str, hw_rev: str | None,
|
|
565
|
+
metadata_root: Path, core: str | None = None,
|
|
566
|
+
compiler_version: str | None = None) -> list[PerfPoint]:
|
|
567
|
+
"""EVERY published point this identity leaves standing, in path order.
|
|
568
|
+
|
|
569
|
+
`[]` for no match, ONE entry for the ordinary case, and MORE THAN ONE when
|
|
570
|
+
the caller could not narrow far enough to separate two real measurements --
|
|
571
|
+
see the block comment above. A caller that cannot choose between them must
|
|
572
|
+
fall through to the next fidelity tier, never take `[0]`.
|
|
573
|
+
|
|
574
|
+
@sku, @backend, @accel_config (exact string; `""` for a backend that has
|
|
575
|
+
none) and @model_sha256 are always compared. The digest is compared
|
|
576
|
+
case-folded -- hex case is a spelling of one value, not a different one --
|
|
577
|
+
and a @model_sha256 that is not a full 64-hex digest is refused outright,
|
|
578
|
+
because a truncated or prefix digest is the near-miss that would let two
|
|
579
|
+
models share one measurement.
|
|
580
|
+
|
|
581
|
+
@toolchain is the toolchain NAME (`vela`, `drp-ai_tvm`, `DX-COM` --
|
|
582
|
+
alp-sdk's own real producer strings, tan-cli#1115 PR review; NOT
|
|
583
|
+
`dxcom`/`translator`, an earlier, unverified guess that matched nothing a
|
|
584
|
+
real document could ever carry), compared CASE-FOLDEDLY against the
|
|
585
|
+
LEADING token of the point's own `compiler_version` (there is no separate
|
|
586
|
+
name field on the wire -- see `_toolchain_name`, whose own docstring
|
|
587
|
+
explains why case-folding and not the whole story). It comes from the
|
|
588
|
+
caller rather than being derived from @backend, so this function never
|
|
589
|
+
invents one for a backend nobody has benched.
|
|
590
|
+
|
|
591
|
+
@hw_rev is required-but-nullable and `None` matches NOTHING: see above.
|
|
592
|
+
@core narrows only when given. @compiler_version, when given, narrows on
|
|
593
|
+
the EXACT wire string (e.g. `"vela 4.1.0"`), not just the tool name --
|
|
594
|
+
today's only real caller (`tan.model.perf_apply`) never passes it, because
|
|
595
|
+
there is no local toolchain in the tier-2 premise to read a version off.
|
|
596
|
+
"""
|
|
597
|
+
root = perf_points_root(metadata_root)
|
|
598
|
+
if not root.is_dir():
|
|
599
|
+
return [] # the ordinary case, not an error
|
|
600
|
+
# No separate `hw_rev is None: return []` early return here:
|
|
601
|
+
# `point.hw_rev == hw_rev` below already refuses on its own --
|
|
602
|
+
# `PerfPoint.hw_rev` is a REQUIRED, always-non-empty field (`_identity`'s
|
|
603
|
+
# own `all(out.values())` gate), so it can never equal a `None` query
|
|
604
|
+
# value.
|
|
605
|
+
wanted = model_sha256.strip().lower()
|
|
606
|
+
if len(wanted) != 64 or any(c not in "0123456789abcdef" for c in wanted):
|
|
607
|
+
return []
|
|
608
|
+
sku_dir = root / sku
|
|
609
|
+
if not sku_dir.is_dir():
|
|
610
|
+
return []
|
|
611
|
+
return [
|
|
612
|
+
point for point in (read_perf_point(p) for p in _perf_point_files(sku_dir))
|
|
613
|
+
if point is not None
|
|
614
|
+
and point.sku == sku
|
|
615
|
+
and point.hw_rev == hw_rev
|
|
616
|
+
and point.backend == backend
|
|
617
|
+
and point.accel_config == accel_config
|
|
618
|
+
and point.model_src_sha.lower() == wanted
|
|
619
|
+
and _toolchain_name(point.compiler_version).casefold() == toolchain.casefold()
|
|
620
|
+
and (core is None or point.core == core)
|
|
621
|
+
and (compiler_version is None or point.compiler_version == compiler_version)
|
|
622
|
+
]
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
def find_perf_point(**query) -> PerfPoint | None:
|
|
626
|
+
"""`find_perf_points`' answer when it is UNAMBIGUOUS -- the sole standing
|
|
627
|
+
point, or `None`.
|
|
628
|
+
|
|
629
|
+
`None` folds "no match" and "more than one point standing" into one silent
|
|
630
|
+
answer on purpose: a caller with no way to choose between two measurements
|
|
631
|
+
of two different machines must do the same thing in both cases, which is
|
|
632
|
+
fall through. A caller that CAN choose calls `find_perf_points` and applies
|
|
633
|
+
its own, explicit, sourced rule."""
|
|
634
|
+
matches = find_perf_points(**query)
|
|
635
|
+
return matches[0] if len(matches) == 1 else None
|