tan-cli 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tan/__init__.py +0 -0
- tan/__main__.py +120 -0
- tan/cli.py +939 -0
- tan/command_table.py +265 -0
- tan/commands/__init__.py +8 -0
- tan/commands/bootstrap_cmd.py +4856 -0
- tan/commands/bootstrap_patches.py +164 -0
- tan/commands/bootstrap_toolchain_fetch.py +155 -0
- tan/commands/bootstrap_toolchain_pin.py +36 -0
- tan/commands/build/__init__.py +1 -0
- tan/commands/build/configure_inputs.py +277 -0
- tan/commands/build/execute.py +1852 -0
- tan/commands/build/host_python.py +59 -0
- tan/commands/build/link_stale.py +78 -0
- tan/commands/build/manifest.py +428 -0
- tan/commands/build/materialise.py +81 -0
- tan/commands/build/token_substitution.py +381 -0
- tan/commands/build/toolchain.py +482 -0
- tan/commands/build/user_defines_wipe.py +115 -0
- tan/commands/build/workspace_patches.py +65 -0
- tan/commands/build_cmd.py +2236 -0
- tan/commands/build_output.py +331 -0
- tan/commands/clean_cmd.py +1183 -0
- tan/commands/completion_cmd.py +786 -0
- tan/commands/debug_config_cmd.py +2425 -0
- tan/commands/diff_cmd.py +940 -0
- tan/commands/doctor_cmd.py +4767 -0
- tan/commands/examples_cmd.py +644 -0
- tan/commands/explain_cmd.py +1319 -0
- tan/commands/faultdecode_cmd.py +931 -0
- tan/commands/flash_atoc_guard.py +240 -0
- tan/commands/flash_cmd.py +5731 -0
- tan/commands/flash_mram_guard.py +150 -0
- tan/commands/flash_ram.py +578 -0
- tan/commands/flash_raw.py +583 -0
- tan/commands/generate_cmd.py +1456 -0
- tan/commands/image_cmd.py +753 -0
- tan/commands/init_cmd.py +1740 -0
- tan/commands/inspect_cmd.py +377 -0
- tan/commands/kconfig_cmd.py +734 -0
- tan/commands/model_cmd.py +1514 -0
- tan/commands/model_device_cmd.py +351 -0
- tan/commands/model_device_live.py +234 -0
- tan/commands/model_host_cmd.py +430 -0
- tan/commands/model_zoo_cmd.py +363 -0
- tan/commands/monitor_actions.py +125 -0
- tan/commands/monitor_cmd.py +626 -0
- tan/commands/monitor_session.py +505 -0
- tan/commands/new_som_cmd.py +1522 -0
- tan/commands/pinmux_cmd.py +568 -0
- tan/commands/presets_cmd.py +1123 -0
- tan/commands/probe_cmd.py +421 -0
- tan/commands/reset_cmd.py +354 -0
- tan/commands/reset_confirm.py +168 -0
- tan/commands/run_cmd.py +637 -0
- tan/commands/scaffold_cmd.py +490 -0
- tan/commands/sdk_cmd.py +1723 -0
- tan/commands/size_cmd.py +1020 -0
- tan/commands/support_bundle_cmd.py +1073 -0
- tan/commands/trace_cmd.py +341 -0
- tan/commands/validate_cmd.py +2009 -0
- tan/commands/west_forward_cmd.py +736 -0
- tan/commands/workspace_patch_check.py +296 -0
- tan/core/__init__.py +1 -0
- tan/core/alp_conf_pregen.py +100 -0
- tan/core/artifact_provenance.py +240 -0
- tan/core/atoc_guard.py +276 -0
- tan/core/atoc_guard_messages.py +232 -0
- tan/core/atoc_replacement.py +118 -0
- tan/core/atomic_write.py +178 -0
- tan/core/board_context.py +351 -0
- tan/core/board_diagnostic.py +111 -0
- tan/core/board_files.py +221 -0
- tan/core/board_validator.py +542 -0
- tan/core/board_validator_compat.py +196 -0
- tan/core/board_validator_run.py +129 -0
- tan/core/board_validator_schema.py +186 -0
- tan/core/board_validator_skew.py +46 -0
- tan/core/board_yaml_edit.py +295 -0
- tan/core/board_yaml_pos.py +118 -0
- tan/core/bootstrap.py +2572 -0
- tan/core/build_plan.py +625 -0
- tan/core/consent.py +104 -0
- tan/core/console_filter.py +245 -0
- tan/core/debug_launch.py +1803 -0
- tan/core/devicetree_lint.py +168 -0
- tan/core/dir_removal.py +229 -0
- tan/core/doctor_git.py +172 -0
- tan/core/doctor_libraries.py +253 -0
- tan/core/doctor_render.py +185 -0
- tan/core/doctor_scope.py +66 -0
- tan/core/doctor_setools.py +237 -0
- tan/core/doctor_stale.py +180 -0
- tan/core/document_guards.py +723 -0
- tan/core/dp_id.py +97 -0
- tan/core/elf_load.py +63 -0
- tan/core/error_catalog.py +171 -0
- tan/core/example_catalog.py +499 -0
- tan/core/example_facets.py +151 -0
- tan/core/faultdecode.py +521 -0
- tan/core/flash_plan.py +2501 -0
- tan/core/flow_d_report.py +381 -0
- tan/core/flow_d_uptime.py +164 -0
- tan/core/fs_confine.py +130 -0
- tan/core/global_flags.py +230 -0
- tan/core/host_python.py +210 -0
- tan/core/image_bundle.py +247 -0
- tan/core/inert.py +147 -0
- tan/core/jlink_binary.py +128 -0
- tan/core/jlink_probe.py +481 -0
- tan/core/jsonc_splice.py +404 -0
- tan/core/launch_provenance.py +257 -0
- tan/core/link_refusal.py +76 -0
- tan/core/metadata_schema.py +376 -0
- tan/core/model_check.py +227 -0
- tan/core/model_device.py +449 -0
- tan/core/model_doctor.py +198 -0
- tan/core/model_host.py +48 -0
- tan/core/model_list.py +112 -0
- tan/core/model_zoo.py +353 -0
- tan/core/module_template.py +340 -0
- tan/core/mram_link.py +87 -0
- tan/core/new_som_ospi.py +94 -0
- tan/core/os_class.py +159 -0
- tan/core/patch_paths.py +87 -0
- tan/core/pending.py +40 -0
- tan/core/plain_zephyr_plan.py +138 -0
- tan/core/plan_exec.py +373 -0
- tan/core/plan_tokens.py +600 -0
- tan/core/probe.py +155 -0
- tan/core/probe_plan.py +237 -0
- tan/core/proxy.py +168 -0
- tan/core/publish.py +31 -0
- tan/core/python_floor.py +178 -0
- tan/core/ram_run.py +694 -0
- tan/core/ram_watch.py +231 -0
- tan/core/raw_write.py +160 -0
- tan/core/reset_plan.py +56 -0
- tan/core/run.py +161 -0
- tan/core/scaffold.py +2203 -0
- tan/core/scaffold_selftest_identity.py +225 -0
- tan/core/sdk_default_registry.py +565 -0
- tan/core/sdk_discovery.py +996 -0
- tan/core/sdk_removal.py +465 -0
- tan/core/serial_actions.py +175 -0
- tan/core/serial_capture.py +185 -0
- tan/core/serial_url.py +45 -0
- tan/core/setools.py +608 -0
- tan/core/setools_scratch.py +243 -0
- tan/core/shapes.py +194 -0
- tan/core/size.py +610 -0
- tan/core/som_buildability.py +204 -0
- tan/core/som_schema_version.py +62 -0
- tan/core/subprocess_env.py +201 -0
- tan/core/sysbuild_image.py +94 -0
- tan/core/system_manifest.py +827 -0
- tan/core/text_layout.py +73 -0
- tan/core/timestamp.py +182 -0
- tan/core/tool_lookup.py +383 -0
- tan/core/toolchain_pin.py +117 -0
- tan/core/toolchain_provision.py +1224 -0
- tan/core/trusted_path.py +134 -0
- tan/core/uboot_breakin.py +154 -0
- tan/core/uri_reference.py +415 -0
- tan/core/user_defines.py +152 -0
- tan/core/venv.py +332 -0
- tan/core/west_patches.py +143 -0
- tan/core/west_workspace_refusal.py +159 -0
- tan/core/zephyr_env.py +124 -0
- tan/env.py +185 -0
- tan/envelope.py +647 -0
- tan/exit_codes.py +12 -0
- tan/lazy_typer.py +185 -0
- tan/model/__init__.py +2 -0
- tan/model/_gen_fixture.py +132 -0
- tan/model/adapters/__init__.py +116 -0
- tan/model/adapters/cpu.py +26 -0
- tan/model/adapters/deepx.py +94 -0
- tan/model/adapters/drpai.py +242 -0
- tan/model/adapters/ethos_u.py +951 -0
- tan/model/adapters/executorch.py +55 -0
- tan/model/analyze.py +694 -0
- tan/model/build.py +304 -0
- tan/model/check.py +682 -0
- tan/model/manifest.py +625 -0
- tan/model/measure.py +216 -0
- tan/model/npu_ops_validate.py +130 -0
- tan/model/package.py +125 -0
- tan/model/perf.py +635 -0
- tan/model/perf_apply.py +721 -0
- tan/model/prep.py +193 -0
- tan/model/sram_fit.py +502 -0
- tan/model/targets.py +374 -0
- tan/model/tensorio.py +287 -0
- tan/net.py +73 -0
- tan/output_format.py +213 -0
- tan/planner/__init__.py +219 -0
- tan/planner/aperture.py +202 -0
- tan/planner/buildplan.py +1092 -0
- tan/planner/camera_owner.py +192 -0
- tan/planner/cameras.py +132 -0
- tan/planner/carveout.py +755 -0
- tan/planner/cli.py +274 -0
- tan/planner/headers.py +315 -0
- tan/planner/kconfig.py +2435 -0
- tan/planner/kconfig_symbols.py +473 -0
- tan/planner/libraries.py +677 -0
- tan/planner/link_target.py +237 -0
- tan/planner/linux_ownership.py +216 -0
- tan/planner/loader.py +1525 -0
- tan/planner/manifest.py +116 -0
- tan/planner/memory.py +161 -0
- tan/planner/memregion.py +56 -0
- tan/planner/models.py +533 -0
- tan/planner/orchestrator.py +874 -0
- tan/planner/ownership.py +203 -0
- tan/planner/partition.py +1160 -0
- tan/planner/paths.py +32 -0
- tan/planner/plain_slice.py +139 -0
- tan/planner/project_emit/__init__.py +60 -0
- tan/planner/project_emit/bom_netlist.py +375 -0
- tan/planner/project_emit/dts.py +580 -0
- tan/planner/project_emit/hw_info.py +221 -0
- tan/planner/project_emit/native_sim.py +136 -0
- tan/planner/project_emit/west_libs.py +213 -0
- tan/planner/project_loader.py +264 -0
- tan/planner/sdk_capability.py +203 -0
- tan/planner/sdk_compat.py +395 -0
- tan/planner/secure.py +350 -0
- tan/planner/slugs.py +394 -0
- tan/planner/som_metadata.py +325 -0
- tan/planner/strict_loaders.py +131 -0
- tan/planner/template.py +1037 -0
- tan/planner/template_pins.py +535 -0
- tan/planner/template_rewrite.py +714 -0
- tan/planner/topology.py +208 -0
- tan/planner/validate.py +365 -0
- tan/planner/whole_device_alias.py +67 -0
- tan/planner/zephyr_board.py +3510 -0
- tan/planner_cli.py +97 -0
- tan/planner_emit.py +617 -0
- tan/planner_root.py +119 -0
- tan/soc_ref.py +58 -0
- tan/templates/__init__.py +34 -0
- tan/templates/vendored/MANIFEST.md +1862 -0
- tan/templates/vendored/diagnostics/E1M-AEN801/CMakeLists.txt +8 -0
- tan/templates/vendored/diagnostics/E1M-AEN801/README.md +112 -0
- tan/templates/vendored/diagnostics/E1M-AEN801/board.yaml +38 -0
- tan/templates/vendored/diagnostics/E1M-AEN801/boards/native_sim_native_64.conf +11 -0
- tan/templates/vendored/diagnostics/E1M-AEN801/boards/native_sim_native_64.overlay +29 -0
- tan/templates/vendored/diagnostics/E1M-AEN801/prj.conf +5 -0
- tan/templates/vendored/diagnostics/E1M-AEN801/src/main.c +266 -0
- tan/templates/vendored/diagnostics/E1M-AEN801/testcase.yaml +53 -0
- tan/templates/vendored/diagnostics/E1M-V2N101/CMakeLists.txt +8 -0
- tan/templates/vendored/diagnostics/E1M-V2N101/README.md +112 -0
- tan/templates/vendored/diagnostics/E1M-V2N101/board.yaml +38 -0
- tan/templates/vendored/diagnostics/E1M-V2N101/boards/native_sim_native_64.conf +11 -0
- tan/templates/vendored/diagnostics/E1M-V2N101/boards/native_sim_native_64.overlay +29 -0
- tan/templates/vendored/diagnostics/E1M-V2N101/prj.conf +5 -0
- tan/templates/vendored/diagnostics/E1M-V2N101/src/main.c +266 -0
- tan/templates/vendored/diagnostics/E1M-V2N101/testcase.yaml +53 -0
- tan/templates/vendored/edge-ai/E1M-AEN801/CMakeLists.txt +10 -0
- tan/templates/vendored/edge-ai/E1M-AEN801/README.md +75 -0
- tan/templates/vendored/edge-ai/E1M-AEN801/board.yaml +38 -0
- tan/templates/vendored/edge-ai/E1M-AEN801/prj.conf +8 -0
- tan/templates/vendored/edge-ai/E1M-AEN801/src/cold_chain.c +371 -0
- tan/templates/vendored/edge-ai/E1M-AEN801/src/cold_chain.h +178 -0
- tan/templates/vendored/edge-ai/E1M-AEN801/src/main.c +373 -0
- tan/templates/vendored/edge-ai/E1M-AEN801/testcase.yaml +62 -0
- tan/templates/vendored/edge-ai/E1M-V2N101/CMakeLists.txt +10 -0
- tan/templates/vendored/edge-ai/E1M-V2N101/README.md +75 -0
- tan/templates/vendored/edge-ai/E1M-V2N101/board.yaml +37 -0
- tan/templates/vendored/edge-ai/E1M-V2N101/prj.conf +8 -0
- tan/templates/vendored/edge-ai/E1M-V2N101/src/cold_chain.c +371 -0
- tan/templates/vendored/edge-ai/E1M-V2N101/src/cold_chain.h +178 -0
- tan/templates/vendored/edge-ai/E1M-V2N101/src/main.c +373 -0
- tan/templates/vendored/edge-ai/E1M-V2N101/testcase.yaml +62 -0
- tan/templates/vendored/iot/E1M-AEN801/CMakeLists.txt +14 -0
- tan/templates/vendored/iot/E1M-AEN801/README.md +103 -0
- tan/templates/vendored/iot/E1M-AEN801/board.yaml +69 -0
- tan/templates/vendored/iot/E1M-AEN801/prj.conf +15 -0
- tan/templates/vendored/iot/E1M-AEN801/src/cc3501e_bridge.c +196 -0
- tan/templates/vendored/iot/E1M-AEN801/src/cc3501e_bridge.h +173 -0
- tan/templates/vendored/iot/E1M-AEN801/src/main.c +262 -0
- tan/templates/vendored/iot/E1M-AEN801/testcase.yaml +68 -0
- tan/templates/vendored/minimal/E1M-AEN801/CMakeLists.txt +8 -0
- tan/templates/vendored/minimal/E1M-AEN801/README.md +108 -0
- tan/templates/vendored/minimal/E1M-AEN801/board.yaml +32 -0
- tan/templates/vendored/minimal/E1M-AEN801/prj.conf +8 -0
- tan/templates/vendored/minimal/E1M-AEN801/src/main.c +111 -0
- tan/templates/vendored/minimal/E1M-AEN801/testcase.yaml +49 -0
- tan/templates/vendored/minimal/E1M-V2N101/CMakeLists.txt +8 -0
- tan/templates/vendored/minimal/E1M-V2N101/README.md +108 -0
- tan/templates/vendored/minimal/E1M-V2N101/board.yaml +32 -0
- tan/templates/vendored/minimal/E1M-V2N101/prj.conf +8 -0
- tan/templates/vendored/minimal/E1M-V2N101/src/main.c +111 -0
- tan/templates/vendored/minimal/E1M-V2N101/testcase.yaml +49 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/CMakeLists.txt +8 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/README.md +120 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/board.yaml +57 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/boards/native_sim_native_64.overlay +40 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/peer/CMakeLists.txt +13 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/peer/main.c +171 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/peer/prj.conf +17 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/peer/testcase.yaml +23 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/prj.conf +22 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/src/main.c +185 -0
- tan/templates/vendored/multicore-mailbox/E1M-AEN801/testcase.yaml +26 -0
- tan/templates/vendored/sensor/E1M-AEN801/CMakeLists.txt +8 -0
- tan/templates/vendored/sensor/E1M-AEN801/README.md +112 -0
- tan/templates/vendored/sensor/E1M-AEN801/board.yaml +57 -0
- tan/templates/vendored/sensor/E1M-AEN801/boards/native_sim_native_64.conf +10 -0
- tan/templates/vendored/sensor/E1M-AEN801/boards/native_sim_native_64.overlay +28 -0
- tan/templates/vendored/sensor/E1M-AEN801/prj.conf +5 -0
- tan/templates/vendored/sensor/E1M-AEN801/src/main.c +186 -0
- tan/templates/vendored/sensor/E1M-AEN801/testcase.yaml +52 -0
- tan/templates/vendored/sensor/E1M-V2N101/CMakeLists.txt +8 -0
- tan/templates/vendored/sensor/E1M-V2N101/README.md +112 -0
- tan/templates/vendored/sensor/E1M-V2N101/board.yaml +57 -0
- tan/templates/vendored/sensor/E1M-V2N101/boards/native_sim_native_64.conf +10 -0
- tan/templates/vendored/sensor/E1M-V2N101/boards/native_sim_native_64.overlay +28 -0
- tan/templates/vendored/sensor/E1M-V2N101/prj.conf +5 -0
- tan/templates/vendored/sensor/E1M-V2N101/src/main.c +186 -0
- tan/templates/vendored/sensor/E1M-V2N101/testcase.yaml +52 -0
- tan/version.py +102 -0
- tan_cli-0.7.0.dist-info/METADATA +125 -0
- tan_cli-0.7.0.dist-info/RECORD +330 -0
- tan_cli-0.7.0.dist-info/WHEEL +5 -0
- tan_cli-0.7.0.dist-info/entry_points.txt +2 -0
- tan_cli-0.7.0.dist-info/top_level.txt +1 -0
tan/core/model_zoo.py
ADDED
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""The model-zoo engine behind `tan model zoo` / `tan model add` (tan-cli#1286,
|
|
3
|
+
ADR-0028).
|
|
4
|
+
|
|
5
|
+
alp-sdk owns the DATA (`metadata/model_zoo/<id>.yaml`, shape
|
|
6
|
+
`metadata/schemas/model-zoo-v1.schema.json`, validated by that repo's own
|
|
7
|
+
`scripts/validate_metadata.py`); tan owns the ENGINE that reads it, so this
|
|
8
|
+
module consumes the manifests and invents no zoo format of its own. It
|
|
9
|
+
re-checks only what a consumer must not take on trust (shape of the fields it
|
|
10
|
+
reads, the `source` oneOf, path confinement of a bundled starter, https-only
|
|
11
|
+
and sha256-pinned downloads) and tolerates fields it does not know, so an
|
|
12
|
+
SDK that adds optional fields does not break an older tan.
|
|
13
|
+
|
|
14
|
+
No zoo directory means an SDK that predates the data: `load_zoo` raises
|
|
15
|
+
`ZooUnavailable`, which the command turns into a refusal naming
|
|
16
|
+
`MIN_SDK_COMMIT`.
|
|
17
|
+
|
|
18
|
+
`validated_soms` is a hardware claim and an EMPTY list means "nothing
|
|
19
|
+
confirmed", never "works everywhere": `filter_by_sku` matches no entry whose
|
|
20
|
+
list is empty.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import hashlib
|
|
26
|
+
import os
|
|
27
|
+
import re
|
|
28
|
+
import tempfile
|
|
29
|
+
import time
|
|
30
|
+
from collections.abc import Callable, Iterable, Iterator
|
|
31
|
+
from dataclasses import dataclass
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
from typing import Any
|
|
34
|
+
|
|
35
|
+
#: The alp-sdk commit that first carried `metadata/model_zoo/` (alp-sdk#2542).
|
|
36
|
+
#: No released tag contains it yet; named in the `zoo-unavailable` refusal.
|
|
37
|
+
MIN_SDK_COMMIT = "7018515f9"
|
|
38
|
+
|
|
39
|
+
#: Model source suffixes `models[].source` accepts (board.schema.json).
|
|
40
|
+
MODEL_SUFFIXES = (".tflite", ".onnx", ".pte")
|
|
41
|
+
|
|
42
|
+
#: Hard ceiling on a downloaded model, so a hostile or broken URL cannot fill
|
|
43
|
+
#: the disk.
|
|
44
|
+
MAX_DOWNLOAD_BYTES = 256 * 1024 * 1024
|
|
45
|
+
|
|
46
|
+
#: Total wall-clock budget for one download.
|
|
47
|
+
MAX_DOWNLOAD_SECONDS = 300.0
|
|
48
|
+
|
|
49
|
+
_ID_RE = re.compile(r"[a-z][a-z0-9-]*")
|
|
50
|
+
_BUNDLED_RE = re.compile(r"starters/[A-Za-z0-9._-]+")
|
|
51
|
+
_SHA256_RE = re.compile(r"[a-f0-9]{64}")
|
|
52
|
+
_URL_RE = re.compile(r"https://[^\s/?#@]+/\S+")
|
|
53
|
+
_SKU_RE = re.compile(r"E1M-(AEN[3-8][0-9]{2}|V2N[0-9]{3}|V2M[0-9]{3})")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class ZooUnavailable(Exception):
|
|
57
|
+
"""The bound SDK has no `metadata/model_zoo/` directory."""
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class ZooFetchError(Exception):
|
|
61
|
+
"""A model could not be fetched or did not match its pinned hash."""
|
|
62
|
+
|
|
63
|
+
def __init__(self, message: str, *, mismatch: bool = False) -> None:
|
|
64
|
+
super().__init__(message)
|
|
65
|
+
self.mismatch = mismatch
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(frozen=True)
|
|
69
|
+
class ZooEntry:
|
|
70
|
+
id: str
|
|
71
|
+
kind: str
|
|
72
|
+
task: str
|
|
73
|
+
description: str
|
|
74
|
+
license: str
|
|
75
|
+
source: dict[str, str]
|
|
76
|
+
validated_soms: tuple[str, ...]
|
|
77
|
+
compile: dict[str, Any] | None
|
|
78
|
+
example_app: str | None
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def suffix(self) -> str:
|
|
82
|
+
"""`.tflite`/`.onnx`/`.pte` -- from the bundled file name or the URL path."""
|
|
83
|
+
raw = self.source.get("bundled") or self.source["url"].split("?", 1)[0]
|
|
84
|
+
return Path(raw).suffix.lower()
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def entry_as_dict(entry: ZooEntry) -> dict[str, Any]:
|
|
88
|
+
"""The `data.entries[]` row `--format json` carries for one entry."""
|
|
89
|
+
return {
|
|
90
|
+
"id": entry.id,
|
|
91
|
+
"kind": entry.kind,
|
|
92
|
+
"task": entry.task,
|
|
93
|
+
"description": entry.description,
|
|
94
|
+
"license": entry.license,
|
|
95
|
+
"source": dict(entry.source),
|
|
96
|
+
"validatedSoms": list(entry.validated_soms),
|
|
97
|
+
"compile": sorted(entry.compile) if entry.compile else [],
|
|
98
|
+
"exampleApp": entry.example_app,
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _check_compile(comp: Any) -> None:
|
|
103
|
+
"""`compile` must be `{backend: {key: str | int | [str|int, ...]}}` -- the
|
|
104
|
+
only shape `models[].compile` takes. Checked shallowly (no recursion), so a
|
|
105
|
+
self-referencing YAML alias is rejected here as a plain type error."""
|
|
106
|
+
if comp is None:
|
|
107
|
+
return
|
|
108
|
+
if not isinstance(comp, dict):
|
|
109
|
+
raise ValueError("`compile` must be a mapping")
|
|
110
|
+
|
|
111
|
+
def scalar(v: Any) -> bool:
|
|
112
|
+
return isinstance(v, (str, int)) and not isinstance(v, bool)
|
|
113
|
+
|
|
114
|
+
for backend, opts in comp.items():
|
|
115
|
+
if not isinstance(backend, str) or not isinstance(opts, dict) or not opts:
|
|
116
|
+
raise ValueError("`compile` must map backend ids to non-empty mappings")
|
|
117
|
+
for key, value in opts.items():
|
|
118
|
+
ok = scalar(value) or (isinstance(value, list) and value and all(scalar(v) for v in value))
|
|
119
|
+
if not isinstance(key, str) or not ok:
|
|
120
|
+
raise ValueError(f"`compile.{backend}.{key}` must be a string, integer or list of them")
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _parse_entry(doc: Any, stem: str) -> ZooEntry:
|
|
124
|
+
"""A `ZooEntry`, or `ValueError` naming the first thing wrong."""
|
|
125
|
+
if not isinstance(doc, dict):
|
|
126
|
+
raise ValueError("not a YAML mapping")
|
|
127
|
+
if doc.get("schema_version") != 1 or isinstance(doc.get("schema_version"), bool):
|
|
128
|
+
raise ValueError(f"unsupported schema_version {doc.get('schema_version')!r} (this tan reads 1)")
|
|
129
|
+
entry_id = doc.get("id")
|
|
130
|
+
if not isinstance(entry_id, str) or not _ID_RE.fullmatch(entry_id):
|
|
131
|
+
raise ValueError("`id` is missing or not lowercase kebab-case")
|
|
132
|
+
if entry_id != stem:
|
|
133
|
+
raise ValueError(f"`id` {entry_id!r} does not match the file name")
|
|
134
|
+
kind = doc.get("kind")
|
|
135
|
+
if kind not in ("model", "fixture"):
|
|
136
|
+
raise ValueError("`kind` must be `model` or `fixture`")
|
|
137
|
+
for key in ("task", "description", "license"):
|
|
138
|
+
if not isinstance(doc.get(key), str) or not doc[key]:
|
|
139
|
+
raise ValueError(f"`{key}` is missing")
|
|
140
|
+
src = doc.get("source")
|
|
141
|
+
if not isinstance(src, dict):
|
|
142
|
+
raise ValueError("`source` is missing")
|
|
143
|
+
if set(src) == {"bundled"}:
|
|
144
|
+
bundled = src["bundled"]
|
|
145
|
+
if not isinstance(bundled, str) or not _BUNDLED_RE.fullmatch(bundled) or ".." in bundled.split("/"):
|
|
146
|
+
raise ValueError("`source.bundled` must be `starters/<file>`")
|
|
147
|
+
source = {"bundled": bundled}
|
|
148
|
+
elif set(src) == {"url", "sha256"}:
|
|
149
|
+
url, digest = src["url"], src["sha256"]
|
|
150
|
+
if not isinstance(url, str) or not _URL_RE.fullmatch(url):
|
|
151
|
+
raise ValueError("`source.url` must be an https:// URL without credentials")
|
|
152
|
+
if not isinstance(digest, str) or not _SHA256_RE.fullmatch(digest):
|
|
153
|
+
raise ValueError("`source.sha256` must be 64 lowercase hex characters")
|
|
154
|
+
source = {"url": url, "sha256": digest}
|
|
155
|
+
else:
|
|
156
|
+
raise ValueError("`source` must be exactly {bundled} or {url, sha256}")
|
|
157
|
+
soms = doc.get("validated_soms")
|
|
158
|
+
if not isinstance(soms, list) or not all(isinstance(s, str) and _SKU_RE.fullmatch(s) for s in soms):
|
|
159
|
+
raise ValueError("`validated_soms` must be a list of SKUs")
|
|
160
|
+
comp = doc.get("compile")
|
|
161
|
+
_check_compile(comp)
|
|
162
|
+
example_app = doc.get("example_app")
|
|
163
|
+
if example_app is not None and not isinstance(example_app, str):
|
|
164
|
+
raise ValueError("`example_app` must be a string")
|
|
165
|
+
entry = ZooEntry(
|
|
166
|
+
entry_id, kind, doc["task"], doc["description"], doc["license"], source,
|
|
167
|
+
tuple(soms), comp or None, example_app,
|
|
168
|
+
)
|
|
169
|
+
if entry.suffix not in MODEL_SUFFIXES:
|
|
170
|
+
raise ValueError(f"source file type {entry.suffix or '(none)'!r} is not one of {', '.join(MODEL_SUFFIXES)}")
|
|
171
|
+
return entry
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def load_zoo(zoo_dir: Path) -> tuple[list[ZooEntry], list[tuple[str, str]]]:
|
|
175
|
+
"""`(entries sorted by id, [(file name, problem), ...])` for every
|
|
176
|
+
`*.yaml` under `zoo_dir`. A bad file is reported and skipped, never fatal
|
|
177
|
+
to the rest. Raises `ZooUnavailable` when `zoo_dir` does not exist."""
|
|
178
|
+
import yaml # noqa: PLC0415 (declared dependency)
|
|
179
|
+
|
|
180
|
+
if not zoo_dir.is_dir():
|
|
181
|
+
raise ZooUnavailable(str(zoo_dir))
|
|
182
|
+
entries: list[ZooEntry] = []
|
|
183
|
+
problems: list[tuple[str, str]] = []
|
|
184
|
+
for path in sorted(zoo_dir.glob("*.yaml")):
|
|
185
|
+
try:
|
|
186
|
+
entries.append(_parse_entry(yaml.safe_load(path.read_text(encoding="utf-8")), path.stem))
|
|
187
|
+
except (OSError, UnicodeDecodeError, yaml.YAMLError, ValueError, RecursionError) as err:
|
|
188
|
+
problems.append((path.name, str(err)))
|
|
189
|
+
entries.sort(key=lambda e: e.id)
|
|
190
|
+
return entries, problems
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def filter_by_sku(entries: list[ZooEntry], sku: str) -> list[ZooEntry]:
|
|
194
|
+
"""Entries bench-validated on exactly `sku`; an empty `validated_soms`
|
|
195
|
+
matches nothing (module doc)."""
|
|
196
|
+
return [e for e in entries if sku in e.validated_soms]
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
#: Largest read from the socket at a time (never an unbounded `read()`).
|
|
200
|
+
READ_CHUNK_BYTES = 64 * 1024
|
|
201
|
+
|
|
202
|
+
#: Redirects followed before giving up.
|
|
203
|
+
MAX_REDIRECTS = 5
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def iter_url(
|
|
207
|
+
url: str,
|
|
208
|
+
*,
|
|
209
|
+
max_seconds: float = MAX_DOWNLOAD_SECONDS,
|
|
210
|
+
_schemes: tuple[str, ...] = ("https://",),
|
|
211
|
+
) -> Iterator[bytes]:
|
|
212
|
+
"""Chunks (<= 64 KiB) of a GET with tan's TLS trust context.
|
|
213
|
+
|
|
214
|
+
Hardened against a hostile server: only identity encoding is accepted (a
|
|
215
|
+
gzip/deflate body is refused, and `Accept-Encoding: identity` is sent, so
|
|
216
|
+
nothing expands transparently); a `Content-Length` over `MAX_DOWNLOAD_BYTES`
|
|
217
|
+
is refused before reading; `read1` returns after one receive, so the total
|
|
218
|
+
wall-clock deadline is checked between every chunk (a slow drip cannot
|
|
219
|
+
hide inside one long read); redirects are capped at `MAX_REDIRECTS` and may
|
|
220
|
+
only stay on https. The body is never trusted to match its
|
|
221
|
+
`Content-Length`: `fetch_source` still counts every byte against the cap.
|
|
222
|
+
`urllib` is imported here so a bare `tan` start-up does not load it."""
|
|
223
|
+
import urllib.error # noqa: PLC0415
|
|
224
|
+
import urllib.request # noqa: PLC0415
|
|
225
|
+
|
|
226
|
+
from tan.net import default_ssl_context # noqa: PLC0415
|
|
227
|
+
|
|
228
|
+
deadline = time.monotonic() + max_seconds
|
|
229
|
+
|
|
230
|
+
class LimitedRedirect(urllib.request.HTTPRedirectHandler):
|
|
231
|
+
max_redirections = MAX_REDIRECTS
|
|
232
|
+
|
|
233
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl): # type: ignore[no-untyped-def]
|
|
234
|
+
if not newurl.lower().startswith(_schemes):
|
|
235
|
+
raise urllib.error.URLError(f"refusing a redirect to a non-https URL: {newurl}")
|
|
236
|
+
if time.monotonic() > deadline:
|
|
237
|
+
raise urllib.error.URLError(f"not complete within {max_seconds:g} s")
|
|
238
|
+
return super().redirect_request(req, fp, code, msg, headers, newurl)
|
|
239
|
+
|
|
240
|
+
try:
|
|
241
|
+
opener = urllib.request.build_opener(
|
|
242
|
+
urllib.request.HTTPSHandler(context=default_ssl_context()), LimitedRedirect
|
|
243
|
+
)
|
|
244
|
+
req = urllib.request.Request(
|
|
245
|
+
url, headers={"User-Agent": "tan-model-zoo", "Accept-Encoding": "identity"}
|
|
246
|
+
)
|
|
247
|
+
with opener.open(req, timeout=max(0.1, min(30.0, max_seconds))) as resp:
|
|
248
|
+
encoding = (resp.headers.get("Content-Encoding") or "identity").strip().lower()
|
|
249
|
+
if encoding != "identity":
|
|
250
|
+
raise ZooFetchError(f"{url}: refusing Content-Encoding {encoding!r}")
|
|
251
|
+
declared = resp.headers.get("Content-Length")
|
|
252
|
+
if declared and declared.strip().isdigit() and int(declared) > MAX_DOWNLOAD_BYTES:
|
|
253
|
+
raise ZooFetchError(f"{url}: Content-Length {declared} exceeds {MAX_DOWNLOAD_BYTES} bytes")
|
|
254
|
+
while True:
|
|
255
|
+
if time.monotonic() > deadline:
|
|
256
|
+
raise ZooFetchError(f"{url}: not complete within {max_seconds:g} s")
|
|
257
|
+
chunk = resp.read1(READ_CHUNK_BYTES)
|
|
258
|
+
if not chunk:
|
|
259
|
+
return
|
|
260
|
+
yield chunk
|
|
261
|
+
except ZooFetchError:
|
|
262
|
+
raise
|
|
263
|
+
except (OSError, ValueError) as err: # URLError is an OSError
|
|
264
|
+
raise ZooFetchError(f"{url}: {err}") from err
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
@dataclass(frozen=True)
|
|
268
|
+
class StagedModel:
|
|
269
|
+
"""A verified model sitting in a temp file the caller must publish or unlink."""
|
|
270
|
+
|
|
271
|
+
path: Path
|
|
272
|
+
sha256: str
|
|
273
|
+
size: int
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def bundled_chunks(zoo_dir: Path, bundled: str) -> Iterator[bytes]:
|
|
277
|
+
starters = (zoo_dir / "starters").resolve()
|
|
278
|
+
path = (zoo_dir / bundled).resolve()
|
|
279
|
+
if path.parent != starters or not path.is_file():
|
|
280
|
+
raise ZooFetchError(f"bundled starter {bundled} is missing or outside {starters}")
|
|
281
|
+
try:
|
|
282
|
+
with path.open("rb") as handle:
|
|
283
|
+
while chunk := handle.read(1 << 20):
|
|
284
|
+
yield chunk
|
|
285
|
+
except OSError as err:
|
|
286
|
+
raise ZooFetchError(f"{path}: {err}") from err
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def closing_iter(chunks: Iterable[bytes]) -> Iterator[bytes]:
|
|
290
|
+
"""Iterate `chunks`, closing it (generator `.close()`, which releases the
|
|
291
|
+
HTTP connection) when the consumer stops early -- a tripped cap or deadline."""
|
|
292
|
+
it = iter(chunks)
|
|
293
|
+
try:
|
|
294
|
+
yield from it
|
|
295
|
+
finally:
|
|
296
|
+
close = getattr(it, "close", None)
|
|
297
|
+
if close is not None:
|
|
298
|
+
close()
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def fetch_source(
|
|
302
|
+
entry: ZooEntry,
|
|
303
|
+
zoo_dir: Path,
|
|
304
|
+
*,
|
|
305
|
+
tmp_dir: Path,
|
|
306
|
+
reader: Callable[[str], Iterable[bytes]] = iter_url,
|
|
307
|
+
max_seconds: float = MAX_DOWNLOAD_SECONDS,
|
|
308
|
+
) -> StagedModel:
|
|
309
|
+
"""Stream the model for `entry` into a temp file in `tmp_dir` (the
|
|
310
|
+
destination directory, so publishing is a same-filesystem link), hashing as
|
|
311
|
+
it goes -- never buffering the model in memory. A download is capped in
|
|
312
|
+
size and total wall-clock time and must hash to the manifest's `sha256`;
|
|
313
|
+
a bundled starter is read from inside `zoo_dir/starters` (confined). The
|
|
314
|
+
temp file is removed on any failure. Raises `ZooFetchError`; a hash
|
|
315
|
+
mismatch sets `.mismatch`."""
|
|
316
|
+
bundled = entry.source.get("bundled")
|
|
317
|
+
chunks = closing_iter(bundled_chunks(zoo_dir, bundled) if bundled else reader(entry.source["url"]))
|
|
318
|
+
label = bundled or entry.source["url"]
|
|
319
|
+
digest = hashlib.sha256()
|
|
320
|
+
size = 0
|
|
321
|
+
deadline = time.monotonic() + max_seconds
|
|
322
|
+
fd, tmp_name = tempfile.mkstemp(dir=tmp_dir, prefix=".tan-zoo-", suffix=".part")
|
|
323
|
+
try:
|
|
324
|
+
# `mkstemp` makes a 0600 file; the published model should carry the
|
|
325
|
+
# mode an ordinary new file would (0666 & ~umask).
|
|
326
|
+
mask = os.umask(0)
|
|
327
|
+
os.umask(mask)
|
|
328
|
+
os.chmod(tmp_name, 0o666 & ~mask)
|
|
329
|
+
with os.fdopen(fd, "wb") as out:
|
|
330
|
+
for chunk in chunks:
|
|
331
|
+
if size + len(chunk) > MAX_DOWNLOAD_BYTES:
|
|
332
|
+
raise ZooFetchError(f"{label}: larger than {MAX_DOWNLOAD_BYTES} bytes")
|
|
333
|
+
size += len(chunk)
|
|
334
|
+
if time.monotonic() > deadline:
|
|
335
|
+
raise ZooFetchError(f"{label}: not complete within {max_seconds:g} s")
|
|
336
|
+
digest.update(chunk)
|
|
337
|
+
out.write(chunk)
|
|
338
|
+
chunks.close()
|
|
339
|
+
got = digest.hexdigest()
|
|
340
|
+
if not bundled and got != entry.source["sha256"]:
|
|
341
|
+
raise ZooFetchError(
|
|
342
|
+
f"{label}: sha256 {got} does not match the manifest's {entry.source['sha256']}",
|
|
343
|
+
mismatch=True,
|
|
344
|
+
)
|
|
345
|
+
except OSError as err:
|
|
346
|
+
chunks.close()
|
|
347
|
+
Path(tmp_name).unlink(missing_ok=True)
|
|
348
|
+
raise ZooFetchError(f"{label}: {err}") from err
|
|
349
|
+
except BaseException:
|
|
350
|
+
chunks.close()
|
|
351
|
+
Path(tmp_name).unlink(missing_ok=True)
|
|
352
|
+
raise
|
|
353
|
+
return StagedModel(Path(tmp_name), got, size)
|
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Module-scaffold template registry, name normalization, and file-content
|
|
3
|
+
generators for `tan scaffold` -- adds one driver/service/stage/check module
|
|
4
|
+
INTO an existing project. Distinct from `tan.core.scaffold`, which is the
|
|
5
|
+
whole-*project* `tan init` engine (a different template id space: six
|
|
6
|
+
project templates there vs. the four module templates here, and neither
|
|
7
|
+
registry's ids overlap the other's).
|
|
8
|
+
|
|
9
|
+
Port of `crates/tan-core/src/wizard/{models,service/registry,service/plan,
|
|
10
|
+
service/module_scaffold}.rs`'s module-scaffold slice only -- the project-
|
|
11
|
+
wizard half of those same files is `tan.core.scaffold`'s job.
|
|
12
|
+
|
|
13
|
+
Diffing planned files against disk, writing them, and rendering the ASCII
|
|
14
|
+
tree preview are deliberately NOT re-implemented here: `tan.core.scaffold`'s
|
|
15
|
+
`PlannedFile` / `collect_file_changes` / `write_files` / `scaffold_tree_preview`
|
|
16
|
+
already do exactly that -- module-scaffold's planned files are the SAME
|
|
17
|
+
`PlannedFile` shape a whole-project plan uses, and `write_files` already
|
|
18
|
+
carries the tan-cli#325 containment fix (`tan.core.fs_confine.resolve_confined`).
|
|
19
|
+
A second copy of any of the four here would be exactly the drift
|
|
20
|
+
`tan.core.fs_confine`'s own module docstring warns a THIRD hand-rolled
|
|
21
|
+
resolver into being.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
from dataclasses import dataclass
|
|
27
|
+
|
|
28
|
+
from tan.core.scaffold import PlannedFile
|
|
29
|
+
|
|
30
|
+
#: Module template ids, in registry order (`ModuleTemplateId::as_str`,
|
|
31
|
+
#: `wizard/models.rs`). Wire contract: `data.templateId` echoes one of these
|
|
32
|
+
#: verbatim.
|
|
33
|
+
MODULE_TEMPLATE_IDS = (
|
|
34
|
+
"sensor-driver",
|
|
35
|
+
"connectivity-service",
|
|
36
|
+
"inference-stage",
|
|
37
|
+
"diagnostics-check",
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
#: The template a non-interactive `tan scaffold` with no `--template` gets --
|
|
41
|
+
#: `resolve_template`'s non-interactive arm in `crates/tan-cli/src/commands/
|
|
42
|
+
#: scaffold.rs` hardcodes `ModuleTemplateId::SensorDriver`, the registry's
|
|
43
|
+
#: first entry. Unlike `tan.core.scaffold.DEFAULT_TEMPLATE_ID` (`tan init`'s
|
|
44
|
+
#: own default), this one has no reason to diverge from "first in the list":
|
|
45
|
+
#: there is no `CMakeLists.txt`-shaped trap here, every module template emits
|
|
46
|
+
#: the same three-file shape.
|
|
47
|
+
DEFAULT_MODULE_TEMPLATE_ID = "sensor-driver"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True)
|
|
51
|
+
class ModuleTemplateDefinition:
|
|
52
|
+
"""One module-template registry entry. `explanation` lines may contain a
|
|
53
|
+
literal `{nm}` placeholder, substituted with the normalized module name
|
|
54
|
+
when a module's `README.md` is rendered (see `_readme`, below)."""
|
|
55
|
+
|
|
56
|
+
id: str
|
|
57
|
+
label: str
|
|
58
|
+
description: str
|
|
59
|
+
function_prefix: str
|
|
60
|
+
explanation: tuple[str, ...]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
#: `MODULE_TEMPLATE_DEFINITIONS` (`wizard/service/registry.rs`), ported
|
|
64
|
+
#: verbatim -- same four ids, same order, same prose.
|
|
65
|
+
_REGISTRY: tuple[ModuleTemplateDefinition, ...] = (
|
|
66
|
+
ModuleTemplateDefinition(
|
|
67
|
+
id="sensor-driver",
|
|
68
|
+
label="Sensor driver module",
|
|
69
|
+
description="Adds a source/header pair for sensor acquisition logic.",
|
|
70
|
+
function_prefix="alp_sensor",
|
|
71
|
+
explanation=(
|
|
72
|
+
"Use {nm}_run to place sensor polling and conversion logic.",
|
|
73
|
+
"Keep hardware-specific register access isolated from upper-level app flow.",
|
|
74
|
+
),
|
|
75
|
+
),
|
|
76
|
+
ModuleTemplateDefinition(
|
|
77
|
+
id="connectivity-service",
|
|
78
|
+
label="Connectivity service module",
|
|
79
|
+
description="Adds module skeleton for network/session orchestration.",
|
|
80
|
+
function_prefix="alp_conn",
|
|
81
|
+
explanation=(
|
|
82
|
+
"Use {nm}_init for stack/session initialization.",
|
|
83
|
+
"Keep retry/backoff and transport health checks localized in this module.",
|
|
84
|
+
),
|
|
85
|
+
),
|
|
86
|
+
ModuleTemplateDefinition(
|
|
87
|
+
id="inference-stage",
|
|
88
|
+
label="Inference stage module",
|
|
89
|
+
description="Adds module skeleton for model pre/post processing path.",
|
|
90
|
+
function_prefix="alp_infer",
|
|
91
|
+
explanation=(
|
|
92
|
+
"Use {nm}_run to host pre-process, infer, and post-process calls.",
|
|
93
|
+
"Keep model IO shaping and feature extraction close to this module boundary.",
|
|
94
|
+
),
|
|
95
|
+
),
|
|
96
|
+
ModuleTemplateDefinition(
|
|
97
|
+
id="diagnostics-check",
|
|
98
|
+
label="Diagnostics check module",
|
|
99
|
+
description="Adds bring-up and runtime health-check module scaffold.",
|
|
100
|
+
function_prefix="alp_diag",
|
|
101
|
+
explanation=(
|
|
102
|
+
"Use {nm}_run for periodic health checks and error probes.",
|
|
103
|
+
"Keep board bring-up assertions and diagnostics output in this module.",
|
|
104
|
+
),
|
|
105
|
+
),
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
_BY_ID = {d.id: d for d in _REGISTRY}
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def list_module_templates() -> list[ModuleTemplateDefinition]:
|
|
112
|
+
"""All registered module-scaffold templates, in registry order."""
|
|
113
|
+
return list(_REGISTRY)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def normalize_module_name(name: str) -> str:
|
|
117
|
+
"""Lowercase `name` and collapse every run of non-`[a-z0-9]` characters
|
|
118
|
+
(after lowering) into a single `_` separator, with none leading or
|
|
119
|
+
trailing. Raises `ValueError` -- its message is `scaffold.invalid-name`'s
|
|
120
|
+
wire text verbatim -- when nothing survives.
|
|
121
|
+
|
|
122
|
+
`wizard::service::plan::normalize_module_name`, ported: Rust lowercases
|
|
123
|
+
with the full Unicode mapping, then keeps only `is_ascii_lowercase() ||
|
|
124
|
+
is_ascii_digit()` chars, treating everything else (accented letters
|
|
125
|
+
included, even after they lower) as a separator run. Python's `.lower()`
|
|
126
|
+
is equally Unicode-aware, so the same two-step -- lower, then filter to
|
|
127
|
+
ASCII alnum -- reproduces it byte-for-byte (measured against the oracle:
|
|
128
|
+
`"Héllo--World123"` -> `"h_llo_world123"`, the accented `é` collapsing
|
|
129
|
+
into the same run as the double dash beside it).
|
|
130
|
+
"""
|
|
131
|
+
lowered = name.strip().lower()
|
|
132
|
+
out: list[str] = []
|
|
133
|
+
in_sep = False
|
|
134
|
+
for ch in lowered:
|
|
135
|
+
if ("a" <= ch <= "z") or ("0" <= ch <= "9"):
|
|
136
|
+
if in_sep and out:
|
|
137
|
+
out.append("_")
|
|
138
|
+
out.append(ch)
|
|
139
|
+
in_sep = False
|
|
140
|
+
else:
|
|
141
|
+
in_sep = True
|
|
142
|
+
result = "".join(out)
|
|
143
|
+
if not result:
|
|
144
|
+
raise ValueError("Module name is empty after normalization.")
|
|
145
|
+
return result
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _header(prefix: str, nm: str) -> str:
|
|
149
|
+
"""`gen_module_header`, ported. `nm` is always already-normalized ASCII
|
|
150
|
+
lowercase/digits/underscore, so a plain `.upper()` reproduces Rust's
|
|
151
|
+
`nm.to_uppercase()` exactly -- no non-ASCII input ever reaches here."""
|
|
152
|
+
upper = nm.upper()
|
|
153
|
+
return (
|
|
154
|
+
"// SPDX-License-Identifier: Apache-2.0\n"
|
|
155
|
+
"\n"
|
|
156
|
+
f"#ifndef ALP_MODULES_{upper}_H\n"
|
|
157
|
+
f"#define ALP_MODULES_{upper}_H\n"
|
|
158
|
+
"\n"
|
|
159
|
+
f"int {prefix}_{nm}_init(void);\n"
|
|
160
|
+
f"int {prefix}_{nm}_run(void);\n"
|
|
161
|
+
"\n"
|
|
162
|
+
f"#endif /* ALP_MODULES_{upper}_H */\n"
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _source(prefix: str, nm: str, board_context: str | None) -> str:
|
|
167
|
+
"""`gen_module_c`, ported, with tan-cli#1031's RESOLVED board line.
|
|
168
|
+
|
|
169
|
+
*board_context* is `"<som.sku> / <os>"` as
|
|
170
|
+
`tan.core.board_context.read_board_context` reads it out of the project's
|
|
171
|
+
own `board.yaml`, or `None` when no board resolved. `None` renders the
|
|
172
|
+
`unavailable` placeholder this line has carried since the Rust oracle --
|
|
173
|
+
unchanged bytes, but it now means "there is genuinely no board.yaml here"
|
|
174
|
+
instead of "this command never looked", which is the whole of tan-cli#1031:
|
|
175
|
+
every project got `unavailable`, including one that passed
|
|
176
|
+
`--board-yaml <p>/board.yaml` explicitly.
|
|
177
|
+
|
|
178
|
+
Not read from disk here: this module plans FILE CONTENT and has no IO, the
|
|
179
|
+
same split `plan_module_files` keeps for diffing and writing (see the
|
|
180
|
+
module docstring). The command resolves, this renders.
|
|
181
|
+
"""
|
|
182
|
+
hint = board_context if board_context else "unavailable"
|
|
183
|
+
return (
|
|
184
|
+
"// SPDX-License-Identifier: Apache-2.0\n"
|
|
185
|
+
"\n"
|
|
186
|
+
f'#include "modules/{nm}.h"\n'
|
|
187
|
+
"\n"
|
|
188
|
+
f"// Board context: {hint}\n"
|
|
189
|
+
"\n"
|
|
190
|
+
f"int {prefix}_{nm}_init(void) {{\n"
|
|
191
|
+
" // TODO: initialize module dependencies.\n"
|
|
192
|
+
" return 0;\n"
|
|
193
|
+
"}\n"
|
|
194
|
+
"\n"
|
|
195
|
+
f"int {prefix}_{nm}_run(void) {{\n"
|
|
196
|
+
" // TODO: implement module main behavior.\n"
|
|
197
|
+
" return 0;\n"
|
|
198
|
+
"}\n"
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _wiring(nm: str) -> str:
|
|
203
|
+
"""The `## Wiring` section (tan-cli#494 defect 5).
|
|
204
|
+
|
|
205
|
+
`tan scaffold` wrote a module NO SHIPPED TEMPLATE CAN BUILD and said
|
|
206
|
+
nothing about it. Measured off the frozen oracle trees in
|
|
207
|
+
`tests/fixtures/oracle_captures/scaffold_trees.json`, which are the six
|
|
208
|
+
project templates' real bytes: not one of the six names `modules` in any
|
|
209
|
+
`CMakeLists.txt`, so `src/modules/<nm>/<nm>.c` is compiled by nothing, and
|
|
210
|
+
only `minimal-app` sets an include directory at all, so
|
|
211
|
+
`#include "modules/<nm>.h"` does not resolve under the other five. The
|
|
212
|
+
command reported `ok:true`, three files written, no issue and no next
|
|
213
|
+
step, and the module was silently dead.
|
|
214
|
+
|
|
215
|
+
Inherited from the frozen oracle
|
|
216
|
+
(`crates/tan-core/src/wizard/service/module_scaffold.rs`), so this is a
|
|
217
|
+
pre-existing PRODUCT gap rather than a port regression -- and the oracle
|
|
218
|
+
is a fixed point for behaviour that is right, not for a scaffold that
|
|
219
|
+
cannot build. Nothing frozen pins these particular bytes:
|
|
220
|
+
`PARITY-COVERAGE.txt` records `scaffold` as covered for its REFUSAL
|
|
221
|
+
envelope only ("STILL UNCOVERED: an actual write"), `contract/envelopes/`
|
|
222
|
+
carries no scaffold success case, and `scaffold_trees.json` freezes `tan
|
|
223
|
+
init`'s PROJECT trees -- it contains no `src/modules/` path at all. The
|
|
224
|
+
only pin is `tests/core/test_module_template.py`'s hand-written
|
|
225
|
+
expectation, updated with this change.
|
|
226
|
+
|
|
227
|
+
**Two shapes, because the six templates have two.** `tan scaffold` adds a
|
|
228
|
+
module to an EXISTING project and is never told which template made it
|
|
229
|
+
(`plan_module_files` receives the definition and the name, nothing else),
|
|
230
|
+
so the section cannot pick one -- and naming only the majority shape would
|
|
231
|
+
hand `minimal-app` users a target that does not exist in their tree. Five
|
|
232
|
+
templates (`zephyr-app`, `iot-starter`, `sensor-starter`,
|
|
233
|
+
`edge-ai-starter`, `board-diagnostics`) use Zephyr's `app` target in the
|
|
234
|
+
top-level `CMakeLists.txt`; `minimal-app` builds `alp_app` from an
|
|
235
|
+
`ALP_APP_SOURCES` list in `src/CMakeLists.txt` and already sets
|
|
236
|
+
`target_include_directories(alp_app PRIVATE ../include)`, so it needs the
|
|
237
|
+
source entry only -- relative to `src/`, since that is where its
|
|
238
|
+
`add_executable` lives.
|
|
239
|
+
|
|
240
|
+
The exact lines are spelled out rather than described, because they are
|
|
241
|
+
the whole content of the fix: an approximate instruction here is worth no
|
|
242
|
+
more than the silence it replaces.
|
|
243
|
+
"""
|
|
244
|
+
return (
|
|
245
|
+
"## Wiring\n"
|
|
246
|
+
"\n"
|
|
247
|
+
"`tan scaffold` writes this module but does NOT edit your build files.\n"
|
|
248
|
+
"\n"
|
|
249
|
+
"Most templates (`zephyr-app`, `iot-starter`, `sensor-starter`,\n"
|
|
250
|
+
"`edge-ai-starter`, `board-diagnostics`) build Zephyr's `app` target -- add\n"
|
|
251
|
+
"both lines to the top-level `CMakeLists.txt`, after its `target_sources`:\n"
|
|
252
|
+
"\n"
|
|
253
|
+
"```cmake\n"
|
|
254
|
+
f"target_sources(app PRIVATE src/modules/{nm}/{nm}.c)\n"
|
|
255
|
+
"target_include_directories(app PRIVATE include)\n"
|
|
256
|
+
"```\n"
|
|
257
|
+
"\n"
|
|
258
|
+
"`minimal-app` builds `alp_app` and already sets its include directory, so\n"
|
|
259
|
+
"it needs the source only -- add this to `ALP_APP_SOURCES` in\n"
|
|
260
|
+
"`src/CMakeLists.txt` (paths there are relative to `src/`):\n"
|
|
261
|
+
"\n"
|
|
262
|
+
"```cmake\n"
|
|
263
|
+
f" modules/{nm}/{nm}.c\n"
|
|
264
|
+
"```\n"
|
|
265
|
+
"\n"
|
|
266
|
+
"Without the source entry the module is never compiled; without an include\n"
|
|
267
|
+
f'directory `#include "modules/{nm}.h"` does not resolve.\n'
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _readme(definition: ModuleTemplateDefinition, nm: str) -> str:
|
|
272
|
+
"""`gen_module_readme`, ported, plus the `## Wiring` section (see
|
|
273
|
+
[`_wiring`] for why the oracle's own output is not the fixed point here)."""
|
|
274
|
+
lines = "".join(f"- {line.replace('{nm}', nm)}\n" for line in definition.explanation)
|
|
275
|
+
return (
|
|
276
|
+
"# Alp Module Scaffold\n"
|
|
277
|
+
"\n"
|
|
278
|
+
f"Template: {definition.id}\n"
|
|
279
|
+
f"Module: {nm}\n"
|
|
280
|
+
"\n"
|
|
281
|
+
"## Notes\n"
|
|
282
|
+
"\n"
|
|
283
|
+
f"{lines}"
|
|
284
|
+
"\n"
|
|
285
|
+
f"{_wiring(nm)}"
|
|
286
|
+
"\n"
|
|
287
|
+
"Generated by Alp: Scaffold module.\n"
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def plan_module_files(
|
|
292
|
+
definition: ModuleTemplateDefinition, nm: str, board_context: str | None = None
|
|
293
|
+
) -> list[PlannedFile]:
|
|
294
|
+
"""The three files a module template lays down: a header, its source, and
|
|
295
|
+
a README. `gen_module_files`, ported -- same relative paths, same order
|
|
296
|
+
(the order the oracle's own `data.fileChanges[]` lists them in, and what
|
|
297
|
+
`contract`-style byte-for-byte JSON comparison depends on).
|
|
298
|
+
|
|
299
|
+
*board_context*: see [`_source`]. Defaults to `None` (the `unavailable`
|
|
300
|
+
line) so a caller with no project in hand -- every existing one but
|
|
301
|
+
`scaffold_cmd` -- is unchanged."""
|
|
302
|
+
return [
|
|
303
|
+
PlannedFile(f"include/modules/{nm}.h", _header(definition.function_prefix, nm)),
|
|
304
|
+
PlannedFile(
|
|
305
|
+
f"src/modules/{nm}/{nm}.c",
|
|
306
|
+
_source(definition.function_prefix, nm, board_context),
|
|
307
|
+
),
|
|
308
|
+
PlannedFile(f"src/modules/{nm}/README.md", _readme(definition, nm)),
|
|
309
|
+
]
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
@dataclass(frozen=True)
|
|
313
|
+
class ModuleScaffoldPlan:
|
|
314
|
+
"""`ModuleScaffoldPlan`, ported: the resolved template plus the module's
|
|
315
|
+
normalized name and its planned files."""
|
|
316
|
+
|
|
317
|
+
template_id: str
|
|
318
|
+
normalized_name: str
|
|
319
|
+
files: list[PlannedFile]
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def create_module_scaffold_plan(
|
|
323
|
+
template_id: str, module_name: str, board_context: str | None = None
|
|
324
|
+
) -> ModuleScaffoldPlan:
|
|
325
|
+
"""Normalize `module_name`, then plan its three-file set against
|
|
326
|
+
`template_id`. Raises `ValueError` when the name normalizes to empty
|
|
327
|
+
(`normalize_module_name`); a `template_id` outside `MODULE_TEMPLATE_IDS`
|
|
328
|
+
is a caller bug, not a user error -- `scaffold_cmd` validates it against
|
|
329
|
+
the registry BEFORE calling this, so `KeyError` here would mean that
|
|
330
|
+
validation was skipped. `create_module_scaffold_plan`, ported.
|
|
331
|
+
|
|
332
|
+
*board_context* is threaded straight through to [`plan_module_files`] and
|
|
333
|
+
is the only input here that is not derivable from the two required
|
|
334
|
+
arguments -- it is a fact about the PROJECT, which this registry has no
|
|
335
|
+
way to reach and (per its own module docstring) no business reading.
|
|
336
|
+
"""
|
|
337
|
+
normalized = normalize_module_name(module_name)
|
|
338
|
+
definition = _BY_ID[template_id]
|
|
339
|
+
files = plan_module_files(definition, normalized, board_context)
|
|
340
|
+
return ModuleScaffoldPlan(template_id=template_id, normalized_name=normalized, files=files)
|