tan-cli 0.7.0__py3-none-any.whl

tan/planner/som_metadata.py

@@ -0,0 +1,325 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""SoM/SoC metadata readers -- the memory map, the capability set, the SoC ref.
+
+RELOCATED (was alp-sdk `scripts/alp_project_loader.py`). These are the last
+functions `tan/planner/*` reached for across the repo boundary: `carveout.py`,
+`partition.py` and `loader.py` want `resolve_memory_map`, `libraries.py` and
+`kconfig.py` want `resolve_capabilities`, and `kconfig.py` wants
+`silicon_to_kconfig` + `som_unpopulated_capabilities`. While they lived in
+alp-sdk every one of those module-scope imports dragged `alp_project` ->
+`alp_project_loader` -> `alp_orchestrate` back into the process, so the planner
+emitted from `tan` but still LOADED the SDK's Python.
+
+**They read metadata; they do not carry it.** Every path below resolves under
+`.paths.METADATA_ROOT`, i.e. the bound alp-sdk checkout -- the SoC JSON, the
+SoM preset, the silicon-Kconfig registry. `metadata/**` stays in alp-sdk
+(ADR-0017 / I-26): what moved is the derivation, never the fact. A copy of any
+of these tables inside `tan` would be a second source of truth.
+
+Implementations are the SDK's, moved substantially as-is: their precedence
+rules (a SoM `memory_map:` override beating the SoC variant, SoC-level
+`memory_regions` beating the per-bank derivation, SoM capabilities beating SoC
+defaults, then the per-SKU `unpopulated` restriction forcing 0/False) are
+accumulated behaviour that `tests/parity/test_planner_emit_parity.py` pins byte
+for byte -- not something to re-derive.
+
+`resolve_soc_path` itself is defined in the leaf module `tan.soc_ref` (ADR-0028
+Task 2 follow-up) and re-exported here, so `tan.model.targets` can use it
+without pulling in this package's own import-time metadata reads.
+"""
+
+from __future__ import annotations
+
+import functools
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+from tan.soc_ref import resolve_soc_path  # noqa: F401  (re-export: see below)
+
+# ---------------------------------------------------------------------
+# SKU-family mapping
+# ---------------------------------------------------------------------
+#
+# The SKU-prefix -> family-dir map is a small, pure derivation with no
+# second on-disk source, so we keep it inline.  When a new SoM family
+# lands, add the entry here + update the schema's `som.sku` pattern.
+# (The silicon -> Kconfig mapping, by contrast, lives in the versioned
+# registry below -- see silicon_to_kconfig().)
+
+_SKU_FAMILY = re.compile(r"^E1M-(AEN|V2N|V2M)")
+
+
+def _sku_family(sku: str) -> str:
+    """Return the SoM family directory name for a SKU string."""
+    m = _SKU_FAMILY.match(sku)
+    if m is None:
+        raise ValueError(f"unrecognised SoM SKU pattern: {sku}")
+    return {"AEN": "aen", "V2N": "v2n", "V2M": "v2n-m1"}[m.group(1)]
+
+
+# Silicon ref -> Zephyr SoC-select Kconfig symbol.
+#
+# This is NOT a hand-maintained table: the symbol is computed from the
+# ref, and the single source of the allowlist is the versioned registry
+# at metadata/registries/silicon-kconfig.json.
+
+
+@functools.lru_cache(maxsize=None)
+def _load_silicon_kconfig(metadata_root: Path) -> tuple[str, frozenset[str]]:
+    """Load (socSymbolPrefix, knownSilicon) from *metadata_root*'s versioned
+    registry. Cache is keyed on *metadata_root* so a second root in the
+    same process doesn't silently reuse the first root's table (#1485)."""
+    registry = Path(metadata_root) / "registries" / "silicon-kconfig.json"
+    data = json.loads(registry.read_text(encoding="utf-8"))
+    return data["socSymbolPrefix"], frozenset(data["knownSilicon"])
+
+
+def silicon_to_kconfig(silicon: str | None, metadata_root: Path) -> str | None:
+    """Return the Zephyr Kconfig symbol that selects *silicon*, or ``None``.
+
+    The symbol is computed as ``socSymbolPrefix + ref.upper().replace(':','_')``;
+    e.g. ``alif:ensemble:e7`` -> ``ALP_SOC_ALIF_ENSEMBLE_E7``.  ``None`` is
+    returned for any ref not in the registry allowlist (so an accelerator
+    such as ``deepx:dx:m1`` -- or an unknown ref -- emits no CONFIG line).
+
+    *metadata_root* is REQUIRED -- callers must pass the project's own
+    ``project.effective_metadata_root()``, never a module default: a
+    silent fall-through to the SDK's own in-tree registry regardless of a
+    project's `--metadata-root` override was alp-sdk#1485's exact defect,
+    and the module-level `SILICON_KCONFIG_REGISTRY` constant this replaces
+    was the last site in tan's planner still shaped that way (tan-cli#868).
+    """
+    if silicon is None:
+        return None
+    prefix, known = _load_silicon_kconfig(metadata_root)
+    if silicon not in known:
+        return None
+    return prefix + silicon.upper().replace(":", "_")
+
+
+# resolve_soc_path() moved to the leaf module `tan.soc_ref` (imported above)
+# so that `tan.model.targets` can use it without pulling in `tan.planner`'s
+# import-time metadata reads. Re-exported here (rather than inlined) so this
+# module's own four call sites, and every external `from
+# tan.planner.som_metadata import resolve_soc_path`, keep working unchanged.
+
+
+def _resolve_silicon_variant(
+    sku_preset: dict[str, Any],
+    metadata_root: Path,
+) -> dict[str, Any] | None:
+    """Resolve a SoM preset to its matching SoC-JSON variant entry.
+
+    Two paths, in order of preference:
+
+    1. Forward: the SoM preset's top-level `silicon_variant:` field
+       names the exact `variants[].order_code` to pick out.
+    2. Reverse fallback: scan the SoC JSON's `variants[]` for one
+       whose `alp_module_skus` array contains the SoM SKU.
+
+    Returns the matched variant dict (with keys order_code, package,
+    mram_mb, sram_kb, optional_features, ...) or None when neither
+    path resolves (the SoC JSON has no variant for this SKU AND the
+    preset declares no silicon_variant, or declares `silicon_variant:
+    TBD` per the no-inventing-values rule).
+    """
+    soc_path = resolve_soc_path(sku_preset.get("silicon"), metadata_root)
+    if soc_path is None or not soc_path.is_file():
+        return None
+    soc_spec = json.loads(soc_path.read_text(encoding="utf-8"))
+    variants = soc_spec.get("variants") or []
+
+    declared = sku_preset.get("silicon_variant")
+    # alp-sdk #1048: case/whitespace-insensitive TBD match ("tbd", "Tbd",
+    # " TBD " all match) -- a bare `!= "TBD"` let an all-lowercase
+    # `silicon_variant: tbd` read as a real order_code instead of being
+    # dropped as the placeholder it was hand-typed to mean.
+    declared_is_tbd = isinstance(declared, str) and declared.strip().upper() == "TBD"
+    if declared and not declared_is_tbd:
+        for v in variants:
+            if v.get("order_code") == declared:
+                return v
+        # Forward declared but not found -- noisy fall-through to reverse lookup.
+
+    sku = sku_preset.get("sku")
+    if sku:
+        for v in variants:
+            if sku in (v.get("alp_module_skus") or []):
+                return v
+    return None
+
+
+def resolve_memory_map(
+    sku_preset: dict[str, Any],
+    metadata_root: Path,
+) -> list[dict[str, Any]]:
+    """Derive the effective memory-region table for a SoM.
+
+    Precedence:
+      1. If the SoM preset declares `memory_map:` explicitly (used
+         only for non-stock partitioning -- e.g. reserving SRAM for
+         a hardware secure enclave outside the default layout), it
+         wins verbatim.
+      2. Otherwise the loader derives the region list from the SoC
+         JSON variant resolved via `silicon_variant:`. Each named
+         SRAM bank becomes one region; the MRAM becomes one region.
+         Per-core TCM banks (SRAM names ending `_<core>_(ITCM|DTCM)`)
+         carry `accessible_from: [<core>]`; un-suffixed banks are
+         shared across every core declared in the variant's parent
+         SoC `cores[]` list.
+
+    The returned dicts have the keys defined by the memory_region
+    schema: `name`, `size_kib`, `accessible_from`, `cacheable`,
+    `write_authority` (always set explicitly on derived rows) (plus
+    optional `base` only when the SoM preset's override declares one
+    -- silicon-default bases stay unset, so downstream emitters know
+    to use the silicon's defaults).
+
+    Returns an empty list when the silicon_variant cannot be resolved
+    (e.g. a preset with `silicon_variant: TBD`) -- callers should treat
+    that as "memory layout pending the HW-config writeup".
+    """
+    declared = sku_preset.get("memory_map")
+    if declared:
+        # SoM-side override wins; the loader trusts the maintainer.
+        return list(declared)
+
+    variant = _resolve_silicon_variant(sku_preset, metadata_root)
+    if not variant:
+        return []
+
+    # Re-load the SoC JSON to grab the cores[] topology (the variant
+    # dict alone doesn't carry per-core ids).
+    soc_path = resolve_soc_path(sku_preset.get("silicon"), metadata_root)
+    if soc_path is None or not soc_path.is_file():
+        return []
+    soc_spec = json.loads(soc_path.read_text(encoding="utf-8"))
+    soc_cores = [c.get("id") for c in soc_spec.get("cores", []) if c.get("id")]
+
+    regions: list[dict[str, Any]] = []
+
+    # Prefer SoC-level fixed memory_regions when present — these carry
+    # authoritative base addresses (e.g. RZ/V2N OCRAM at 0x00010000).
+    soc_memory_regions = soc_spec.get("memory_regions")
+    if soc_memory_regions:
+        # Every SoC-level region is RAM (no SoC declares a flash window
+        # here), so each is runtime-writable by the application.
+        return [{**r, "write_authority": "customer_runtime"}
+                for r in soc_memory_regions]
+
+    # MRAM as one region (size in KiB; mram_mb -> *1024).
+    mram_mb = variant.get("mram_mb")
+    if isinstance(mram_mb, (int, float)) and mram_mb > 0:
+        regions.append({
+            "name": "mram_main",
+            "size_kib": int(mram_mb * 1024),
+            "accessible_from": list(soc_cores),
+            "cacheable": True,
+            # The whole-device alias spans MCUboot/ATOC/slot0/storage
+            # rows of differing authority, so no single value is true;
+            # `composite` is the schema's explicit word for that
+            # (ADR-0034 clause 4: stated, never defaulted).
+            "write_authority": "composite",
+        })
+
+    # SRAM banks. Per-core TCM banks get `accessible_from: [<core>]`;
+    # the rest are shared across every core in the SoC.
+    sram_banks = variant.get("sram_banks_kb") or {}
+    for bank_name, size_kib in sram_banks.items():
+        if not isinstance(size_kib, int) or size_kib <= 0:
+            continue
+        # Detect per-core TCM by suffix.
+        accessible: list[str] = list(soc_cores)
+        for core_id in soc_cores:
+            suffix_token = f"_{core_id.upper()}_"
+            if suffix_token in bank_name.upper():
+                accessible = [core_id]
+                break
+        # TCMs are typically non-cacheable; bulk SRAM is cacheable.
+        is_tcm = "ITCM" in bank_name.upper() or "DTCM" in bank_name.upper()
+        regions.append({
+            "name": bank_name.lower(),
+            "size_kib": int(size_kib),
+            "accessible_from": accessible,
+            "cacheable": not is_tcm,
+            # SRAM/TCM is RAM: runtime-writable by the application.
+            "write_authority": "customer_runtime",
+        })
+
+    return regions
+
+
+def resolve_capabilities(
+    sku_preset: dict[str, Any],
+    metadata_root: Path,
+) -> dict[str, Any]:
+    """Compose silicon capabilities from the SoC JSON with SoM-side overrides/extensions.
+
+    Returns a dict where SoM-declared keys override SoC-declared defaults for the same key
+    (e.g., V2N's `cau: true` via the GD32 bridge overrides the RZ/V2N silicon's `cau: false`).
+    Keys that exist on only one side are passed through unchanged.
+
+    Precedence:
+      1. Resolve the SoC ref via the ``silicon:`` field (same path used by
+         ``_resolve_silicon_variant`` and ``resolve_memory_map``).
+      2. Read ``soc["capabilities"]`` (defaults to ``{}`` when absent --
+         older SoC JSONs that pre-date this field continue to work).
+      3. Read ``sku_preset.get("capabilities", {})``.
+      4. Merge; SoM-side wins on key collision so that SoM add-on chips /
+         GD32 bridge capabilities can override the host silicon's defaults.
+      5. Apply the SKU's ``silicon_capabilities.unpopulated`` RESTRICTION
+         list: each listed silicon capability is forced to 0 (count) /
+         ``False`` (flag).  A SKU can only remove what the silicon offers
+         (enforced by scripts/validate_metadata.py); presets without the
+         field keep the full silicon capability set.
+    """
+    soc_path = resolve_soc_path(sku_preset.get("silicon"), metadata_root)
+    soc_caps: dict[str, Any] = {}
+    if soc_path is not None and soc_path.is_file():
+        soc_spec = json.loads(soc_path.read_text(encoding="utf-8"))
+        soc_caps = soc_spec.get("capabilities") or {}
+
+    som_caps: dict[str, Any] = sku_preset.get("capabilities") or {}
+
+    # SoM side wins on collision (bridge / add-on overrides silicon default).
+    merged: dict[str, Any] = {**soc_caps, **som_caps}
+
+    # GPU: a per-die fact (variants[].optional_features.gpu_mali_g31, the one
+    # source) -- the SoC-level capabilities block cannot carry it because
+    # four of the eight RZ/V2N dies are fused without the Mali-G31.
+    variant = _resolve_silicon_variant(sku_preset, metadata_root)
+    mali = ((variant or {}).get("optional_features") or {}).get("gpu_mali_g31")
+    if mali is not None:
+        merged["gpu2d"] = bool(mali)
+
+    # SKU-level restriction: capabilities the silicon offers but this SKU
+    # leaves unpopulated (per-SKU granularity -- one family, many SKUs).
+    # Preserve the value class so count-style caps (ethos_u55_count, ...)
+    # restrict to 0 and flag-style caps restrict to False.
+    for name in som_unpopulated_capabilities(sku_preset):
+        base = soc_caps.get(name)
+        if isinstance(base, int) and not isinstance(base, bool):
+            merged[name] = 0
+        else:
+            merged[name] = False
+    return merged
+
+
+def som_unpopulated_capabilities(sku_preset: dict[str, Any]) -> list[str]:
+    """Return the SKU's `silicon_capabilities.unpopulated` list (or []).
+
+    Single accessor for the per-SKU capability RESTRICTION block so this
+    module (`resolve_capabilities`), the emitters (`kconfig.py`, which passes
+    `-DALP_SOM_<TOKEN>` only for restricted SKUs) and alp-sdk's header
+    generator (`gen_soc_caps.py`) agree on where the field lives.
+    """
+    block = sku_preset.get("silicon_capabilities") or {}
+    if not isinstance(block, dict):
+        return []
+    names = block.get("unpopulated") or []
+    if not isinstance(names, list):
+        return []
+    return [str(n) for n in names]

tan/planner/strict_loaders.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""Duplicate-key-rejecting YAML/JSON loaders (alp-sdk issue #1127).
+
+RELOCATED (was alp-sdk `scripts/strict_loaders.py`, a top-level module outside
+`scripts/alp_orchestrate/`, hand-ported the same way `zephyr_board.py` /
+`project_loader.py` were -- named in `HAND_PORT_SOURCES` (not `PINNED_HASHES`,
+since it has no `scripts/alp_orchestrate/` counterpart to pin by filename), but
+its staleness hash lives in its OWN `STRICT_LOADERS_HASH` /
+`STRICT_LOADERS_PINNED_SDK_COMMIT` pair, not `HAND_PORT_HASHES` -- see
+`test_planner_relocation_freshness.py`'s STRICT_LOADERS_* comment for why).
+
+`yaml.safe_load` and `json.loads` silently keep only the LAST value of a
+duplicated mapping key -- `yaml.safe_load("som: a\\nsom: b\\n")` returns
+`{'som': 'b'}` with no error or warning. In a hand-edited board.yaml,
+SoM preset, or chip/SoC manifest, a duplicated key is invisible in a
+diff and silently drops hardware configuration. `strict_yaml_load()`
+and `strict_json_loads()` are the one place `tan/planner/loader.py`'s
+`_load_yaml` / `_load_json` route through instead of the raw stdlib
+loaders, so a board.yaml `tan build` accepts is the same board.yaml
+`tan validate` (which shells the SDK validator, itself routed through
+alp-sdk's own copy of this module) accepts.
+
+`fast_safe_load()` is the lenient sibling: exactly `yaml.safe_load`
+semantics (duplicate keys keep the last value) for hot readers that do not
+opt into rejection (`sdk_compat.load_family_table`). Both YAML paths parse
+with libyaml's C parser whenever PyYAML was built with it, and fall back to
+the pure-Python one otherwise (alp-sdk #2328).
+
+Zero dependencies beyond PyYAML on purpose, same rationale as alp-sdk's
+copy: this is a leaf every metadata-ingestion boundary can import without
+creating a cycle.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+class DuplicateKeyError(ValueError):
+    """Raised when a YAML/JSON mapping repeats a key."""
+
+
+def _no_duplicates_mapping_constructor(
+    loader: yaml.constructor.SafeConstructor, node: yaml.MappingNode
+) -> dict[str, Any]:
+    # Duplicate-key detection must run on the node's OWN explicit pairs
+    # BEFORE `flatten_mapping()` splices in `<<: *anchor` merge-key
+    # pairs -- otherwise a spec-legal merge override (the anchor
+    # supplies a default, a sibling explicit key overrides it) reads as
+    # the same key twice and gets rejected as a duplicate, when the
+    # explicit value should just win, exactly like the stdlib loader
+    # does. Scanning only the node's own pairs also leaves two merge
+    # SOURCES quietly disagreeing on a key to the normal YAML
+    # first-wins merge semantics, not a hard error -- this loader's job
+    # is to catch a literal duplicate key typed twice in one mapping.
+    seen: set[Any] = set()
+    for key_node, _value_node in node.value:
+        if key_node.tag == "tag:yaml.org,2002:merge":
+            continue
+        key = loader.construct_object(key_node, deep=True)
+        if key in seen:
+            mark = key_node.start_mark
+            raise DuplicateKeyError(
+                f"duplicate key {key!r} at line {mark.line + 1}, "
+                f"column {mark.column + 1}"
+            )
+        seen.add(key)
+
+    loader.flatten_mapping(node)
+    mapping: dict[str, Any] = {}
+    for key_node, value_node in node.value:
+        key = loader.construct_object(key_node, deep=True)
+        mapping[key] = loader.construct_object(value_node, deep=True)
+    return mapping
+
+
+# libyaml's C parser when PyYAML was built with it (its wheels are), the
+# pure-Python one otherwise. Same SafeConstructor either way, so the same
+# values and the same duplicate-key rejection -- only ~10x faster to parse
+# (alp-sdk #2328).
+_SAFE_LOADER = getattr(yaml, "CSafeLoader", yaml.SafeLoader)
+
+
+class _StrictLoader(_SAFE_LOADER):
+    pass
+
+
+_StrictLoader.add_constructor(
+    yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _no_duplicates_mapping_constructor
+)
+
+
+def strict_yaml_load(text: str, source: str | Path = "<string>") -> Any:
+    """`yaml.safe_load`, but raise `DuplicateKeyError` on a repeated mapping key."""
+    try:
+        return yaml.load(text, Loader=_StrictLoader)
+    except DuplicateKeyError as e:
+        raise DuplicateKeyError(f"{source}: {e}") from e
+
+
+def fast_safe_load(text: str) -> Any:
+    """Exactly `yaml.safe_load(text)`, parsed by libyaml when available.
+
+    For the hot metadata readers that want stdlib semantics (duplicate keys
+    keep the last value) rather than `strict_yaml_load`'s rejection.
+    """
+    return yaml.load(text, Loader=_SAFE_LOADER)
+
+
+def _no_duplicates_object_pairs_hook(
+    pairs: list[tuple[str, Any]],
+) -> dict[str, Any]:
+    mapping: dict[str, Any] = {}
+    for key, value in pairs:
+        if key in mapping:
+            raise DuplicateKeyError(f"duplicate key {key!r}")
+        mapping[key] = value
+    return mapping
+
+
+def strict_json_loads(text: str, source: str | Path = "<string>") -> Any:
+    """`json.loads`, but raise `DuplicateKeyError` on a repeated object key."""
+    try:
+        return json.loads(text, object_pairs_hook=_no_duplicates_object_pairs_hook)
+    except DuplicateKeyError as e:
+        raise DuplicateKeyError(f"{source}: {e}") from e