dau-core 0.1.1__py3-none-any.whl

dau_core/__init__.py

@@ -0,0 +1,118 @@
+"""dau-core: the tile palette, the device contracts and the goldens.
+
+Exports load on first use (:mod:`dau_core._lazy`). Importing this package
+must not import ccflow and its stack, because hydra's searchpath plugin
+imports ``dau_core.config`` to discover what is installed, and discovery
+wants none of these names.
+"""
+
+from typing import TYPE_CHECKING
+
+from ._lazy import lazy_exports
+
+if TYPE_CHECKING:
+    from .capabilities import DeviceCapabilities, DmaConstraints, OperatorCapability
+    from .capability_contract import (
+        CAPABILITY_CONTRACT_SCHEMA,
+        CapabilityContract,
+        LaneCapability,
+        TileCapability,
+        capability_contract,
+        opcode_bitmap,
+        tile_capability,
+    )
+    from .design import DauDesign, OperatorUnit, UnitKind, format_units, parse_units
+    from .operators import (
+        EmptyInputError,
+        UnsupportedAggregationError,
+        aggregate_values,
+        pack_splitters,
+        synthetic_values,
+    )
+    from .registers import (
+        CURRENT_REGISTER_MAP_VERSION,
+        DAU_MAGIC_WORD,
+        DEFAULT_STREAM_JOB_REGISTER_CONTRACT,
+        JobControlFlag,
+        JobStatusFlag,
+        RegisterBlockSnapshot,
+        RegisterMapVersion,
+        RegisterOffset,
+        RegisterWriteError,
+        ResetScope,
+        StreamJobRegisterContract,
+        StreamProtocolVersion,
+    )
+    from .stream import (
+        DEFAULT_RECORD_BATCH_LIMITS,
+        BatchHeader,
+        ColumnDescriptor,
+        GroupRole,
+        InvalidStreamError,
+        LogicalType,
+        OperationCode,
+        OperationDescriptor,
+        PolicyFlag,
+        RecordBatchLimits,
+        StreamAggregationErrorCode,
+        StreamFlag,
+        validate_record_batch,
+    )
+
+__version__ = "0.1.1"
+
+# {submodule: the names it exports}. ``__all__`` and the lazy resolver both
+# come from this one declaration, so a name cannot be in one and missing
+# from the other; ``tests/test_lazy_exports.py`` checks it against the
+# TYPE_CHECKING block above and resolves every name.
+_EXPORTS = {
+    "capabilities": ("DeviceCapabilities", "DmaConstraints", "OperatorCapability"),
+    "capability_contract": (
+        "CAPABILITY_CONTRACT_SCHEMA",
+        "CapabilityContract",
+        "LaneCapability",
+        "TileCapability",
+        "capability_contract",
+        "opcode_bitmap",
+        "tile_capability",
+    ),
+    "design": ("DauDesign", "OperatorUnit", "UnitKind", "format_units", "parse_units"),
+    "operators": (
+        "EmptyInputError",
+        "UnsupportedAggregationError",
+        "aggregate_values",
+        "pack_splitters",
+        "synthetic_values",
+    ),
+    "registers": (
+        "CURRENT_REGISTER_MAP_VERSION",
+        "DAU_MAGIC_WORD",
+        "DEFAULT_STREAM_JOB_REGISTER_CONTRACT",
+        "JobControlFlag",
+        "JobStatusFlag",
+        "RegisterBlockSnapshot",
+        "RegisterMapVersion",
+        "RegisterOffset",
+        "RegisterWriteError",
+        "ResetScope",
+        "StreamJobRegisterContract",
+        "StreamProtocolVersion",
+    ),
+    "stream": (
+        "DEFAULT_RECORD_BATCH_LIMITS",
+        "BatchHeader",
+        "ColumnDescriptor",
+        "GroupRole",
+        "InvalidStreamError",
+        "LogicalType",
+        "OperationCode",
+        "OperationDescriptor",
+        "PolicyFlag",
+        "RecordBatchLimits",
+        "StreamAggregationErrorCode",
+        "StreamFlag",
+        "validate_record_batch",
+    ),
+}
+
+__getattr__, __all__ = lazy_exports(__name__, _EXPORTS)

dau_core/_lazy.py

@@ -0,0 +1,46 @@
+"""Import a package's exports on first use.
+
+Every model in this stack is a ccflow model, and importing ccflow imports
+csp, pandas and the rest of its stack: about 0.9 s. A package whose
+``__init__`` imports its model modules therefore pays all of it just to be
+imported -- including when hydra's searchpath plugin imports the config
+package to discover what is installed, which is the slow step in composing
+the registry and wants none of those names.
+
+So a package declares which submodule provides each exported name and this
+builds the ``__getattr__`` that imports the one submodule the name needs.
+The names stay importable exactly as before (``from dau_core import
+RegisterOffset``), and a type checker reads them from the ``TYPE_CHECKING``
+block the package keeps beside the declaration.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from importlib import import_module
+
+__all__ = ("lazy_exports",)
+
+
+def lazy_exports(package: str, exports: dict[str, tuple[str, ...]]) -> tuple[Callable[[str], object], list[str]]:
+    """A module ``__getattr__`` and ``__all__`` for ``{submodule: names}``.
+
+    Returns both so the two cannot disagree: a name exported by nothing is
+    not reachable, and a name declared here and missing from its submodule
+    raises on first access rather than at import, which a guard test is what
+    turns back into an import-time error.
+    """
+    providers = {}
+    for module, names in exports.items():
+        for name in names:
+            if name in providers:
+                raise ValueError(f"{package}: {name!r} is exported by both {providers[name]!r} and {module!r}")
+            providers[name] = module
+
+    def getattr_(name: str) -> object:
+        module = providers.get(name)
+        if module is None:
+            raise AttributeError(f"module {package!r} has no attribute {name!r}")
+        return getattr(import_module(f"{package}.{module}"), name)
+
+    return getattr_, sorted(providers)

dau_core/capabilities.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .registers import RegisterMapVersion, StreamProtocolVersion
+from .stream import LogicalType, OperationCode
+
+# DAU platform variant 1 (dpv1) identity — the single home for these
+# values; drivers, orchestration, and tooling import them rather than
+# repeating literals. The wire identity word is the platform-variant name
+# "DPV1" (the board's own product name is an implementation detail); shells
+# report it in the PLATFORM_ID register.
+DPV1_PLATFORM_ID = "DPV1"
+XILINX_VENDOR_ID = 0x10EE
+DPV1_DEVICE_ID = 0x7011
+DPV1_PCI_ID = f"{XILINX_VENDOR_ID:04x}:{DPV1_DEVICE_ID:04x}"
+
+
+@dataclass(frozen=True)
+class DmaConstraints:
+    alignment_bytes: int
+    descriptor_bytes: int
+    max_transfer_bytes: int
+    read_channels: int
+    write_channels: int
+
+    def __post_init__(self) -> None:
+        if self.alignment_bytes <= 0:
+            raise ValueError("alignment_bytes must be positive")
+        if self.descriptor_bytes <= 0:
+            raise ValueError("descriptor_bytes must be positive")
+        if self.max_transfer_bytes <= 0:
+            raise ValueError("max_transfer_bytes must be positive")
+        if self.read_channels < 0 or self.write_channels < 0:
+            raise ValueError("DMA channel counts cannot be negative")
+
+    def is_aligned(self, value: int) -> bool:
+        return value % self.alignment_bytes == 0
+
+    @property
+    def max_aligned_transfer_bytes(self) -> int:
+        return self.max_transfer_bytes - (self.max_transfer_bytes % self.alignment_bytes)
+
+
+@dataclass(frozen=True)
+class OperatorCapability:
+    opcode: OperationCode
+    input_types: tuple[LogicalType, ...]
+    output_type: LogicalType
+    supports_group_by: bool
+
+    def supports(self, opcode: OperationCode, logical_type: LogicalType) -> bool:
+        return self.opcode == opcode and logical_type in self.input_types
+
+
+@dataclass(frozen=True)
+class DeviceCapabilities:
+    """What a probed device advertises.
+
+    ``operators`` are hardware capabilities — opcodes some tile executes in
+    HDL. ``host_operators`` are protocol-level capabilities the host derives
+    from hardware results (AVERAGE = host divide of SUM/COUNT); they must
+    never be submitted to the device as operations.
+    """
+
+    platform_id: str
+    pci_vendor_id: int
+    pci_device_id: int
+    register_map_version: RegisterMapVersion
+    stream_protocol_version: StreamProtocolVersion
+    dma: DmaConstraints
+    operators: tuple[OperatorCapability, ...]
+    host_operators: tuple[OperatorCapability, ...] = ()
+
+    def supports(self, opcode: OperationCode, logical_type: LogicalType) -> bool:
+        return any(operator.supports(opcode, logical_type) for operator in self.operators)
+
+    def supports_host_computed(self, opcode: OperationCode, logical_type: LogicalType) -> bool:
+        return any(operator.supports(opcode, logical_type) for operator in self.host_operators)

dau_core/capability_contract.py

@@ -0,0 +1,274 @@
+"""Composition-derived capability contract: the single source of truth for
+what a generated shell can do.
+
+``capability_contract(spec)`` derives the contract from the same
+``ScanCompositionSpec`` that generates the shell top, using the registry's
+declared HDL facts (opcodes, input types, count ports, contract kinds, row
+capacities). Three consumers read it:
+
+- the identity register block (``operator_bitmap`` / ``lane_count`` /
+  ``host_opcode_bitmap`` / ``sort_capacity`` words, register map 0.2);
+- the shell-build artlink manifest (``to_mapping()``, JSON-able);
+- host planners (``to_design()`` for the pushdown scheduler,
+  driver capability decode via the manifest mapping).
+
+Hardware truth vs protocol truth is explicit: ``operator_bitmap`` carries
+only OperationCodes some tile executes in HDL (MIN/MAX/SUM/COUNT);
+AVERAGE is never a hardware opcode — when a tile provides both SUM and
+COUNT it appears in ``host_opcodes`` (the host divides the results at
+protocol level), never in ``opcodes``.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from pydantic import Field
+
+from .composition import ScanCompositionSpec, TileSpec
+from .cores import ContractKind
+from .design import DauDesign, OperatorUnit, UnitKind
+from .models import CheckedModel
+from .registers import CURRENT_REGISTER_MAP_VERSION, RegisterMapVersion
+from .registry import core
+from .stream import LogicalType, OperationCode
+
+CAPABILITY_CONTRACT_SCHEMA = 1
+
+
+def opcode_bitmap(opcodes: tuple[OperationCode, ...]) -> int:
+    """The register encoding: bit ``1 << opcode`` per OperationCode."""
+    bitmap = 0
+    for opcode in opcodes:
+        bitmap |= 1 << int(opcode)
+    return bitmap
+
+
+class TileCapability(CheckedModel):
+    """One operator tile instance as the contract describes it: registry
+    identity, stream contract, HDL opcode truth, and instance config."""
+
+    core: str
+    module: str
+    contract_kind: ContractKind | None
+    count_port: str | None
+    opcodes: tuple[OperationCode, ...] = ()
+    input_types: tuple[LogicalType, ...] = ()
+    unit_kind: UnitKind | None = None
+    row_capacity: int | None = None
+    config: tuple[tuple[str, str], ...] = ()
+
+    @property
+    def host_opcodes(self) -> tuple[OperationCode, ...]:
+        """Protocol-level opcodes the host can compute from this tile's
+        hardware results: AVERAGE = SUM/COUNT divided host-side. Never
+        hardware opcodes and never advertised as such."""
+        if OperationCode.SUM in self.opcodes and OperationCode.COUNT in self.opcodes:
+            return (OperationCode.AVERAGE,)
+        return ()
+
+    def to_mapping(self) -> dict[str, Any]:
+        return {
+            "core": self.core,
+            "module": self.module,
+            "contract_kind": self.contract_kind.value if self.contract_kind is not None else None,
+            "count_port": self.count_port,
+            "opcodes": [opcode.name.lower() for opcode in self.opcodes],
+            "input_types": [logical_type.name.lower() for logical_type in self.input_types],
+            "unit_kind": self.unit_kind.value if self.unit_kind is not None else None,
+            "row_capacity": self.row_capacity,
+            "config": dict(self.config),
+        }
+
+    @classmethod
+    def from_mapping(cls, mapping: Mapping[str, Any]) -> TileCapability:
+        return cls(
+            core=mapping["core"],
+            module=mapping["module"],
+            contract_kind=ContractKind(mapping["contract_kind"]) if mapping.get("contract_kind") else None,
+            count_port=mapping.get("count_port"),
+            opcodes=tuple(OperationCode[token.upper()] for token in mapping.get("opcodes", ())),
+            input_types=tuple(LogicalType[token.upper()] for token in mapping.get("input_types", ())),
+            unit_kind=UnitKind(mapping["unit_kind"]) if mapping.get("unit_kind") else None,
+            row_capacity=mapping.get("row_capacity"),
+            config=tuple(sorted((str(key), str(value)) for key, value in dict(mapping.get("config", {})).items())),
+        )
+
+
+class LaneCapability(CheckedModel):
+    """One scan lane: the terminal operator tile, its optional partition
+    filter, and the ordered mid-lane operator ``chain`` between them."""
+
+    tile: TileCapability
+    partition: TileCapability | None = None
+    chain: tuple[TileCapability, ...] = ()
+
+    def to_mapping(self) -> dict[str, Any]:
+        return {
+            "tile": self.tile.to_mapping(),
+            "partition": self.partition.to_mapping() if self.partition is not None else None,
+            "chain": [stage.to_mapping() for stage in self.chain],
+        }
+
+    @classmethod
+    def from_mapping(cls, mapping: Mapping[str, Any]) -> LaneCapability:
+        partition = mapping.get("partition")
+        return cls(
+            tile=TileCapability.from_mapping(mapping["tile"]),
+            partition=TileCapability.from_mapping(partition) if partition else None,
+            chain=tuple(TileCapability.from_mapping(stage) for stage in mapping.get("chain", ())),
+        )
+
+
+class CapabilityContract(CheckedModel):
+    """The advertised capability truth of one composed design.
+
+    A config-data model (it is what the shell-build manifest carries and
+    what a planner matches against), so a ccflow model like the rest of the
+    registry's declarations; ``to_mapping``/``from_mapping`` remain the
+    manifest serialization, unchanged, because manifests already on disk
+    carry that form."""
+
+    name: str
+    lanes: tuple[LaneCapability, ...]
+    partitioner: TileCapability | None = None
+    register_map_version: RegisterMapVersion = Field(default_factory=lambda: CURRENT_REGISTER_MAP_VERSION)
+    # whether the lanes' chain config ports are runtime registers the host
+    # rewrites per job, so a planner matches this design structurally
+    # (enough stages of the right kinds) instead of config for config. A
+    # fact about the bitstream, carried in its manifest; never a caller's
+    # declaration.
+    reconfigurable: bool = False
+
+    @property
+    def lane_count(self) -> int:
+        return len(self.lanes)
+
+    @property
+    def hardware_opcodes(self) -> tuple[OperationCode, ...]:
+        """Union of the lanes' HDL opcodes (terminal tiles and chain
+        stages), ordered by opcode value."""
+        opcodes = {opcode for lane in self.lanes for tile in (lane.tile, *lane.chain) for opcode in tile.opcodes}
+        return tuple(sorted(opcodes, key=int))
+
+    @property
+    def host_opcodes(self) -> tuple[OperationCode, ...]:
+        """Protocol-level opcodes derivable host-side (AVERAGE when a tile
+        provides both SUM and COUNT); explicitly not hardware opcodes."""
+        opcodes = {opcode for lane in self.lanes for tile in (lane.tile, *lane.chain) for opcode in tile.host_opcodes}
+        return tuple(sorted(opcodes, key=int))
+
+    @property
+    def operator_bitmap(self) -> int:
+        return opcode_bitmap(self.hardware_opcodes)
+
+    @property
+    def host_opcode_bitmap(self) -> int:
+        return opcode_bitmap(self.host_opcodes)
+
+    @property
+    def sort_capacity(self) -> int:
+        capacities = [lane.tile.row_capacity or 0 for lane in self.lanes if lane.tile.unit_kind == UnitKind.SORT]
+        return max(capacities, default=0)
+
+    def to_design(self) -> DauDesign:
+        """The operator inventory this contract advertises: identical lane
+        tiles that contribute a design unit collapse into one counted unit."""
+        counts: dict[TileCapability, int] = {}
+        for lane in self.lanes:
+            if lane.tile.unit_kind is None:
+                continue
+            key = TileCapability(
+                core=lane.tile.core,
+                module=lane.tile.module,
+                contract_kind=lane.tile.contract_kind,
+                count_port=lane.tile.count_port,
+                opcodes=lane.tile.opcodes,
+                input_types=lane.tile.input_types,
+                unit_kind=lane.tile.unit_kind,
+                row_capacity=lane.tile.row_capacity,
+            )
+            counts[key] = counts.get(key, 0) + 1
+        unit_list: list[OperatorUnit] = []
+        for tile, count in counts.items():
+            if tile.unit_kind is not None:
+                unit_list.append(OperatorUnit(kind=tile.unit_kind, count=count, opcodes=tile.opcodes, input_types=tile.input_types))
+        units = tuple(unit_list)
+        return DauDesign(name=self.name, units=units)
+
+    def to_mapping(self) -> dict[str, Any]:
+        """JSON-able serialization for the shell-build manifest."""
+        return {
+            "contract": "dau-capability-contract",
+            "schema": CAPABILITY_CONTRACT_SCHEMA,
+            "name": self.name,
+            "register_map_version": f"{self.register_map_version.major}.{self.register_map_version.minor}",
+            "operator_bitmap": f"0x{self.operator_bitmap:08x}",
+            "host_opcode_bitmap": f"0x{self.host_opcode_bitmap:08x}",
+            "lane_count": self.lane_count,
+            "sort_capacity": self.sort_capacity,
+            "lanes": [lane.to_mapping() for lane in self.lanes],
+            "partitioner": self.partitioner.to_mapping() if self.partitioner is not None else None,
+            "reconfigurable": self.reconfigurable,
+        }
+
+    @classmethod
+    def from_mapping(cls, mapping: Mapping[str, Any]) -> CapabilityContract:
+        if mapping.get("contract") != "dau-capability-contract":
+            raise ValueError("not a dau capability contract mapping")
+        schema = mapping.get("schema")
+        if schema != CAPABILITY_CONTRACT_SCHEMA:
+            raise ValueError(f"unsupported capability contract schema {schema!r}")
+        major, _, minor = str(mapping["register_map_version"]).partition(".")
+        partitioner = mapping.get("partitioner")
+        return cls(
+            name=mapping["name"],
+            lanes=tuple(LaneCapability.from_mapping(lane) for lane in mapping.get("lanes", ())),
+            partitioner=TileCapability.from_mapping(partitioner) if partitioner else None,
+            register_map_version=RegisterMapVersion(int(major), int(minor)),
+            # absent in manifests written before the fact was advertised:
+            # every one of those designs binds its configs as literals
+            reconfigurable=bool(mapping.get("reconfigurable", False)),
+        )
+
+
+def tile_capability(tile: TileSpec | str) -> TileCapability:
+    """The registry-declared capability facts of one tile instance."""
+    spec = TileSpec(core=tile) if isinstance(tile, str) else tile
+    definition = core(spec.core)
+    return TileCapability(
+        core=definition.name,
+        module=definition.module,
+        contract_kind=definition.contract_kind,
+        count_port=definition.count_port,
+        opcodes=definition.opcodes,
+        input_types=definition.opcode_input_types,
+        unit_kind=definition.unit_kind,
+        row_capacity=definition.row_capacity,
+        config=tuple(sorted((str(key), str(value)) for key, value in spec.config.items())),
+    )
+
+
+def capability_contract(spec: ScanCompositionSpec) -> CapabilityContract:
+    """Derive the capability contract from the composition that generates
+    the shell top — same spec, same registry, one source of truth.
+
+    ``reconfigurable`` stays false: the generated walker binds every config
+    port to the literal the composition names, so no composed design has
+    runtime config registers yet. When the generator can bind them to
+    registers, the composition will say so and this is where the contract
+    reads it."""
+    spec = spec.check()
+    return CapabilityContract(
+        name=spec.name,
+        lanes=tuple(
+            LaneCapability(
+                tile=tile_capability(lane.tile),
+                partition=tile_capability(lane.partition) if lane.partition is not None else None,
+                chain=tuple(tile_capability(stage) for stage in lane.chain),
+            )
+            for lane in spec.lanes
+        ),
+        partitioner=tile_capability(spec.partitioner) if spec.partitioner is not None else None,
+    )