can-api-generator 1.0.0__py3-none-any.whl

can_api_generator/__about__.py

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: 2025-present Alexande Becker <nabla.becker@mailbox.org>
+#
+# SPDX-License-Identifier: LGPL-3.0-only
+__version__ = "1.0.0"

can_api_generator/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2025-present Alexande Becker <nabla.becker@mailbox.org>
+#
+# SPDX-License-Identifier: MIT

can_api_generator/cbdb_generator.py

@@ -0,0 +1,208 @@
+"""
+Cbdb_generator
+==============
+
+Compiles a KCD database into the binary descriptor cbdb consumed by the
+dynamic decoder running on a CAN-to-BLE bridge, plus a name -> signal_id
+map for clients that need to reference signals symbolically (e.g. the
+Android dashboard app sidecar).
+
+Unlike the static API path, the cbdb path has no node: the bridge is a bus
+observer, so every signal of every message is included and no
+direction filtering is applied.
+
+Cbdb format v1 (all little-endian):
+
+    header (48 B):  magic 4s = b"CDB1", format_version u16 = 1,
+                    msg_count u16, signal_count u16, reserved u16 = 0,
+                    cbdb_size u32 (total bytes incl. header),
+                    kcd_sha256 32s (hash of the KCD file bytes)
+    per message (8 B + signals):
+                    arb_id u32 (bit31 = extended-id flag), dlc u8,
+                    n_signals u8, reserved u16 = 0
+    per signal (20 B):
+                    signal_id u32 (FNV-1a of "<msg>.<sig>" resolved names),
+                    start_bit u8, bit_length u8, flags u8,
+                    mux_sel_idx u8 (index into this message's signal list,
+                    0xFF = not muxed), mux_value u32,
+                    scale f32, offset f32
+
+Messages are sorted ascending by the stored u32 arb_id key so the decoder
+can binary-search. Within a message, mux selector signals come first, then
+plain signals, then muxed signals, so a single decode pass has selector raw
+values available before evaluating mux membership.
+"""
+import hashlib
+import struct
+from pathlib import Path
+
+import can_db_model
+
+from .msg_processor import CAPIMessage
+from .utils import converrt_to_c_identifier
+
+CBDB_MAGIC = b"CDB1"
+CBDB_FORMAT_VERSION = 1
+
+HEADER_FMT = "<4sHHHHI32s"
+MSG_FMT = "<IBBH"
+SIG_FMT = "<IBBBBIff"
+
+FLAG_SIGNED = 1 << 0
+FLAG_WIRE_FLOAT = 1 << 1
+FLAG_MUXED = 1 << 2
+
+EXT_ID_FLAG = 1 << 31
+NO_MUX_SEL = 0xFF
+
+FNV1A32_OFFSET_BASIS = 0x811C9DC5
+FNV1A32_PRIME = 0x01000193
+
+# Node name used when constructing CAPIMessage objects for the cbdb path.
+# It matches no real node, so no direction filtering takes place and every
+# signal stays in the message.
+_OBSERVER_NODE = "__cbdb_observer__"
+
+
+class CbdbGenerationException(Exception):
+    pass
+
+
+def fnv1a32(text: str) -> int:
+    """FNV-1a 32-bit hash of the UTF-8 encoding of `text`."""
+    digest = FNV1A32_OFFSET_BASIS
+    for byte in text.encode("utf-8"):
+        digest ^= byte
+        digest = (digest * FNV1A32_PRIME) & 0xFFFFFFFF
+    return digest
+
+
+def _ordered_signals(msg: CAPIMessage) -> list:
+    """All signals of a message in cbdb order.
+
+    Mux selectors first, then plain signals, then muxed signals; original
+    KCD order is preserved within each group.
+    """
+    selectors = [s for s in msg.all_signals if s.is_multiplex]
+    plain = [s for s in msg.all_signals
+             if not s.is_multiplex and s.multiplexer is None]
+    muxed = [s for s in msg.all_signals
+             if not s.is_multiplex and s.multiplexer is not None]
+    return selectors + plain + muxed
+
+
+def _signal_flags(sig) -> int:
+    flags = 0
+    if sig.is_signed and not sig.is_float:
+        flags |= FLAG_SIGNED
+    if sig.is_float:
+        flags |= FLAG_WIRE_FLOAT
+    if sig.multiplexer is not None:
+        flags |= FLAG_MUXED
+    return flags
+
+
+def _arb_key(msg: CAPIMessage) -> int:
+    return msg.arb_id | (EXT_ID_FLAG if msg.is_extended else 0)
+
+
+def build_capi_messages(db_path: Path) -> list[CAPIMessage]:
+    """Load a KCD/YAML database and build node-less CAPIMessage objects."""
+    db = can_db_model.load(Path(db_path))
+    bus_name = db.bus.name if db.bus is not None else ""
+    msgs = [CAPIMessage(msg, api_name="cbdb", node_name=_OBSERVER_NODE,
+                        bus_name=bus_name, require_endpoints=False)
+            for msg in db.messages]
+    for msg in msgs:
+        for sig in msg.signals:
+            sig.resolve_muxed_signal_name()
+    return msgs
+
+
+def generate_cbdb(kcd_path: Path) -> tuple[bytes, dict[str, int]]:
+    """Compile `kcd_path` into (cbdb bytes, resolved-name -> signal_id map)."""
+    kcd_path = Path(kcd_path)
+    # Sort by the plain (masked) id: received frames carry no extended-id
+    # flag in their arbitration-id member, so the decoder binary-searches on
+    # the masked value. The flag bit in the stored key is metadata.
+    msgs = sorted(build_capi_messages(kcd_path), key=lambda m: m.arb_id)
+
+    id_map: dict[str, int] = {}
+    body = bytearray()
+    signal_count = 0
+    for msg in msgs:
+        signals = _ordered_signals(msg)
+        signal_count += len(signals)
+        body += struct.pack(MSG_FMT, _arb_key(msg), msg.length,
+                            len(signals), 0)
+        for sig in signals:
+            name = f"{converrt_to_c_identifier(msg.name)}.{sig.api_var_name}"
+            signal_id = fnv1a32(name)
+            if signal_id in id_map.values():
+                raise CbdbGenerationException(
+                    f"signal_id hash collision on {name}")
+            id_map[name] = signal_id
+            if sig.multiplexer is not None:
+                mux_sel_idx = next(
+                    i for i, s in enumerate(signals)
+                    if s.is_multiplex and s.name == sig.multiplexer)
+                mux_value = sig.mux_group
+            else:
+                mux_sel_idx = NO_MUX_SEL
+                mux_value = 0
+            body += struct.pack(SIG_FMT, signal_id, sig.start_bit,
+                                sig.length, _signal_flags(sig),
+                                mux_sel_idx, mux_value,
+                                float(sig.scale), float(sig.offset))
+
+    kcd_sha256 = hashlib.sha256(kcd_path.read_bytes()).digest()
+    cbdb_size = struct.calcsize(HEADER_FMT) + len(body)
+    header = struct.pack(HEADER_FMT, CBDB_MAGIC, CBDB_FORMAT_VERSION,
+                         len(msgs), signal_count, 0, cbdb_size, kcd_sha256)
+    return bytes(header + body), id_map
+
+
+def build_sidecar(kcd_path: Path) -> dict:
+    """The JSON sidecar consumed by client apps: the signal id map plus a
+    per-message description (arbitration id, dlc, signal membership) so a
+    client can label raw bus traffic against this database."""
+    kcd_path = Path(kcd_path)
+    _, id_map = generate_cbdb(kcd_path)
+    messages = {}
+    signal_info = {}
+    for msg in sorted(build_capi_messages(kcd_path), key=lambda m: m.arb_id):
+        msg_name = converrt_to_c_identifier(msg.name)
+        messages[msg_name] = {
+            "arb_id": msg.arb_id,
+            "extended": msg.is_extended,
+            "dlc": msg.length,
+            "signals": [f"{msg_name}.{sig.api_var_name}"
+                        for sig in _ordered_signals(msg)],
+        }
+        for sig in _ordered_signals(msg):
+            info = _signal_info(sig)
+            if info:
+                signal_info[f"{msg_name}.{sig.api_var_name}"] = info
+    return {
+        "format_version": CBDB_FORMAT_VERSION,
+        "kcd_sha256": hashlib.sha256(kcd_path.read_bytes()).hexdigest(),
+        "signals": id_map,
+        "messages": messages,
+        "signal_info": signal_info,
+    }
+
+
+def _signal_info(sig) -> dict:
+    """Presentation info for a signal: unit, physical range, and enum value
+    labels — only the fields the KCD actually defines. Additive sidecar data
+    (the binary cbdb and the `signals` id map are unchanged)."""
+    info = {}
+    if sig.unit:
+        info["unit"] = sig.unit
+    if sig.min is not None:
+        info["min"] = sig.min
+    if sig.max is not None:
+        info["max"] = sig.max
+    if sig.labels:
+        info["labels"] = {str(value): name for value, name in sig.labels.items()}
+    return info

can_api_generator/code_generator.py

@@ -0,0 +1,148 @@
+from can_api_generator.msg_processor import CAPIMessage
+from pathlib import Path
+import can_db_model
+
+
+def build_msgs_data_dicts_from_kcd(
+        db_path: Path, api_name: str, node_name: str):
+    """Load a KCD or YAML database and build CAPIMessage dicts for NODE_NAME.
+
+    Returns (list-of-message-dicts, CanDb). The path may be .kcd or .yaml/.yml;
+    can_db_model.load dispatches on the extension.
+    """
+    db = can_db_model.load(Path(db_path))
+    bus_name = db.bus.name if db.bus is not None else ""
+    msgs = [CAPIMessage(
+            msg,
+            api_name=api_name,
+            node_name=node_name,
+            bus_name=bus_name)
+            for msg in db.messages]
+    for msg in msgs:
+        for sig in msg.signals:
+            sig.resolve_muxed_signal_name()
+    return ([msg.to_dict() for msg in msgs], db)
+
+
+def generate_api_implementation(msgs: list[dict],
+                                jinja_env, api_name: str,
+                                with_encode_all_func: bool,
+                                generate_everything: bool,
+                                arb_id_member_name: str,
+                                length_member_name: str,
+                                payload_member_name: str,
+                                global_rx: bool,
+                                periodic_tx: bool,
+                                ti_compatible: bool = False,
+                                send_now: bool = False,
+                                double_buffered: bool = False):
+    header_template = jinja_env.get_template("header_template.jinja")
+    source_template = jinja_env.get_template("source_template.jinja")
+    canframe_template = jinja_env.get_template("canframe.jinja")
+    lib_name_struct_prefix = ''.join(map(lambda s: s.capitalize(),
+                                         api_name.split('_')))
+    header_code = header_template.render(
+        messages=msgs,
+        api_name=api_name,
+        encode_every_signal_func=with_encode_all_func,
+        generate_everything=generate_everything,
+        arb_id_member_name=arb_id_member_name,
+        length_member_name=length_member_name,
+        payload_member_name=payload_member_name,
+        global_rx=global_rx,
+        api_name_struct_prefix=lib_name_struct_prefix,
+        periodic_tx=periodic_tx,
+        send_now=send_now,
+        double_buffered=double_buffered,
+    )
+    source_code = source_template.render(
+        messages=msgs,
+        api_name = api_name,
+        header_file_name=api_name + ".h",
+        encode_every_signal_func=with_encode_all_func,
+        generate_everything=generate_everything,
+        arb_id_member_name=arb_id_member_name,
+        length_member_name=length_member_name,
+        payload_member_name=payload_member_name,
+        global_rx=global_rx,
+        api_name_struct_prefix=lib_name_struct_prefix,
+        periodic_tx=periodic_tx,
+        ti_compatible=ti_compatible,
+        send_now=send_now,
+        double_buffered=double_buffered,
+    )
+    canframe_header_code = canframe_template.render(
+        arb_id_member_name=arb_id_member_name,
+        length_member_name=length_member_name,
+        payload_member_name=payload_member_name
+    )
+    return header_code, source_code, canframe_header_code
+
+
+def generate_dyndec_implementation(jinja_env,
+                                   arb_id_member_name: str,
+                                   length_member_name: str,
+                                   payload_member_name: str,
+                                   header_file_name: str = "dyndec.h"):
+    """Render the KCD-independent dynamic-decoder module.
+
+    The module decodes frames at runtime against a descriptor cbdb compiled
+    by the `cbdb` path; it is generated once per firmware, not per database.
+    Returns (header_code, source_code, canframe_header_code).
+    """
+    header_template = jinja_env.get_template("dyndec_header.jinja")
+    source_template = jinja_env.get_template("dyndec_source.jinja")
+    canframe_template = jinja_env.get_template("canframe.jinja")
+    header_code = header_template.render()
+    source_code = source_template.render(
+        header_file_name=header_file_name,
+        arb_id_member_name=arb_id_member_name,
+        length_member_name=length_member_name,
+        payload_member_name=payload_member_name,
+    )
+    canframe_header_code = canframe_template.render(
+        arb_id_member_name=arb_id_member_name,
+        length_member_name=length_member_name,
+        payload_member_name=payload_member_name,
+    )
+    return header_code, source_code, canframe_header_code
+
+
+def generate_canframe_header(dest_path: Path, canframe_code: str):
+    with dest_path.open() as canframe_header:
+        canframe_header.write(canframe_code)
+
+
+def build_test_files_from_source_str(capi_header, capi_source,
+                                     canframe_header_code,
+                                     cantools_header, cantools_source,
+                                     test_source,
+                                     kcd_stem,
+                                     test_name):
+    here = Path('.')
+    source_path = here / (test_name + ".c")
+    header_path = here / (test_name + ".h")
+    canframe_header_path = here / 'canframe.h'
+    cantools_source_path = here / (kcd_stem + ".c")
+    cantools_header_path = here / (kcd_stem + ".h")
+    test_source_path = here / ("test_kcd_api_" + kcd_stem + ".c")
+    print(canframe_header_path)
+    with canframe_header_path.open('w+') as canframe_header:
+        canframe_header.write(canframe_header_code)
+    with source_path.open('w+') as source_file:
+        source_file.write(capi_source)
+    print(header_path)
+    with header_path.open('w+') as header_file:
+        header_file.write(capi_header)
+    print(cantools_source_path)
+    with cantools_source_path.open('w+') as cantools_source_file:
+        cantools_source_file.write(cantools_source)
+    print(cantools_header_path)
+    with cantools_header_path.open('w+') as cantools_header_file:
+        cantools_header_file.write(cantools_header)
+    print(test_source_path)
+    with test_source_path.open('w+') as test_source_file:
+        test_source_file.write(test_source)
+    return (header_path, source_path, canframe_header_path,
+            cantools_header_path, cantools_source_path,
+            test_source_path)

can_api_generator/main.py

@@ -0,0 +1,216 @@
+from can_api_generator.types import SignalDirection
+from importlib.metadata import files
+import hashlib
+import json
+import click
+import rich
+from can_api_generator.msg_processor import CAPIMessage
+from can_api_generator.msg_processor import MessageProcessingException
+from can_api_generator.msg_processor import SignalProcessingException
+from can_api_generator.code_generator import generate_api_implementation, build_msgs_data_dicts_from_kcd, generate_dyndec_implementation
+from can_api_generator.cbdb_generator import generate_cbdb, build_sidecar, CbdbGenerationException, CBDB_FORMAT_VERSION
+from can_api_generator.utils import converrt_to_c_identifier
+from jinja2 import Environment, FileSystemLoader
+from pathlib import Path
+
+
+@click.group()
+def main():
+    """
+    A tool to generate C source code and descriptor cbdbs to interact with
+    the signals encoded in CAN frames according to DBC/KCD schemata
+    """
+
+
+@main.command()
+@click.argument("DB_File", type=click.Path(exists=True, dir_okay=False),
+                required=True, nargs=1)
+@click.argument("Library_Name", type=str, required=True)
+@click.argument("Node_Name", type=str, required=True)
+@click.option("-f", "--force", is_flag=True, required=False, default=False,
+              help="Overwrite existing output files")
+@click.option("-a", "--encode-all-signals-func", is_flag=True, default=False,
+              help="Generate a function for each message that encodes every signal into frames at least once,"
+                   " this is helpful when some signals are inside of a mux, but all signals need to be sent "
+                   " regardless of how many frames need to be encoded to send the state of each signal at least"
+                   " once")
+@click.option("-im", "--id-member-name", type=str, default="arbitration_id",
+              help="The name of the CanFrame struct member that holds the "
+                   "arbitration id as uint32")
+@click.option("-lm", "--length-member-name", type=str, default="length",
+              help="The name of the CanFrame struct member that holds the "
+                   "message length as uint16")
+@click.option("-pm", "--payload-member-name", type=str, default="payload",
+              help="The name of the CanFrame struct member that holds the "
+                   "payload as uint64")
+@click.option("-r", "--global-rx", is_flag=True, default=False,
+              help="Generate a function and api-wide struct that updates an"
+                   " api message struct upon reception of a matching frame")
+@click.option("-p", "--periodic-tx", is_flag=True, default=False,
+              help="Generate code that will define a TX struct as well as a function "
+              "that periodically sends the messages according to their "
+              "periodicity")
+@click.option("--ti-compatible", is_flag=True, default=False,
+              help="Generate C code that can be compiled using the TI C2000 compiler"
+                   " The TI compiler does not handle variable declarations inside of case"
+                   " blocks so the variable declaration is moved to the top of a switch")
+@click.option("--send-now", is_flag=True, default=False,
+              help="Add mechanism in to the periodic tx function that allows to immediately "
+                   "send a message even if it is not enabled or has just been sent."
+                   " This is useful for updating critical signals as early as possible after a change")
+@click.option("--double-buffered", is_flag=True, default=False,
+              help="Generate double-buffered signal storage so that an ISR can safely decode"
+                   " RX frames without exposing a partially-decoded struct to readers, and so"
+                   " that user code can stage TX signal updates without racing the periodic TX"
+                   " encoder. Swaps are performed via per-message pointer swaps; an explicit"
+                   " <api>_<msg>_commit call is required to publish staged TX values.")
+@click.option("-v", "--verbose", is_flag=True, default=False,
+              help="Print the Data that is passed to the template that "
+              "generates the code")
+def api(db_file, library_name, node_name, force, encode_all_signals_func,
+        id_member_name, length_member_name, payload_member_name, global_rx,
+        periodic_tx, ti_compatible, send_now, double_buffered, verbose):
+    """
+    Generate the static C API for NODE_NAME from DB_FILE (.kcd or .yaml)
+    """
+    db_path = Path(db_file)
+    try:
+        basepath = Path(library_name)
+        source_path = basepath.parent / f"{basepath.stem}.c"
+        header_path = basepath.parent / f"{basepath.stem}.h"
+        if not force:
+            if source_path.exists():
+                raise FileExistsError(
+                    f"The File {source_path} already exists, aborting")
+            if header_path.exists():
+                raise FileExistsError(
+                    f"The File {header_path} already exists, aborting")
+    except FileExistsError as e:
+        click.echo(e)
+    try:
+        msgs, kcd_db = build_msgs_data_dicts_from_kcd(db_path, library_name, node_name)
+    except MessageProcessingException as e:
+        click.echo(f"Got Error while processing message {e.can_message_name}"
+                   f" for {e.node_name}: {e}")
+        return 1
+    except SignalProcessingException as e:
+        click.echo(f"Got Error while processing signal {e.signal_name}"
+                   f" in {e.message_name} for {e.node_name}: {e}")
+        return 1
+    if (verbose):
+        rich.print(msgs)
+    env = Environment(
+        loader=FileSystemLoader(
+            [str(Path(__file__).parent / "templates")]
+        )
+    )
+    header_code, source_code, canframe_header = generate_api_implementation(
+        msgs,
+        env,
+        library_name,
+        encode_all_signals_func,
+        False,
+        id_member_name,
+        length_member_name,
+        payload_member_name,
+        global_rx,
+        periodic_tx=periodic_tx,
+        ti_compatible=ti_compatible,
+        send_now=send_now,
+        double_buffered=double_buffered,
+    )
+    with (header_path.parent / 'canframe.h').open('w+') as canframe_header_file:
+        canframe_header_file.write(canframe_header)
+    with header_path.open('w+') as header_file:
+        header_file.write(header_code)
+    with source_path.open('w+') as source_file:
+        source_file.write(source_code)
+
+
+@main.command()
+@click.argument("DB_File", type=click.Path(exists=True, dir_okay=False),
+                required=True, nargs=1)
+@click.argument("Output_Base", type=str, required=True)
+@click.option("-f", "--force", is_flag=True, required=False, default=False,
+              help="Overwrite existing output files")
+def cbdb(db_file, output_base, force):
+    """
+    Compile DB_FILE (.kcd or .yaml) into a dynamic-decoder descriptor cbdb
+
+    Writes OUTPUT_BASE.cbdb (the binary descriptor table consumed by the
+    dyndec C runtime) and OUTPUT_BASE_ids.json (resolved signal name ->
+    signal_id map for client applications). The cbdb contains every signal
+    of every message: the dynamic decoder is a bus observer, not a node.
+    """
+    db_path = Path(db_file)
+    base_path = Path(output_base)
+    cbdb_path = base_path.parent / f"{base_path.name}.cbdb"
+    sidecar_path = base_path.parent / f"{base_path.name}_ids.json"
+    if not force:
+        for path in (cbdb_path, sidecar_path):
+            if path.exists():
+                click.echo(f"The File {path} already exists, aborting")
+                raise SystemExit(1)
+    try:
+        cbdb_bytes, id_map = generate_cbdb(db_path)
+        sidecar = build_sidecar(db_path)
+    except (MessageProcessingException, SignalProcessingException,
+            CbdbGenerationException) as e:
+        click.echo(f"Got Error while compiling cbdb: {e}")
+        raise SystemExit(1)
+    cbdb_path.write_bytes(cbdb_bytes)
+    with sidecar_path.open('w+') as sidecar_file:
+        json.dump(sidecar, sidecar_file, indent=2)
+        sidecar_file.write('\n')
+    click.echo(f"Wrote {cbdb_path} ({len(cbdb_bytes)} bytes, "
+               f"{len(id_map)} signals) and {sidecar_path}")
+
+
+@main.command()
+@click.argument("Library_Name", type=str, required=True)
+@click.option("-f", "--force", is_flag=True, required=False, default=False,
+              help="Overwrite existing output files")
+@click.option("-im", "--id-member-name", type=str, default="arbitration_id",
+              help="The name of the CanFrame struct member that holds the "
+                   "arbitration id as uint32")
+@click.option("-lm", "--length-member-name", type=str, default="length",
+              help="The name of the CanFrame struct member that holds the "
+                   "message length as uint16")
+@click.option("-pm", "--payload-member-name", type=str, default="payload",
+              help="The name of the CanFrame struct member that holds the "
+                   "payload as uint64")
+def dyndec(library_name, force, id_member_name, length_member_name,
+           payload_member_name):
+    """
+    Generate the KCD-independent dynamic-decoder C module
+
+    The module decodes CAN frames at runtime against a descriptor cbdb
+    compiled with the `cbdb` subcommand. It is generated once per firmware,
+    not per database: provisioning a new cbdb requires no recompilation.
+    """
+    basepath = Path(library_name)
+    source_path = basepath.parent / f"{basepath.stem}.c"
+    header_path = basepath.parent / f"{basepath.stem}.h"
+    if not force:
+        for path in (source_path, header_path):
+            if path.exists():
+                click.echo(f"The File {path} already exists, aborting")
+                raise SystemExit(1)
+    env = Environment(
+        loader=FileSystemLoader(
+            [str(Path(__file__).parent / "templates")]
+        )
+    )
+    header_code, source_code, canframe_header = generate_dyndec_implementation(
+        env,
+        arb_id_member_name=id_member_name,
+        length_member_name=length_member_name,
+        payload_member_name=payload_member_name,
+        header_file_name=header_path.name,
+    )
+    with (header_path.parent / 'canframe.h').open('w+') as canframe_header_file:
+        canframe_header_file.write(canframe_header)
+    with header_path.open('w+') as header_file:
+        header_file.write(header_code)
+    with source_path.open('w+') as source_file:
+        source_file.write(source_code)