lvkit 0.2.0__tar.gz → 0.3.0__tar.gz

{lvkit-0.2.0 → lvkit-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lvkit
-Version: 0.2.0
+Version: 0.3.0
 Summary: Understand and convert LabVIEW VIs to Python without a LabVIEW license
 Project-URL: Repository, https://github.com/pragmatest-dev/lvkit
 Project-URL: Issues, https://github.com/pragmatest-dev/lvkit/issues

{lvkit-0.2.0 → lvkit-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "lvkit"
-version = "0.2.0"
+version = "0.3.0"
 description = "Understand and convert LabVIEW VIs to Python without a LabVIEW license"
 readme = "README.md"
 requires-python = ">=3.10"

lvkit-0.3.0/scripts/audit_unresolved.py

@@ -0,0 +1,311 @@
+#!/usr/bin/env python3
+"""Audit primitives and vilib VIs referenced in samples but not in our data.
+
+Walks samples/, parses each VI via XML pairs, cross-references primitives and
+vilib dependencies against the JSON data files, and emits a markdown report
+to .tmp/audit-unresolved.md sorted by usage count (highest first).
+
+Usage: uv run python scripts/audit_unresolved.py
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from datetime import date
+from pathlib import Path
+
+from lvkit.parser import parse_vi
+from lvkit.parser.node_types import PrimitiveNode
+from lvkit.primitive_resolver import PrimitiveResolver
+from lvkit.vilib_resolver import VIEntry, VILibResolver
+
+REPO_ROOT = Path(__file__).parent.parent
+SAMPLES = REPO_ROOT / "samples"
+REPORT_PATH = REPO_ROOT / ".tmp" / "audit-unresolved.md"
+
+# Status buckets
+CAPTURED = "captured"
+PARTIAL = "partial"
+MISSING = "missing"
+
+
+def find_bd_xml_pairs(samples_root: Path) -> list[tuple[Path, Path | None]]:
+    """Find all (bd_xml, main_xml) pairs under samples/.
+
+    Returns a list of (bd_xml, main_xml) tuples. main_xml is optional.
+    Dedups by bd_xml absolute path.
+    """
+    seen: set[Path] = set()
+    pairs: list[tuple[Path, Path | None]] = []
+    for bd_xml in samples_root.rglob("*_BDHb.xml"):
+        resolved = bd_xml.resolve()
+        if resolved in seen:
+            continue
+        seen.add(resolved)
+        # Derive main_xml path: same stem minus _BDHb
+        main_xml_name = bd_xml.name.replace("_BDHb.xml", ".xml")
+        main_xml = bd_xml.parent / main_xml_name
+        if not main_xml.exists():
+            main_xml = None
+        pairs.append((bd_xml, main_xml))
+    return pairs
+
+
+def classify_primitive(entry: dict | None) -> str:
+    """Classify a primitive entry as captured/partial/missing."""
+    if entry is None:
+        return MISSING
+    if entry.get("placeholder"):
+        return PARTIAL
+    python_code = entry.get("python_code")
+    if not python_code:
+        return PARTIAL
+    terminals = entry.get("terminals") or []
+    if not terminals:
+        return PARTIAL
+    for t in terminals:
+        if t.get("index") is None:
+            return PARTIAL
+    return CAPTURED
+
+
+def classify_vilib(entry: VIEntry | None) -> str:
+    """Classify a vilib VI entry as captured/partial/missing."""
+    if entry is None:
+        return MISSING
+    if entry.status == "needs_terminals":
+        return PARTIAL
+    for t in entry.terminals:
+        if t.index is None:
+            return PARTIAL
+    return CAPTURED
+
+
+def main() -> None:
+    prim_resolver = PrimitiveResolver()
+    vilib_resolver = VILibResolver()
+
+    # prim_id -> {"count": int, "name": str, "vis": set[str]}
+    prim_usage: dict[str, dict] = defaultdict(
+        lambda: {"count": 0, "name": None, "vis": set()}
+    )
+    # vilib_path -> {"count": int, "name": str, "vis": set[str]}
+    vilib_usage: dict[str, dict] = defaultdict(
+        lambda: {"count": 0, "name": None, "vis": set()}
+    )
+
+    pairs = find_bd_xml_pairs(SAMPLES)
+    print(f"Scanning {len(pairs)} VIs from {SAMPLES}...")
+
+    parse_failures: list[tuple[Path, str]] = []
+    for bd_xml, main_xml in pairs:
+        try:
+            parsed = parse_vi(bd_xml=bd_xml, main_xml=main_xml)
+        except Exception as e:
+            parse_failures.append((bd_xml, f"{type(e).__name__}: {e}"))
+            continue
+
+        vi_label = bd_xml.name.replace("_BDHb.xml", "")
+
+        # Primitives pass
+        for node in parsed.block_diagram.nodes:
+            if not isinstance(node, PrimitiveNode):
+                continue
+            if node.prim_res_id is None:
+                continue
+            key = str(node.prim_res_id)
+            prim_usage[key]["count"] += 1
+            prim_usage[key]["vis"].add(vi_label)
+            if node.name and prim_usage[key]["name"] is None:
+                prim_usage[key]["name"] = node.name
+
+        # VIlib pass (requires main_xml)
+        for ref in parsed.metadata.dependency_refs:
+            if not ref.path_tokens or ref.path_tokens[0] != "<vilib>":
+                continue
+            vilib_path = "/".join(ref.path_tokens[1:])
+            vilib_usage[vilib_path]["count"] += 1
+            vilib_usage[vilib_path]["vis"].add(vi_label)
+            if vilib_usage[vilib_path]["name"] is None:
+                vilib_usage[vilib_path]["name"] = ref.name
+
+    # Classify primitives
+    prim_buckets: dict[str, list] = {CAPTURED: [], PARTIAL: [], MISSING: []}
+    for prim_id, info in prim_usage.items():
+        entry = prim_resolver.get_by_id(prim_id)
+        status = classify_primitive(entry)
+        name = (entry or {}).get("name") or info["name"] or "Unknown"
+        prim_buckets[status].append({
+            "id": prim_id,
+            "name": name,
+            "count": info["count"],
+            "vis": sorted(info["vis"]),
+        })
+
+    # Classify vilib
+    vilib_buckets: dict[str, list] = {CAPTURED: [], PARTIAL: [], MISSING: []}
+    for path, info in vilib_usage.items():
+        entry = vilib_resolver.resolve(path)
+        if entry is None and info["name"]:
+            entry = vilib_resolver.resolve_by_name(info["name"])
+        status = classify_vilib(entry)
+        name = (entry.name if entry else info["name"]) or "Unknown"
+        vilib_buckets[status].append({
+            "path": path,
+            "name": name,
+            "count": info["count"],
+            "vis": sorted(info["vis"]),
+            "status": entry.status if entry else None,
+        })
+
+    # Sort by count desc
+    for bucket in prim_buckets.values():
+        bucket.sort(key=lambda x: -x["count"])
+    for bucket in vilib_buckets.values():
+        bucket.sort(key=lambda x: -x["count"])
+
+    # JSON-level audit: count all entries in vilib JSON (not just sample refs).
+    # Each entry appears under multiple keys in _by_name — dedup by object id.
+    seen_ids: set[int] = set()
+    json_vilib_total = 0
+    json_vilib_captured = 0
+    json_vilib_needs_terminals = 0
+    json_vilib_missing_idx = 0
+    for entry in vilib_resolver._by_name.values():
+        if id(entry) in seen_ids:
+            continue
+        seen_ids.add(id(entry))
+        json_vilib_total += 1
+        if entry.status == "needs_terminals":
+            json_vilib_needs_terminals += 1
+        elif any(t.index is None for t in entry.terminals):
+            json_vilib_missing_idx += 1
+        else:
+            json_vilib_captured += 1
+
+    json_prim_total = len(prim_resolver.get_all_ids())
+    json_prim_captured = sum(
+        1 for pid in prim_resolver.get_all_ids()
+        if classify_primitive(prim_resolver.get_by_id(pid)) == CAPTURED
+    )
+    json_prim_partial = json_prim_total - json_prim_captured
+
+    # Write report
+    REPORT_PATH.parent.mkdir(parents=True, exist_ok=True)
+    lines: list[str] = []
+    lines.append(f"# Unresolved Audit — {date.today().isoformat()}\n")
+    lines.append(f"Scanned {len(pairs)} VI block diagrams from `samples/`.\n")
+
+    lines.append("## Summary (sample usage)\n")
+    lines.append(
+        f"- **Primitives**: "
+        f"{len(prim_buckets[CAPTURED])} captured / "
+        f"{len(prim_buckets[PARTIAL])} partial / "
+        f"{len(prim_buckets[MISSING])} missing "
+        f"({len(prim_usage)} unique primResIDs seen)"
+    )
+    lines.append(
+        f"- **VIlib**: "
+        f"{len(vilib_buckets[CAPTURED])} captured / "
+        f"{len(vilib_buckets[PARTIAL])} partial / "
+        f"{len(vilib_buckets[MISSING])} missing "
+        f"({len(vilib_usage)} unique paths seen)"
+    )
+    lines.append("")
+    lines.append("## JSON data state (all entries, regardless of samples)\n")
+    lines.append(
+        f"- **Primitives JSON**: {json_prim_total} total — "
+        f"{json_prim_captured} captured / {json_prim_partial} partial"
+    )
+    lines.append(
+        f"- **VIlib JSON**: {json_vilib_total} total — "
+        f"{json_vilib_captured} captured / "
+        f"{json_vilib_needs_terminals} needs_terminals / "
+        f"{json_vilib_missing_idx} missing indices"
+    )
+    if parse_failures:
+        lines.append(f"- **Parse failures**: {len(parse_failures)}")
+    lines.append("")
+
+    def _sample_vis(vis: list[str], limit: int = 3) -> str:
+        if len(vis) <= limit:
+            return ", ".join(vis)
+        return ", ".join(vis[:limit]) + f" (+{len(vis) - limit} more)"
+
+    # Missing primitives
+    lines.append("## Missing Primitives\n")
+    lines.append("| primResID | Name | Count | Sample VIs |")
+    lines.append("|-----------|------|-------|------------|")
+    for p in prim_buckets[MISSING]:
+        lines.append(
+            f"| {p['id']} | {p['name']} | {p['count']} | {_sample_vis(p['vis'])} |"
+        )
+    if not prim_buckets[MISSING]:
+        lines.append("| _(none)_ |  |  |  |")
+    lines.append("")
+
+    # Partial primitives
+    lines.append("## Partial Primitives (needs terminals or python_code)\n")
+    lines.append("| primResID | Name | Count | Sample VIs |")
+    lines.append("|-----------|------|-------|------------|")
+    for p in prim_buckets[PARTIAL]:
+        lines.append(
+            f"| {p['id']} | {p['name']} | {p['count']} | {_sample_vis(p['vis'])} |"
+        )
+    if not prim_buckets[PARTIAL]:
+        lines.append("| _(none)_ |  |  |  |")
+    lines.append("")
+
+    # Missing vilib
+    lines.append("## Missing VIlib VIs\n")
+    lines.append("| Path | Count | Sample VIs |")
+    lines.append("|------|-------|------------|")
+    for v in vilib_buckets[MISSING]:
+        lines.append(
+            f"| `{v['path']}` | {v['count']} | {_sample_vis(v['vis'])} |"
+        )
+    if not vilib_buckets[MISSING]:
+        lines.append("| _(none)_ |  |  |")
+    lines.append("")
+
+    # Partial vilib
+    lines.append("## Partial VIlib VIs (needs terminal indices)\n")
+    lines.append("| Path | Status | Count | Sample VIs |")
+    lines.append("|------|--------|-------|------------|")
+    for v in vilib_buckets[PARTIAL]:
+        lines.append(
+            f"| `{v['path']}` | {v['status']} | {v['count']} | "
+            f"{_sample_vis(v['vis'])} |"
+        )
+    if not vilib_buckets[PARTIAL]:
+        lines.append("| _(none)_ |  |  |  |")
+    lines.append("")
+
+    # Parse failures (if any)
+    if parse_failures:
+        lines.append("## Parse Failures\n")
+        for path, err in parse_failures[:30]:
+            rel = path.relative_to(REPO_ROOT)
+            lines.append(f"- `{rel}` — {err}")
+        if len(parse_failures) > 30:
+            lines.append(f"- _(+{len(parse_failures) - 30} more)_")
+        lines.append("")
+
+    REPORT_PATH.write_text("\n".join(lines))
+    print(f"Report written to {REPORT_PATH.relative_to(REPO_ROOT)}")
+    print(
+        f"  Primitives: {len(prim_buckets[CAPTURED])} captured / "
+        f"{len(prim_buckets[PARTIAL])} partial / "
+        f"{len(prim_buckets[MISSING])} missing"
+    )
+    print(
+        f"  VIlib: {len(vilib_buckets[CAPTURED])} captured / "
+        f"{len(vilib_buckets[PARTIAL])} partial / "
+        f"{len(vilib_buckets[MISSING])} missing"
+    )
+    if parse_failures:
+        print(f"  Parse failures: {len(parse_failures)}")
+
+
+if __name__ == "__main__":
+    main()

lvkit-0.3.0/scripts/dump_primitive_context.py

@@ -0,0 +1,253 @@
+#!/usr/bin/env python3
+"""Dump PrimitiveResolutionNeeded-shaped context for missing primitives.
+
+For each primResID seen in samples but not captured in primitives.json,
+emit a markdown section matching the PrimitiveResolutionNeeded message
+format (terminal indices, directions, types, usage count, sample VIs).
+
+The output feeds the `lvkit-resolve-primitive` skill — one section per
+primitive, sorted by usage count descending.
+
+Usage:
+    uv run python scripts/dump_primitive_context.py [--limit 25]
+"""
+
+from __future__ import annotations
+
+import argparse
+from collections import defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+
+from lvkit.parser import parse_vi
+from lvkit.parser.node_types import PrimitiveNode
+from lvkit.primitive_resolver import PrimitiveResolver
+
+REPO_ROOT = Path(__file__).parent.parent
+SAMPLES = REPO_ROOT / "samples"
+REPORT_PATH = REPO_ROOT / ".tmp" / "missing-primitives-context.md"
+
+# primResID range → hint (copied from SKILL.md Step 4)
+RANGE_HINTS: list[tuple[int, int, str]] = [
+    (1044, 1064, "Array operations"),
+    (1061, 1081, "Numeric/arithmetic"),
+    (1083, 1128, "Path/comparison/boolean"),
+    (1140, 1170, "Type conversion, variant, data manipulation"),
+    (1300, 1340, "Timing, constants, clusters"),
+    (1419, 1435, "Path operations"),
+    (1500, 1540, "String operations"),
+    (1600, 1610, "Flatten/unflatten"),
+    (1809, 1911, "Array index/sort/delete"),
+    (1999, 1999, "Path constant"),
+    (2073, 2076, "Error handling"),
+    (2401, 2401, "Merge Errors"),
+    (8003, 8083, "File I/O"),
+    (8100, 8101, "VI info"),
+    (8201, 8205, "Variant operations"),
+    (9000, 9114, "VI Server, references, scripting"),
+]
+
+
+def range_hint(prim_id: int) -> str:
+    for lo, hi, hint in RANGE_HINTS:
+        if lo <= prim_id <= hi:
+            return f"{lo}-{hi}: {hint}"
+    return "(no range hint)"
+
+
+@dataclass
+class TerminalObservation:
+    index: int | None
+    direction: str
+    type_name: str
+    name: str | None
+
+
+@dataclass
+class Invocation:
+    vi_label: str
+    bd_xml: Path
+    terminals: list[TerminalObservation]
+
+
+def find_bd_xml_pairs(samples_root: Path) -> list[tuple[Path, Path | None]]:
+    seen: set[Path] = set()
+    pairs: list[tuple[Path, Path | None]] = []
+    for bd_xml in samples_root.rglob("*_BDHb.xml"):
+        resolved = bd_xml.resolve()
+        if resolved in seen:
+            continue
+        seen.add(resolved)
+        main_xml = bd_xml.parent / bd_xml.name.replace("_BDHb.xml", ".xml")
+        if not main_xml.exists():
+            main_xml = None
+        pairs.append((bd_xml, main_xml))
+    return pairs
+
+
+def collect_invocations(
+    pairs: list[tuple[Path, Path | None]],
+    missing_ids: set[int],
+    max_per_id: int,
+) -> dict[int, tuple[int, list[Invocation]]]:
+    """Scan all pairs, collect up to max_per_id invocations per primResID.
+
+    Returns: prim_id -> (total_count, invocations)
+    """
+    counts: dict[int, int] = defaultdict(int)
+    invocations: dict[int, list[Invocation]] = defaultdict(list)
+
+    for bd_xml, main_xml in pairs:
+        try:
+            parsed = parse_vi(bd_xml=bd_xml, main_xml=main_xml)
+        except Exception:
+            continue
+
+        vi_label = bd_xml.name.replace("_BDHb.xml", "")
+        for node in parsed.block_diagram.nodes:
+            if not isinstance(node, PrimitiveNode):
+                continue
+            if node.prim_res_id not in missing_ids:
+                continue
+            counts[node.prim_res_id] += 1
+
+            # Only record detailed invocations up to max_per_id
+            if len(invocations[node.prim_res_id]) >= max_per_id:
+                continue
+
+            terms: list[TerminalObservation] = []
+            for _uid, t_info in parsed.block_diagram.terminal_info.items():
+                if t_info.parent_uid != node.uid:
+                    continue
+                type_name = (
+                    t_info.parsed_type.type_name
+                    if t_info.parsed_type else "?"
+                )
+                terms.append(TerminalObservation(
+                    index=t_info.index,
+                    direction="output" if t_info.is_output else "input",
+                    type_name=type_name,
+                    name=t_info.name,
+                ))
+            terms.sort(key=lambda t: (t.index is None, t.index or 0))
+            invocations[node.prim_res_id].append(Invocation(
+                vi_label=vi_label,
+                bd_xml=bd_xml,
+                terminals=terms,
+            ))
+
+    return {pid: (counts[pid], invs) for pid, invs in invocations.items()}
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument(
+        "--limit", type=int, default=25,
+        help="Number of missing primitives to include (top N by usage)",
+    )
+    ap.add_argument(
+        "--invocations", type=int, default=3,
+        help="Max sample invocations per primResID",
+    )
+    args = ap.parse_args()
+
+    resolver = PrimitiveResolver()
+
+    # First pass: count all primResIDs in samples to build the missing set
+    # sorted by count. Reuse audit logic: classify missing vs captured via
+    # resolver.get_by_id + python_code/terminals check.
+    pairs = find_bd_xml_pairs(SAMPLES)
+    print(f"Scanning {len(pairs)} VIs...")
+
+    all_counts: dict[int, int] = defaultdict(int)
+    for bd_xml, main_xml in pairs:
+        try:
+            parsed = parse_vi(bd_xml=bd_xml, main_xml=main_xml)
+        except Exception:
+            continue
+        for node in parsed.block_diagram.nodes:
+            if isinstance(node, PrimitiveNode) and node.prim_res_id is not None:
+                all_counts[node.prim_res_id] += 1
+
+    # Filter to missing primitives
+    missing: list[tuple[int, int]] = []
+    for pid, count in all_counts.items():
+        entry = resolver.get_by_id(str(pid))
+        if entry is None:
+            missing.append((pid, count))
+            continue
+        # Partial: no python_code or any terminal missing index
+        if not entry.get("python_code"):
+            missing.append((pid, count))
+            continue
+        terms = entry.get("terminals") or []
+        if not terms or any(t.get("index") is None for t in terms):
+            missing.append((pid, count))
+
+    missing.sort(key=lambda x: -x[1])
+    top = missing[:args.limit]
+    print(f"Found {len(missing)} missing/partial primitives, taking top {len(top)}")
+
+    # Second pass: collect invocation details only for the top N
+    top_ids = {pid for pid, _ in top}
+    details = collect_invocations(pairs, top_ids, args.invocations)
+
+    # Write the report
+    REPORT_PATH.parent.mkdir(parents=True, exist_ok=True)
+    lines: list[str] = []
+    lines.append("# Missing primitives — resolution context\n")
+    lines.append(
+        f"Top {len(top)} of {len(missing)} missing/partial primitives "
+        f"by usage count across {len(pairs)} sample VIs.\n"
+    )
+    lines.append(
+        "Each section matches the `PrimitiveResolutionNeeded` diagnostic "
+        "format. Feed one section at a time to the "
+        "`lvkit-resolve-primitive` skill.\n"
+    )
+
+    for pid, count in top:
+        invs = details.get(pid, (count, []))[1]
+        lines.append(f"## primResID {pid} — {count} uses\n")
+        lines.append(f"**Range hint**: {range_hint(pid)}\n")
+
+        existing = resolver.get_by_id(str(pid))
+        if existing is not None:
+            status = "partial"
+            reason = []
+            if not existing.get("python_code"):
+                reason.append("no python_code")
+            terms = existing.get("terminals") or []
+            if not terms:
+                reason.append("no terminals")
+            elif any(t.get("index") is None for t in terms):
+                reason.append("missing terminal indices")
+            lines.append(
+                f"**Current entry**: {existing.get('name', '?')} "
+                f"({status}: {', '.join(reason)})\n"
+            )
+        else:
+            lines.append("**Current entry**: (none)\n")
+
+        for i, inv in enumerate(invs, 1):
+            lines.append(f"### Sample {i}: {inv.vi_label}")
+            rel = inv.bd_xml.relative_to(REPO_ROOT)
+            lines.append(f"File: `{rel}`")
+            lines.append("Wired terminals from graph:")
+            for t in inv.terminals:
+                parts = [
+                    f"index={t.index}",
+                    f"direction={t.direction}",
+                    f"type={t.type_name}",
+                ]
+                if t.name:
+                    parts.append(f"name={t.name}")
+                lines.append(f"- {' '.join(parts)}")
+            lines.append("")
+
+    REPORT_PATH.write_text("\n".join(lines))
+    print(f"Report written to {REPORT_PATH.relative_to(REPO_ROOT)}")
+
+
+if __name__ == "__main__":
+    main()