simdref 0.0.0__py3-none-any.whl

simdref/lsp.py

@@ -0,0 +1,208 @@
+from __future__ import annotations
+
+import json
+import re
+import sys
+from dataclasses import dataclass
+
+from simdref.perf import best_cpi, best_latency
+from simdref.queries import linked_instruction_records
+from simdref.search import search_records
+from simdref.storage import (
+    load_intrinsic_from_db,
+    load_instruction_from_db,
+    open_db,
+    search_intrinsic_candidates_from_db,
+    search_instruction_candidates_from_db,
+)
+
+
+WORD_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_.]*")
+
+
+@dataclass
+class Session:
+    documents: dict[str, str]
+
+
+def _jsonrpc_write(payload: dict) -> None:
+    body = json.dumps(payload).encode("utf-8")
+    sys.stdout.buffer.write(f"Content-Length: {len(body)}\r\n\r\n".encode("ascii"))
+    sys.stdout.buffer.write(body)
+    sys.stdout.buffer.flush()
+
+
+def _jsonrpc_read() -> dict | None:
+    headers = {}
+    while True:
+        line = sys.stdin.buffer.readline()
+        if not line:
+            return None
+        if line in (b"\r\n", b"\n"):
+            break
+        key, value = line.decode("ascii").split(":", 1)
+        headers[key.strip().lower()] = value.strip()
+    length = int(headers.get("content-length", "0"))
+    if length <= 0:
+        return None
+    body = sys.stdin.buffer.read(length)
+    return json.loads(body)
+
+
+def _word_at(text: str, line: int, character: int) -> str | None:
+    lines = text.splitlines()
+    if line >= len(lines):
+        return None
+    current = lines[line]
+    for match in WORD_RE.finditer(current):
+        if match.start() <= character <= match.end():
+            return match.group(0)
+    return None
+
+
+def _line_prefix(text: str, line: int, character: int) -> str:
+    lines = text.splitlines()
+    if line >= len(lines):
+        return ""
+    current = lines[line][:character]
+    match = re.search(r"[A-Za-z_][A-Za-z0-9_.]*$", current)
+    return match.group(0) if match else ""
+
+
+def _hover_markdown(conn, word: str) -> str | None:
+    intrinsic = load_intrinsic_from_db(conn, word)
+    if intrinsic is not None:
+        lines = [f"```c\n{intrinsic.signature}\n```"]
+        if intrinsic.description:
+            lines.append(intrinsic.description)
+        meta = []
+        if intrinsic.header:
+            meta.append(f"header `{intrinsic.header}`")
+        if intrinsic.isa:
+            meta.append(f"ISA {', '.join(intrinsic.isa)}")
+        if intrinsic.category:
+            meta.append(f"category {intrinsic.category}")
+        if intrinsic.url:
+            meta.append(f"[source]({intrinsic.url})")
+        if meta:
+            lines.append(" | ".join(meta))
+        if intrinsic.instructions:
+            lines.append(f"Instructions: {', '.join(intrinsic.instructions[:6])}")
+        linked = linked_instruction_records(None, intrinsic, conn=conn)
+        if linked:
+            latencies = [best_latency(item.arch_details) for item in linked if best_latency(item.arch_details) != "-"]
+            throughputs = [best_cpi(item.arch_details) for item in linked if best_cpi(item.arch_details) != "-"]
+            perf = []
+            if latencies:
+                perf.append(f"best latency {min(latencies, key=lambda value: float(value))} cycles")
+            if throughputs:
+                perf.append(f"best cycle/instr {min(throughputs, key=lambda value: float(value))}")
+            if perf:
+                lines.append("Performance: " + ", ".join(perf))
+        return "\n\n".join(lines)
+
+    instruction = load_instruction_from_db(conn, word)
+    if instruction is not None:
+        lines = [f"```asm\n{instruction.key}\n```"]
+        if instruction.summary:
+            lines.append(instruction.summary)
+        meta = []
+        if instruction.isa:
+            meta.append(f"ISA {', '.join(instruction.isa)}")
+        if instruction.metadata.get("category"):
+            meta.append(f"category {instruction.metadata['category']}")
+        if meta:
+            lines.append(" | ".join(meta))
+        if instruction.linked_intrinsics:
+            lines.append(f"Intrinsics: {', '.join(instruction.linked_intrinsics[:6])}")
+        perf = []
+        lat = best_latency(instruction.arch_details)
+        cpi = best_cpi(instruction.arch_details)
+        if lat != "-":
+            perf.append(f"best latency {lat} cycles")
+        if cpi != "-":
+            perf.append(f"best cycle/instr {cpi}")
+        if perf:
+            lines.append("Performance: " + ", ".join(perf))
+        return "\n\n".join(lines)
+    return None
+
+
+def _completion_candidates(conn, prefix: str, limit: int = 50) -> list[dict]:
+    prefix_folded = prefix.casefold()
+    emitted: set[tuple[str, str]] = set()
+    items: list[dict] = []
+    candidate_limit = max(limit * 3, 100)
+    intrinsics = search_intrinsic_candidates_from_db(conn, prefix or "_mm", limit=candidate_limit)
+    instructions = search_instruction_candidates_from_db(conn, prefix or "_mm", limit=candidate_limit)
+    for result in search_records(intrinsics, instructions, prefix or "_mm", limit=candidate_limit):
+        label = result.title
+        if prefix_folded and not label.casefold().startswith(prefix_folded):
+            continue
+        key = (result.kind, label)
+        if key in emitted:
+            continue
+        emitted.add(key)
+        kind = 3 if result.kind == "intrinsic" else 14
+        items.append({"label": label, "kind": kind, "detail": result.subtitle, "insertText": label})
+        if len(items) >= limit:
+            return items
+    return items
+
+
+def main() -> int:
+    conn = open_db()
+    session = Session(documents={})
+    while True:
+        message = _jsonrpc_read()
+        if message is None:
+            return 0
+        method = message.get("method")
+        if method == "initialize":
+            _jsonrpc_write(
+                {
+                    "jsonrpc": "2.0",
+                    "id": message["id"],
+                    "result": {
+                        "capabilities": {
+                            "hoverProvider": True,
+                            "textDocumentSync": 1,
+                            "completionProvider": {"resolveProvider": False, "triggerCharacters": ["_", ".", "m", "v"]},
+                        }
+                    },
+                }
+            )
+        elif method == "initialized":
+            continue
+        elif method == "shutdown":
+            _jsonrpc_write({"jsonrpc": "2.0", "id": message["id"], "result": None})
+        elif method == "exit":
+            return 0
+        elif method == "textDocument/didOpen":
+            params = message["params"]
+            session.documents[params["textDocument"]["uri"]] = params["textDocument"]["text"]
+        elif method == "textDocument/didChange":
+            params = message["params"]
+            session.documents[params["textDocument"]["uri"]] = params["contentChanges"][-1]["text"]
+        elif method == "textDocument/didClose":
+            params = message["params"]
+            session.documents.pop(params["textDocument"]["uri"], None)
+        elif method == "textDocument/hover":
+            params = message["params"]
+            uri = params["textDocument"]["uri"]
+            text = session.documents.get(uri, "")
+            word = _word_at(text, params["position"]["line"], params["position"]["character"])
+            contents = None
+            if word:
+                body = _hover_markdown(conn, word)
+                if body:
+                    contents = {"kind": "markdown", "value": body}
+            _jsonrpc_write({"jsonrpc": "2.0", "id": message["id"], "result": {"contents": contents} if contents else None})
+        elif method == "textDocument/completion":
+            params = message["params"]
+            uri = params["textDocument"]["uri"]
+            text = session.documents.get(uri, "")
+            prefix = _line_prefix(text, params["position"]["line"], params["position"]["character"])
+            items = _completion_candidates(conn, prefix)
+            _jsonrpc_write({"jsonrpc": "2.0", "id": message["id"], "result": {"isIncomplete": False, "items": items}})
+    return 0

simdref/manpages.py

@@ -0,0 +1,139 @@
+"""Roff manpage generation for intrinsics and instructions.
+
+Generates man7 pages that can be viewed with ``man -M share/man <name>``.
+"""
+
+from __future__ import annotations
+
+import subprocess
+from collections.abc import Callable
+from pathlib import Path
+
+from simdref.display import display_architecture
+from simdref.models import Catalog, InstructionRecord, IntrinsicRecord
+from simdref.queries import (
+    build_intrinsic_instruction_index,
+    instruction_rows_for_intrinsic,
+    instruction_rows_for_intrinsic_indexed,
+)
+
+
+def _roff_escape(text: str) -> str:
+    return text.replace("\\", "\\\\").replace("-", "\\-")
+
+
+def _section(title: str, body: str) -> str:
+    return f".SH {title}\n{body}\n"
+
+
+def _metric_lines(record: InstructionRecord) -> list[str]:
+    lines = []
+    for arch, values in sorted(record.metrics.items()):
+        rendered = ", ".join(f"{key}={value}" for key, value in sorted(values.items()))
+        lines.append(f"{arch}: {rendered}")
+    return lines
+
+
+def _instruction_perf_lines(linked: list[InstructionRecord]) -> list[str]:
+    """Format linked instruction performance data as text lines for man pages."""
+    lines = []
+    for row in instruction_rows_for_intrinsic_indexed(linked):
+        instruction = row.get("instruction", "-")
+        uarch = row.get("uarch", "-")
+        metrics = {k: v for k, v in row.items() if k not in {"instruction", "uarch"}}
+        if metrics and uarch != "-":
+            rendered = ", ".join(f"{key}={value}" for key, value in sorted(metrics.items()))
+            lines.append(f"{instruction} | {uarch}: {rendered}")
+        else:
+            lines.append(f"{instruction} | no performance metrics available")
+    return lines
+
+
+def intrinsic_page(
+    record: IntrinsicRecord,
+    catalog: Catalog,
+    linked_instructions: list[InstructionRecord] | None = None,
+) -> str:
+    if linked_instructions is None:
+        linked_instructions = [
+            instruction
+            for instruction in catalog.instructions
+            if record.name in instruction.linked_intrinsics
+        ]
+    parts = [f'.TH "{record.name}" "7" "simdref" "simdref" "SIMD Intrinsic Reference"\n']
+    parts.append(_section("NAME", f"{_roff_escape(record.name)} \\- {_roff_escape(record.description or 'intrinsic')}"))
+    parts.append(_section("SYNOPSIS", f".nf\n{_roff_escape(record.signature)}\n.fi"))
+    parts.append(_section("DESCRIPTION", _roff_escape(record.description or "No description available.")))
+    parts.append(_section("HEADER", _roff_escape(record.header or "Unknown")))
+    if record.url:
+        parts.append(_section("SOURCE", _roff_escape(record.url)))
+    parts.append(_section("ARCHITECTURE", _roff_escape(display_architecture(record.architecture or "Unknown"))))
+    parts.append(_section("ISA", _roff_escape(", ".join(record.isa) or "Unknown")))
+    parts.append(_section("CATEGORY", _roff_escape(record.category or "Unknown")))
+    parts.append(_section("INSTRUCTIONS", _roff_escape(", ".join(record.instructions) or "None linked")))
+    perf_text = _roff_escape("\n".join(_instruction_perf_lines(linked_instructions)) or "No performance metrics available.")
+    parts.append(_section("PERFORMANCE SUMMARY", perf_text))
+    parts.append(_section("PERFORMANCE DETAILS", perf_text))
+    parts.append(_section("NOTES", _roff_escape("; ".join(record.notes) or "None")))
+    parts.append(_section("SEE ALSO", _roff_escape(", ".join(record.instructions) or "simdref-search(7)")))
+    return "".join(parts)
+
+
+def instruction_page(record: InstructionRecord) -> str:
+    parts = [f'.TH "{record.mnemonic}" "7" "simdref" "simdref" "SIMD Instruction Reference"\n']
+    parts.append(_section("NAME", f"{_roff_escape(record.key)} \\- {_roff_escape(record.summary)}"))
+    if record.description.get("Description"):
+        parts.append(_section("DESCRIPTION", _roff_escape(record.description["Description"])))
+    else:
+        parts.append(_section("DESCRIPTION", _roff_escape(record.summary)))
+    if record.description.get("Operation"):
+        parts.append(_section("OPERATION", f".nf\n{_roff_escape(record.description['Operation'])}\n.fi"))
+    parts.append(_section("ARCHITECTURE", _roff_escape(display_architecture(record.architecture or "Unknown"))))
+    parts.append(_section("ISA", _roff_escape(", ".join(record.isa) or "Unknown")))
+    parts.append(_section("OPERANDS", _roff_escape("\n".join(record.operands) or "No operand details available.")))
+    parts.append(_section("INTRINSICS", _roff_escape(", ".join(record.linked_intrinsics) or "None linked")))
+    parts.append(_section("PERFORMANCE DETAILS", _roff_escape("\n".join(_metric_lines(record)) or "No performance metrics available.")))
+    if record.description.get("Flags Affected"):
+        parts.append(_section("FLAGS AFFECTED", _roff_escape(record.description["Flags Affected"])))
+    for exc_key in ("Exceptions", "SIMD Floating-Point Exceptions", "Numeric Exceptions", "Other Exceptions"):
+        if record.description.get(exc_key):
+            parts.append(_section(exc_key.upper(), _roff_escape(record.description[exc_key])))
+    return "".join(parts)
+
+
+def write_manpages(
+    catalog: Catalog,
+    man_dir: Path,
+    on_progress: Callable[[int, int], None] | None = None,
+) -> None:
+    section_dir = man_dir / "man7"
+    section_dir.mkdir(parents=True, exist_ok=True)
+    index = build_intrinsic_instruction_index(catalog)
+    total = len(catalog.intrinsics) + len(catalog.instructions)
+    done = 0
+    for intrinsic in catalog.intrinsics:
+        linked = index.get(intrinsic.name, [])
+        (section_dir / f"{intrinsic.name}.7").write_text(intrinsic_page(intrinsic, catalog, linked))
+        done += 1
+        if on_progress is not None:
+            on_progress(done, total)
+    for instruction in catalog.instructions:
+        filename = f"instruction-{record_slug(instruction.architecture)}-{record_slug(instruction.key)}.7"
+        (section_dir / filename).write_text(instruction_page(instruction))
+        if instruction.architecture == "x86":
+            (section_dir / f"{instruction.mnemonic}.7").write_text(instruction_page(instruction))
+        (section_dir / f"{record_slug(instruction.architecture)}-{instruction.mnemonic}.7").write_text(instruction_page(instruction))
+        done += 1
+        if on_progress is not None:
+            on_progress(done, total)
+
+
+def record_slug(value: str) -> str:
+    return "".join(char.lower() if char.isalnum() else "-" for char in value).strip("-")
+
+
+def open_manpage(name: str, man_dir: Path) -> int:
+    target = man_dir / "man7" / f"{name}.7"
+    if not target.exists():
+        return 1
+    return subprocess.call(["man", "-M", str(man_dir), name])

simdref/models.py

@@ -0,0 +1,225 @@
+"""Core data models for simdref.
+
+Defines the four dataclasses that represent the catalog:
+:class:`SourceVersion`, :class:`IntrinsicRecord`, :class:`InstructionRecord`,
+and :class:`Catalog`.  All use ``slots=True`` for memory efficiency when
+holding thousands of records.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from simdref.pdfrefs import apply_legacy_pdf_metadata, normalize_pdf_refs
+
+
+@dataclass(slots=True)
+class SourceVersion:
+    source: str
+    version: str
+    fetched_at: str
+    url: str
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "source": self.source,
+            "version": self.version,
+            "fetched_at": self.fetched_at,
+            "url": self.url,
+        }
+
+
+@dataclass(slots=True)
+class IntrinsicRecord:
+    name: str
+    signature: str
+    description: str
+    header: str
+    url: str = ""
+    architecture: str = "x86"
+    isa: list[str] = field(default_factory=list)
+    category: str = ""
+    subcategory: str = ""
+    instructions: list[str] = field(default_factory=list)
+    instruction_refs: list[dict[str, str]] = field(default_factory=list)
+    metadata: dict[str, str] = field(default_factory=dict)
+    doc_sections: dict[str, str] = field(default_factory=dict)
+    notes: list[str] = field(default_factory=list)
+    aliases: list[str] = field(default_factory=list)
+    source: str = "intel"
+    _search_blob: str = field(default="", repr=False)
+
+    def __post_init__(self) -> None:
+        fields = [
+            self.name, self.signature, self.description, self.header, self.url,
+            self.architecture,
+            self.category, " ".join(self.isa), " ".join(self.instructions),
+            " ".join(self.aliases), " ".join(self.metadata.values()), " ".join(self.doc_sections.values()),
+        ]
+        self._search_blob = " ".join(x for x in fields if x)
+
+    @property
+    def search_blob(self) -> str:
+        return self._search_blob
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "signature": self.signature,
+            "description": self.description,
+            "header": self.header,
+            "url": self.url,
+            "architecture": self.architecture,
+            "isa": self.isa,
+            "category": self.category,
+            "subcategory": self.subcategory,
+            "instructions": self.instructions,
+            "instruction_refs": self.instruction_refs,
+            "metadata": self.metadata,
+            "doc_sections": self.doc_sections,
+            "notes": self.notes,
+            "aliases": self.aliases,
+            "source": self.source,
+        }
+
+
+@dataclass(slots=True)
+class InstructionRecord:
+    mnemonic: str
+    form: str
+    summary: str
+    architecture: str = "x86"
+    isa: list[str] = field(default_factory=list)
+    operand_details: list[dict[str, str]] = field(default_factory=list)
+    metadata: dict[str, str] = field(default_factory=dict)
+    arch_details: dict[str, dict[str, Any]] = field(default_factory=dict)
+    linked_intrinsics: list[str] = field(default_factory=list)
+    aliases: list[str] = field(default_factory=list)
+    description: dict[str, str] = field(default_factory=dict)
+    pdf_refs: list[dict[str, str]] = field(default_factory=list)
+    source: str = "uops.info"
+    _search_blob: str = field(default="", repr=False)
+    _key: str = field(default="", init=False, repr=False)
+    _db_key: str = field(default="", init=False, repr=False)
+
+    def __post_init__(self) -> None:
+        self.pdf_refs = normalize_pdf_refs(self.pdf_refs, self.metadata)
+        self.metadata = apply_legacy_pdf_metadata(dict(self.metadata), self.pdf_refs)
+        self._key = self.form.strip()
+        if not self._key:
+            self._key = self.mnemonic
+        elif not self._key.casefold().startswith(self.mnemonic.casefold()):
+            self._key = f"{self.mnemonic} {self._key}".strip()
+        self._db_key = f"{self.architecture}:{self._key.casefold()}"
+        fields = [
+            self.mnemonic, self.form, self.summary, self.architecture,
+            " ".join(self.isa), " ".join(self.operands),
+            " ".join(self.linked_intrinsics), " ".join(self.aliases),
+        ]
+        self._search_blob = " ".join(x for x in fields if x)
+
+    @property
+    def key(self) -> str:
+        return self._key
+
+    @property
+    def db_key(self) -> str:
+        return self._db_key
+
+    @property
+    def operands(self) -> list[str]:
+        rendered: list[str] = []
+        for operand in self.operand_details:
+            rendered_rw = "".join(flag for flag in ("r", "w") if operand.get(flag) == "1")
+            idx = operand.get("idx", "")
+            text_parts = [
+                f"idx={idx}" if idx else "",
+                rendered_rw,
+                operand.get("type", ""),
+                operand.get("width", ""),
+                operand.get("xtype", ""),
+                operand.get("name", ""),
+            ]
+            text = " ".join(part for part in text_parts if part)
+            if text:
+                rendered.append(text)
+        return rendered
+
+    @property
+    def search_blob(self) -> str:
+        return self._search_blob
+
+    @property
+    def metrics(self) -> dict[str, dict[str, str]]:
+        return {
+            arch: measurement
+            for arch, details in self.arch_details.items()
+            if (measurement := details.get("measurement"))
+        }
+
+    def to_dict(self) -> dict[str, Any]:
+        metadata = apply_legacy_pdf_metadata(dict(self.metadata), self.pdf_refs)
+        return {
+            "mnemonic": self.mnemonic,
+            "form": self.form,
+            "summary": self.summary,
+            "architecture": self.architecture,
+            "isa": self.isa,
+            "operand_details": self.operand_details,
+            "metadata": metadata,
+            "arch_details": self.arch_details,
+            "linked_intrinsics": self.linked_intrinsics,
+            "aliases": self.aliases,
+            "description": self.description,
+            "pdf_refs": self.pdf_refs,
+            "source": self.source,
+        }
+
+    @classmethod
+    def from_dict(cls, payload: dict[str, Any]) -> "InstructionRecord":
+        data = dict(payload)
+        data.pop("metrics", None)
+        data.pop("operands", None)
+        data.setdefault("architecture", "x86")
+        data.setdefault("description", {})
+        metadata = dict(data.get("metadata") or {})
+        data["metadata"] = metadata
+        data["pdf_refs"] = normalize_pdf_refs(data.get("pdf_refs"), metadata)
+        return cls(**data)
+
+
+@dataclass(slots=True)
+class Catalog:
+    intrinsics: list[IntrinsicRecord]
+    instructions: list[InstructionRecord]
+    sources: list[SourceVersion]
+    generated_at: str
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "intrinsics": [item.to_dict() for item in self.intrinsics],
+            "instructions": [item.to_dict() for item in self.instructions],
+            "sources": [item.to_dict() for item in self.sources],
+            "generated_at": self.generated_at,
+        }
+
+    @classmethod
+    def from_dict(cls, payload: dict[str, Any]) -> "Catalog":
+        return cls(
+            intrinsics=[
+                IntrinsicRecord(architecture="x86", **item) if "architecture" not in item else IntrinsicRecord(**item)
+                for item in payload.get("intrinsics", [])
+            ],
+            instructions=[InstructionRecord.from_dict(item) for item in payload.get("instructions", [])],
+            sources=[
+                SourceVersion(
+                    source=item.get("source", ""),
+                    version=item.get("version", ""),
+                    fetched_at=item.get("fetched_at", ""),
+                    url=item.get("url", ""),
+                )
+                for item in payload.get("sources", [])
+            ],
+            generated_at=payload["generated_at"],
+        )

simdref/pdfparse/__init__.py

@@ -0,0 +1,13 @@
+"""PDF parsing pipeline for extracting instruction descriptions."""
+
+from simdref.pdfparse.registry import get_pdf_source, iter_pdf_sources, register_pdf_source
+from simdref.pdfparse.types import PdfDescriptionPayload, PdfEnrichmentResult, PdfSourceSpec
+
+__all__ = [
+    "PdfDescriptionPayload",
+    "PdfEnrichmentResult",
+    "PdfSourceSpec",
+    "get_pdf_source",
+    "iter_pdf_sources",
+    "register_pdf_source",
+]