cseq 0.0.1__py3-none-any.whl

cseq/fast_parser.py

@@ -0,0 +1,286 @@
+from __future__ import annotations
+
+from bisect import bisect_right
+from dataclasses import dataclass
+from functools import lru_cache
+import re
+import sys
+
+from .model import (
+    CallSiteRecord,
+    CompactFunctionRecord,
+    CompactFunctionTable,
+    FileParseArtifact,
+    FunctionRecord,
+    ParseStatus,
+    SourceRange,
+    TranslationUnit,
+)
+from .parser import ParserBackend
+
+_C_KEYWORDS = {
+    "if", "for", "while", "switch", "return", "sizeof", "_Alignof", "_Generic",
+}
+
+_FUNC_RE = re.compile(
+    r"(?P<header>(?:(?P<storage>static)\s+)?(?:inline\s+)?"
+    r"(?:const\s+)?(?:void|char|short|int|long|float|double|signed|unsigned|[A-Za-z_]\w*)"
+    r"(?:\s+|\s*\*\s*)(?P<name>[A-Za-z_]\w*)\s*\((?P<params>[^;{}()]*)\)\s*)\{",
+    re.MULTILINE,
+)
+_CALL_RE = re.compile(r"\b([A-Za-z_]\w*)\s*\(")
+
+# Constructs whose semantics/provenance require the semantic backend.  The fast
+# path deliberately claims only a narrow, auditable subset.
+_COMPLEX_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (
+    ("preprocessor-conditional", re.compile(r"(?m)^\s*#\s*(?:if|ifdef|ifndef|elif|else|endif)\b")),
+    ("macro-definition", re.compile(r"(?m)^\s*#\s*define\b")),
+    ("function-pointer", re.compile(r"\(\s*\*\s*[A-Za-z_]\w*\s*\)")),
+    ("member-access", re.compile(r"(?:->|\.[A-Za-z_]\w*)")),
+    ("aggregate-or-type", re.compile(r"\b(?:struct|union|enum|typedef)\b")),
+    ("switch", re.compile(r"\b(?:switch|case|default)\b")),
+    ("loop", re.compile(r"\b(?:for|while|do)\b")),
+    ("goto", re.compile(r"\b(?:goto|setjmp|longjmp)\b")),
+    ("extension", re.compile(r"\b(?:asm|__asm__|__attribute__|__declspec|typeof|__typeof__)\b")),
+    ("reserved-extension", re.compile(r"\b__[A-Za-z_]\w*\b")),
+    ("ternary", re.compile(r"\?[^:]+:")),
+)
+
+
+@dataclass(frozen=True, slots=True)
+class FastEligibility:
+    eligible: bool
+    reasons: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class FastPrepared:
+    raw: str
+    code_only: str
+    spans: tuple[tuple[str, int, int, int, int, str | None, str], ...]
+    line_starts: tuple[int, ...]
+    eligibility: FastEligibility
+
+
+class FastLexicalParser(ParserBackend):
+    """Conservative lexical parser for simple C translation units.
+
+    It is intentionally not a replacement C compiler.  It only returns an
+    artifact when the TU is inside a small subset whose direct-call structure is
+    unambiguous.  Everything else is refined by HybridParser's semantic backend.
+    """
+
+    name = "fast-lexical-v1"
+
+    def prepare(self, translation_unit: TranslationUnit) -> FastPrepared:
+        raw = translation_unit.source_text if translation_unit.source_text is not None else translation_unit.source.path.read_text(encoding="utf-8", errors="replace")
+        masked = _mask_comments_and_literals(raw)
+        # Preprocessor include/pragma lines do not participate in lexical C
+        # structure.  Blank them while preserving byte/character offsets.
+        code_only = re.sub(r"(?m)^[ \t]*#.*$", lambda m: " " * len(m.group(0)), masked)
+        reasons: list[str] = []
+        if translation_unit.arguments:
+            # Compile flags can alter parsing/active code.  Until the fast path
+            # has a real preprocessor, defer such TUs to clang.
+            reasons.append("compile-arguments")
+        for name, pattern in _COMPLEX_PATTERNS[:2]:
+            if pattern.search(masked):
+                reasons.append(name)
+        for name, pattern in _COMPLEX_PATTERNS[2:]:
+            if pattern.search(code_only):
+                reasons.append(name)
+        spans = tuple(_function_spans(code_only))
+        if not spans and re.search(r"\b[A-Za-z_]\w*\s*\([^;{}]*\)\s*\{", code_only):
+            reasons.append("unrecognized-function-header")
+        # Calls nested under control flow need region/guard provenance.  Do not
+        # rescan every function body here: parse_prepared already scans bodies
+        # for calls.  If that scan finds a real call in a function containing an
+        # `if`, it raises and HybridParser refines the TU with clang.
+        return FastPrepared(
+            raw=raw,
+            code_only=code_only,
+            spans=spans,
+            line_starts=tuple(_line_starts(raw)),
+            eligibility=FastEligibility(not reasons, tuple(dict.fromkeys(reasons))),
+        )
+
+    def parse(self, translation_unit: TranslationUnit) -> FileParseArtifact:
+        prepared = self.prepare(translation_unit)
+        if not prepared.eligibility.eligible:
+            raise ValueError("TU is not eligible for fast parser: " + ", ".join(prepared.eligibility.reasons))
+        return self.parse_prepared(translation_unit, prepared)
+
+    def parse_prepared(self, translation_unit: TranslationUnit, prepared: FastPrepared) -> FileParseArtifact:
+        raw = prepared.raw
+        masked = prepared.code_only
+        line_starts = prepared.line_starts
+        functions = CompactFunctionTable(translation_unit.source.project_relative_path)
+        callsites: list[CallSiteRecord] = []
+
+        line_cursor = 0
+        line_count = len(line_starts)
+
+        def line_col_monotonic(pos: int) -> tuple[int, int]:
+            nonlocal line_cursor
+            while line_cursor + 1 < line_count and line_starts[line_cursor + 1] <= pos:
+                line_cursor += 1
+            return line_cursor + 1, pos - line_starts[line_cursor] + 1
+
+        for name, body_start, body_end, header_start, name_start, storage, params_text in prepared.spans:
+            start_line, start_col = line_col_monotonic(name_start)
+            end_line, end_col = line_col_monotonic(body_end)
+            storage = sys.intern(storage) if storage else None
+            tq = raw[header_start:name_start].strip()
+            type_qualifier = sys.intern(tq) if tq else None
+            functions.append_fields(
+                name, start_line, start_col, end_line, end_col,
+                storage, type_qualifier, _parameter_names(params_text),
+            )
+
+            body = masked[body_start:body_end]
+            caller_id: str | None = None
+            has_if = bool(re.search(r"\bif\s*\(", body))
+            for match in _CALL_RE.finditer(body):
+                callee = match.group(1)
+                if callee in _C_KEYWORDS:
+                    continue
+                # A real call in a function that contains control flow needs
+                # region provenance. Refine rather than approximating it.
+                if has_if:
+                    raise ValueError("call-under-if requires semantic refinement")
+                absolute = body_start + match.start(1)
+                open_paren = body_start + match.end(0) - 1
+                call_end = _balanced_call_end(masked, open_paren, body_end)
+                if call_end is None:
+                    continue
+                line, col = line_col_monotonic(absolute)
+                end_l, end_c = line_col_monotonic(call_end)
+                if caller_id is None:
+                    caller_id = _function_id(
+                        translation_unit, name, start_line, start_col, storage, type_qualifier
+                    )
+                callsites.append(CallSiteRecord(
+                    caller_id=caller_id,
+                    caller_name=name,
+                    callee_name=callee,
+                    raw_text=raw[absolute:call_end + 1].strip(),
+                    source_path=translation_unit.source.project_relative_path,
+                    source_range=SourceRange(line, col, end_l, end_c),
+                ))
+
+        return FileParseArtifact(
+            translation_unit=translation_unit,
+            status=ParseStatus.OK,
+            functions=functions,
+            callsites=callsites,
+            parser_backend=self.name,
+        )
+
+
+def _function_spans(text: str) -> list[tuple[str, int, int, int, int, str | None, str]]:
+    out: list[tuple[str, int, int, int, int, str | None, str]] = []
+    for m in _FUNC_RE.finditer(text):
+        brace = m.end() - 1
+        end = _matching_brace(text, brace)
+        if end is None:
+            continue
+        out.append((m.group("name"), brace + 1, end, m.start("header"), m.start("name"), m.group("storage"), m.group("params")))
+    return out
+
+
+def _matching_brace(text: str, start: int) -> int | None:
+    depth = 0
+    for i in range(start, len(text)):
+        c = text[i]
+        if c == "{":
+            depth += 1
+        elif c == "}":
+            depth -= 1
+            if depth == 0:
+                return i
+    return None
+
+
+def _balanced_call_end(text: str, open_paren: int, body_end: int) -> int | None:
+    depth = 0
+    limit = min(len(text), body_end + 1)
+    for i in range(open_paren, limit):
+        c = text[i]
+        if c == "(":
+            depth += 1
+        elif c == ")":
+            depth -= 1
+            if depth == 0:
+                return i
+    return None
+
+
+def _line_starts(text: str) -> list[int]:
+    starts = [0]
+    starts.extend(i + 1 for i, c in enumerate(text) if c == "\n")
+    return starts
+
+
+def _line_col(pos: int, line_starts: tuple[int, ...] | list[int]) -> tuple[int, int]:
+    idx = bisect_right(line_starts, pos) - 1
+    return idx + 1, pos - line_starts[idx] + 1
+
+
+@lru_cache(maxsize=4096)
+def _parameter_names(params_text: str) -> tuple[str, ...]:
+    text = params_text.strip()
+    if not text or text == "void":
+        return ()
+    names: list[str] = []
+    for part in text.split(","):
+        part = part.strip()
+        if not part or part == "...":
+            continue
+        ids = re.findall(r"[A-Za-z_]\w*", part)
+        if ids:
+            names.append(sys.intern(ids[-1]))
+    return tuple(names)
+
+
+def _function_id(tu: TranslationUnit, name: str, line: int, col: int, storage: str | None, qual_type: str | None) -> str:
+    return f"{tu.source.project_relative_path}:{line}:{col}:{storage or 'external'}:{name}:{qual_type or '?'}"
+
+
+def _mask_comments_and_literals(text: str) -> str:
+    out = list(text)
+    i = 0
+    n = len(out)
+    while i < n:
+        if i + 1 < n and text[i:i+2] == "//":
+            j = text.find("\n", i + 2)
+            j = n if j < 0 else j
+            for k in range(i, j):
+                out[k] = " "
+            i = j
+            continue
+        if i + 1 < n and text[i:i+2] == "/*":
+            j = text.find("*/", i + 2)
+            j = n - 2 if j < 0 else j
+            for k in range(i, min(n, j + 2)):
+                if out[k] != "\n":
+                    out[k] = " "
+            i = min(n, j + 2)
+            continue
+        if text[i] in {'"', "'"}:
+            quote = text[i]
+            j = i + 1
+            while j < n:
+                if text[j] == "\\":
+                    j += 2
+                    continue
+                if text[j] == quote:
+                    j += 1
+                    break
+                j += 1
+            for k in range(i, min(n, j)):
+                if out[k] != "\n":
+                    out[k] = " "
+            i = j
+            continue
+        i += 1
+    return "".join(out)