cseq 0.0.1__py3-none-any.whl

cseq/cache.py

@@ -0,0 +1,420 @@
+from __future__ import annotations
+
+from dataclasses import asdict
+from hashlib import sha256
+import json
+from pathlib import Path
+from typing import Any
+
+from .model import (
+    CallSiteRecord, CallTargetKind, CompactFunctionRecord, CompactFunctionTable, Diagnostic, FileParseArtifact, FlowEdge, FlowEdgeKind,
+    FunctionRecord, GuardedSourceRegion, ParseStatus, SourceRange, TranslationUnit, ValueSlot, ValueSlotKind,
+    ProjectIndex, CpuEvidenceRecord, CpuEvidenceLayer, CpuValueKind,
+)
+
+CACHE_SCHEMA_VERSION = 7
+
+
+def build_cache_key(tu: TranslationUnit, *, parser_backend: str, parser_version: str = "1", include_dependency_hash: str | None = None) -> str:
+    payload = {
+        "schema": CACHE_SCHEMA_VERSION,
+        "source_hash": tu.source.content_hash,
+        "source_path": tu.source.project_relative_path,
+        "arguments": list(tu.arguments),
+        "configuration": tu.configuration_name,
+        "parser_backend": parser_backend,
+        "parser_version": parser_version,
+        "include_dependency_hash": include_dependency_hash,
+    }
+    raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
+    return sha256(raw).hexdigest()
+
+
+class JsonCache:
+    def __init__(self, root: str | Path) -> None:
+        self.root = Path(root)
+        self.root.mkdir(parents=True, exist_ok=True)
+
+    def path_for(self, key: str) -> Path:
+        return self.root / key[:2] / f"{key}.json"
+
+    def put(self, key: str, payload: dict[str, Any]) -> Path:
+        path = self.path_for(key)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        wrapper = {"schema": CACHE_SCHEMA_VERSION, "payload": payload}
+        tmp = path.with_suffix(".tmp")
+        tmp.write_text(json.dumps(wrapper, ensure_ascii=False, sort_keys=True), encoding="utf-8")
+        tmp.replace(path)
+        return path
+
+    def get(self, key: str) -> dict[str, Any] | None:
+        path = self.path_for(key)
+        if not path.exists():
+            return None
+        try:
+            wrapper = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            return None
+        if wrapper.get("schema") != CACHE_SCHEMA_VERSION:
+            return None
+        payload = wrapper.get("payload")
+        return payload if isinstance(payload, dict) else None
+
+    def remove(self, key: str) -> bool:
+        path = self.path_for(key)
+        if not path.exists():
+            return False
+        path.unlink()
+        return True
+
+
+
+def _range_to_dict(r: SourceRange) -> dict[str, int | None]:
+    return {
+        "start_line": r.start_line,
+        "start_column": r.start_column,
+        "end_line": r.end_line,
+        "end_column": r.end_column,
+    }
+
+
+def _range_from_dict(d: dict[str, Any] | None) -> SourceRange:
+    d = d or {}
+    return SourceRange(d.get("start_line"), d.get("start_column"), d.get("end_line"), d.get("end_column"))
+
+
+def serialize_parse_artifact(artifact: FileParseArtifact) -> dict[str, Any]:
+    """Serialize parser output without embedding source bytes.
+
+    Fast lexical artifacts use a compact representation because their function
+    records are numerous and highly repetitive on generated/large projects.
+    The semantic representation remains explicit for debuggability.
+    """
+    if artifact.parser_backend.startswith("fast-lexical"):
+        funcs = artifact.functions
+        function_index = None if isinstance(funcs, CompactFunctionTable) else {f.qualified_id: i for i, f in enumerate(funcs)}
+        def caller_index(c):
+            if isinstance(funcs, CompactFunctionTable):
+                return funcs.index_of_qid(c.caller_id)
+            return function_index.get(c.caller_id, -1)
+        return {
+            "format": "fast-compact-v2",
+            "status": artifact.status.value,
+            "parser_backend": artifact.parser_backend,
+            "functions": [
+                [
+                    f.name, f.source_range.start_line, f.source_range.start_column,
+                    f.source_range.end_line, f.source_range.end_column,
+                    f.storage_class, f.type_qualifier, list(f.parameters),
+                ]
+                for f in artifact.functions
+            ],
+            "callsites": [
+                [
+                    c.caller_id, c.caller_name, c.callee_name, c.raw_text,
+                    c.source_range.start_line, c.source_range.start_column,
+                    c.source_range.end_line, c.source_range.end_column,
+                ]
+                for c in artifact.callsites
+            ],
+        }
+    return {
+        "status": artifact.status.value,
+        "parser_backend": artifact.parser_backend,
+        "functions": [
+            {
+                "name": f.name, "qualified_id": f.qualified_id, "source_path": f.source_path,
+                "source_range": _range_to_dict(f.source_range), "storage_class": f.storage_class,
+                "type_qualifier": f.type_qualifier, "parameters": list(f.parameters),
+            } for f in artifact.functions
+        ],
+        "callsites": [
+            {
+                "caller_id": c.caller_id, "caller_name": c.caller_name, "callee_name": c.callee_name,
+                "raw_text": c.raw_text, "source_path": c.source_path,
+                "source_range": _range_to_dict(c.source_range), "target_kind": c.target_kind.value,
+                "target_function_id": c.target_function_id, "target_function_ids": list(c.target_function_ids),
+                "unknown_possible": c.unknown_possible, "callee_slot_key": c.callee_slot_key,
+                "argument_slot_keys": list(c.argument_slot_keys), "control_context": list(c.control_context),
+                "macro_expanded": c.macro_expanded, "spelling_range": _range_to_dict(c.spelling_range) if c.spelling_range else None,
+                "expansion_range": _range_to_dict(c.expansion_range) if c.expansion_range else None, "guard_expression": c.guard_expression,
+            } for c in artifact.callsites
+        ],
+        "value_slots": [
+            {
+                "key": v.key, "kind": v.kind.value, "name": v.name, "function_id": v.function_id,
+                "field_name": v.field_name, "index": v.index,
+            } for v in artifact.value_slots
+        ],
+        "flow_edges": [
+            {
+                "source_key": e.source_key, "target_key": e.target_key, "kind": e.kind.value,
+                "source_path": e.source_path, "source_range": _range_to_dict(e.source_range),
+            } for e in artifact.flow_edges
+        ],
+        "diagnostics": [{"severity": d.severity, "message": d.message, "raw": d.raw} for d in artifact.diagnostics],
+        "opaque_regions": [_range_to_dict(r) for r in artifact.opaque_regions],
+        "guarded_regions": [
+            {"source_path": g.source_path, "source_range": _range_to_dict(g.source_range), "guard_expression": g.guard_expression}
+            for g in artifact.guarded_regions
+        ],
+    }
+
+
+def serialize_parse_artifact_light(artifact: FileParseArtifact) -> dict[str, Any]:
+    """Serialize a parse artifact without serializing its function table at all.
+
+    This is intentionally separate from serialize_parse_artifact(): building a
+    full million-function JSON object only to delete ``functions`` defeats the
+    out-of-core design. Caller IDs are self-contained in cache schema v7.
+    """
+    if artifact.parser_backend.startswith("fast-lexical"):
+        return {
+            "format": "fast-compact-v2",
+            "status": artifact.status.value,
+            "parser_backend": artifact.parser_backend,
+            "functions": [],
+            "callsites": [
+                [
+                    c.caller_id, c.caller_name, c.callee_name, c.raw_text,
+                    c.source_range.start_line, c.source_range.start_column,
+                    c.source_range.end_line, c.source_range.end_column,
+                ]
+                for c in artifact.callsites
+            ],
+        }
+    return {
+        "status": artifact.status.value,
+        "parser_backend": artifact.parser_backend,
+        "functions": [],
+        "callsites": [
+            {
+                "caller_id": c.caller_id, "caller_name": c.caller_name, "callee_name": c.callee_name,
+                "raw_text": c.raw_text, "source_path": c.source_path,
+                "source_range": _range_to_dict(c.source_range), "target_kind": c.target_kind.value,
+                "target_function_id": c.target_function_id, "target_function_ids": list(c.target_function_ids),
+                "unknown_possible": c.unknown_possible, "callee_slot_key": c.callee_slot_key,
+                "argument_slot_keys": list(c.argument_slot_keys), "control_context": list(c.control_context),
+                "macro_expanded": c.macro_expanded, "spelling_range": _range_to_dict(c.spelling_range) if c.spelling_range else None,
+                "expansion_range": _range_to_dict(c.expansion_range) if c.expansion_range else None, "guard_expression": c.guard_expression,
+            } for c in artifact.callsites
+        ],
+        "value_slots": [
+            {
+                "key": v.key, "kind": v.kind.value, "name": v.name, "function_id": v.function_id,
+                "field_name": v.field_name, "index": v.index,
+            } for v in artifact.value_slots
+        ],
+        "flow_edges": [
+            {
+                "source_key": e.source_key, "target_key": e.target_key, "kind": e.kind.value,
+                "source_path": e.source_path, "source_range": _range_to_dict(e.source_range),
+            } for e in artifact.flow_edges
+        ],
+        "diagnostics": [{"severity": d.severity, "message": d.message, "raw": d.raw} for d in artifact.diagnostics],
+        "opaque_regions": [_range_to_dict(r) for r in artifact.opaque_regions],
+        "guarded_regions": [
+            {"source_path": g.source_path, "source_range": _range_to_dict(g.source_range), "guard_expression": g.guard_expression}
+            for g in artifact.guarded_regions
+        ],
+    }
+
+
+def strip_parse_artifact_functions(payload: dict[str, Any]) -> dict[str, Any]:
+    """Return a lightweight parse payload whose function table lives elsewhere.
+
+    CallSite caller IDs are self-contained in cache schema v7, so Static Symbol
+    Store reuse can restore every non-function parser artifact without loading
+    the often-dominant function array.
+    """
+    light = dict(payload)
+    light["functions"] = []
+    return light
+
+
+def deserialize_parse_artifact(payload: dict[str, Any], tu: TranslationUnit, *, include_functions: bool = True) -> FileParseArtifact:
+    if payload.get("format") == "fast-compact-v2":
+        functions = CompactFunctionTable(tu.source.project_relative_path)
+        if include_functions:
+            for x in payload.get("functions", []):
+                name, sl, sc, el, ec, storage, type_qualifier, params = x
+                functions.append_fields(
+                    name, sl, sc, el, ec, storage, type_qualifier, tuple(params or ()),
+                )
+        callsites: list[CallSiteRecord] = []
+        for x in payload.get("callsites", []):
+            caller_id, caller_name, callee_name, raw_text, sl, sc, el, ec = x
+            callsites.append(CallSiteRecord(
+                caller_id=caller_id, caller_name=caller_name, callee_name=callee_name,
+                raw_text=raw_text, source_path=tu.source.project_relative_path,
+                source_range=SourceRange(sl, sc, el, ec),
+            ))
+        return FileParseArtifact(
+            translation_unit=tu, status=ParseStatus(payload["status"]),
+            functions=functions, callsites=callsites,
+            parser_backend=str(payload.get("parser_backend") or "fast-lexical-v1"),
+        )
+    return FileParseArtifact(
+        translation_unit=tu,
+        status=ParseStatus(payload["status"]),
+        functions=[FunctionRecord(
+            name=x["name"], qualified_id=x["qualified_id"], source_path=x["source_path"],
+            source_range=_range_from_dict(x.get("source_range")), storage_class=x.get("storage_class"),
+            type_qualifier=x.get("type_qualifier"), parameters=tuple(x.get("parameters") or ()),
+        ) for x in payload.get("functions", [])] if include_functions else [],
+        callsites=[CallSiteRecord(
+            caller_id=x["caller_id"], caller_name=x["caller_name"], callee_name=x.get("callee_name"),
+            raw_text=x["raw_text"], source_path=x["source_path"], source_range=_range_from_dict(x.get("source_range")),
+            target_kind=CallTargetKind(x.get("target_kind", "UNKNOWN")), target_function_id=x.get("target_function_id"),
+            target_function_ids=tuple(x.get("target_function_ids") or ()), unknown_possible=bool(x.get("unknown_possible")),
+            callee_slot_key=x.get("callee_slot_key"), argument_slot_keys=tuple(x.get("argument_slot_keys") or ()), control_context=tuple(x.get("control_context") or ()),
+            macro_expanded=bool(x.get("macro_expanded")), spelling_range=_range_from_dict(x.get("spelling_range")) if x.get("spelling_range") else None,
+            expansion_range=_range_from_dict(x.get("expansion_range")) if x.get("expansion_range") else None, guard_expression=x.get("guard_expression"),
+        ) for x in payload.get("callsites", [])],
+        value_slots=[ValueSlot(
+            key=x["key"], kind=ValueSlotKind(x["kind"]), name=x["name"], function_id=x.get("function_id"),
+            field_name=x.get("field_name"), index=x.get("index"),
+        ) for x in payload.get("value_slots", [])],
+        flow_edges=[FlowEdge(
+            source_key=x["source_key"], target_key=x["target_key"], kind=FlowEdgeKind(x["kind"]),
+            source_path=x.get("source_path"), source_range=_range_from_dict(x.get("source_range")),
+        ) for x in payload.get("flow_edges", [])],
+        diagnostics=[Diagnostic(x["severity"], x["message"], x.get("raw")) for x in payload.get("diagnostics", [])],
+        opaque_regions=[_range_from_dict(x) for x in payload.get("opaque_regions", [])],
+        guarded_regions=[GuardedSourceRegion(
+            source_path=x.get("source_path") or tu.source.project_relative_path,
+            source_range=_range_from_dict(x.get("source_range")),
+            guard_expression=str(x.get("guard_expression") or "<unknown>"),
+        ) for x in payload.get("guarded_regions", [])],
+        parser_backend=str(payload.get("parser_backend") or "unknown"),
+    )
+
+
+ANALYSIS_CACHE_SCHEMA_VERSION = 7
+
+
+def _config_fingerprint_payload(config, defined_macros: tuple[str, ...]) -> dict[str, Any]:
+    return {
+        "cpu_groups": {k: list(v) for k, v in sorted(config.cpu_groups.items())},
+        "preprocessor_rules": [
+            [r.when_defined, r.assign_group, r.assign_cpu] for r in config.preprocessor_rules
+        ],
+        "path_rules": [[r.glob, r.assign_group, r.assign_cpu] for r in config.path_rules],
+        "override_rules": [[r.file, r.assign_group, r.assign_cpu] for r in config.override_rules],
+        "configurations": {k: list(v) for k, v in sorted(config.configurations.items())},
+        "defined_macros": list(defined_macros),
+    }
+
+
+def build_analysis_cache_key(index: ProjectIndex, config, defined_macros: tuple[str, ...]) -> str:
+    """Fingerprint project-wide symbol/value-flow inputs.
+
+    Parser cache is TU-local. This key intentionally spans every parse artifact,
+    so a symbol change in another translation unit invalidates project-wide call
+    resolution without forcing unrelated files to reparse.
+    """
+    payload = {
+        "schema": ANALYSIS_CACHE_SCHEMA_VERSION,
+        "artifacts": [serialize_parse_artifact(a) for a in index.parse_artifacts],
+        "config": _config_fingerprint_payload(config, defined_macros),
+    }
+    raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
+    return sha256(raw).hexdigest()
+
+
+def build_analysis_cache_key_from_fingerprints(
+    artifact_fingerprints: list[str] | tuple[str, ...], config, defined_macros: tuple[str, ...]
+) -> str:
+    """Build the project-analysis key from TU parser fingerprints.
+
+    Each parser fingerprint already covers source content, compile arguments,
+    configuration, parser backend/schema and include dependencies. Re-serializing
+    every FunctionRecord/CallSite to build a second hash is redundant and becomes
+    very expensive on million-function projects.
+    """
+    payload = {
+        "schema": ANALYSIS_CACHE_SCHEMA_VERSION,
+        "artifact_fingerprints": list(artifact_fingerprints),
+        "config": _config_fingerprint_payload(config, defined_macros),
+    }
+    raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
+    return sha256(raw).hexdigest()
+
+
+def serialize_analysis_snapshot(index: ProjectIndex) -> dict[str, Any]:
+    return {
+        "analysis_schema": ANALYSIS_CACHE_SCHEMA_VERSION,
+        "callsites": [
+            {
+                "caller_id": c.caller_id,
+                "caller_name": c.caller_name,
+                "callee_name": c.callee_name,
+                "raw_text": c.raw_text,
+                "source_path": c.source_path,
+                "source_range": _range_to_dict(c.source_range),
+                "target_kind": c.target_kind.value,
+                "target_function_id": c.target_function_id,
+                "target_function_ids": list(c.target_function_ids),
+                "unknown_possible": c.unknown_possible,
+                "callee_slot_key": c.callee_slot_key,
+                "argument_slot_keys": list(c.argument_slot_keys), "control_context": list(c.control_context),
+                "macro_expanded": c.macro_expanded, "spelling_range": _range_to_dict(c.spelling_range) if c.spelling_range else None,
+                "expansion_range": _range_to_dict(c.expansion_range) if c.expansion_range else None, "guard_expression": c.guard_expression,
+            }
+            for c in index.callsites
+        ],
+        "flow_edges": [
+            {
+                "source_key": e.source_key,
+                "target_key": e.target_key,
+                "kind": e.kind.value,
+                "source_path": e.source_path,
+                "source_range": _range_to_dict(e.source_range),
+            }
+            for e in index.flow_edges
+        ],
+        "cpu_evidence": [
+            {
+                "target_function_id": e.target_function_id,
+                "layer": e.layer.value,
+                "value_kind": e.value_kind.value,
+                "value": e.value,
+                "provenance": e.provenance,
+                "explicit_override": e.explicit_override,
+            }
+            for e in index.cpu_evidence
+        ],
+    }
+
+
+def restore_analysis_snapshot(index: ProjectIndex, payload: dict[str, Any]) -> bool:
+    if payload.get("analysis_schema") != ANALYSIS_CACHE_SCHEMA_VERSION:
+        return False
+    index.callsites[:] = [
+        CallSiteRecord(
+            caller_id=x["caller_id"], caller_name=x["caller_name"], callee_name=x.get("callee_name"),
+            raw_text=x["raw_text"], source_path=x["source_path"], source_range=_range_from_dict(x.get("source_range")),
+            target_kind=CallTargetKind(x.get("target_kind", "UNKNOWN")), target_function_id=x.get("target_function_id"),
+            target_function_ids=tuple(x.get("target_function_ids") or ()), unknown_possible=bool(x.get("unknown_possible")),
+            callee_slot_key=x.get("callee_slot_key"), argument_slot_keys=tuple(x.get("argument_slot_keys") or ()), control_context=tuple(x.get("control_context") or ()),
+            macro_expanded=bool(x.get("macro_expanded")), spelling_range=_range_from_dict(x.get("spelling_range")) if x.get("spelling_range") else None,
+            expansion_range=_range_from_dict(x.get("expansion_range")) if x.get("expansion_range") else None, guard_expression=x.get("guard_expression"),
+        )
+        for x in payload.get("callsites", [])
+    ]
+    index.flow_edges[:] = [
+        FlowEdge(
+            source_key=x["source_key"], target_key=x["target_key"], kind=FlowEdgeKind(x["kind"]),
+            source_path=x.get("source_path"), source_range=_range_from_dict(x.get("source_range")),
+        )
+        for x in payload.get("flow_edges", [])
+    ]
+    index.cpu_evidence[:] = [
+        CpuEvidenceRecord(
+            target_function_id=x["target_function_id"], layer=CpuEvidenceLayer(x["layer"]),
+            value_kind=CpuValueKind(x["value_kind"]), value=x["value"], provenance=x["provenance"],
+            explicit_override=bool(x.get("explicit_override")),
+        )
+        for x in payload.get("cpu_evidence", [])
+    ]
+    return True