hypergumbo-lang-scip-python 8.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hypergumbo_lang_scip_python/__init__.py +42 -0
- hypergumbo_lang_scip_python/analyzer.py +141 -0
- hypergumbo_lang_scip_python/gate.py +56 -0
- hypergumbo_lang_scip_python/invoke.py +103 -0
- hypergumbo_lang_scip_python/translate.py +133 -0
- hypergumbo_lang_scip_python-8.1.0.dist-info/METADATA +84 -0
- hypergumbo_lang_scip_python-8.1.0.dist-info/RECORD +9 -0
- hypergumbo_lang_scip_python-8.1.0.dist-info/WHEEL +4 -0
- hypergumbo_lang_scip_python-8.1.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
"""Hypergumbo SCIP-backed Python analyzer (scip-python / pyright), WI-nanom.
|
|
3
|
+
|
|
4
|
+
The second opt-in fidelity backend and the first that executes nothing from
|
|
5
|
+
the analysed repository. Its parts mirror ``hypergumbo_lang_rust_analyzer``:
|
|
6
|
+
|
|
7
|
+
- ``invoke.py`` shells out to ``scip-python index`` to produce a SCIP blob.
|
|
8
|
+
- ``translate.py`` turns that blob into ``(symbols, edges)`` through the
|
|
9
|
+
shared importer, plus the one thing the Rust arm never did: external
|
|
10
|
+
callees keep the receiver's MODULE.
|
|
11
|
+
- ``gate.py`` answers "did the user ask?" — flag > environment > config
|
|
12
|
+
tiers (a preference, ADR-0045 ruling 5) — and whether the binary is there.
|
|
13
|
+
- ``analyzer.py`` registers ``scip_python`` beside the ``python`` analyzer
|
|
14
|
+
with the merge anchor the ADR-0057 pass pairs on.
|
|
15
|
+
"""
|
|
16
|
+
from hypergumbo_lang_scip_python.gate import should_use_scip_python_backend
|
|
17
|
+
from hypergumbo_lang_scip_python.invoke import (
|
|
18
|
+
ScipPythonError,
|
|
19
|
+
ScipPythonInvocationFailed,
|
|
20
|
+
ScipPythonNoOutput,
|
|
21
|
+
ScipPythonNotInstalled,
|
|
22
|
+
run_scip_python_index,
|
|
23
|
+
)
|
|
24
|
+
from hypergumbo_lang_scip_python.translate import translate_scip_python_to_hg
|
|
25
|
+
|
|
26
|
+
__version__ = "8.1.0"
|
|
27
|
+
|
|
28
|
+
#: Module paths for analyzer discovery via entry-points (ADR-0012 Step 1).
|
|
29
|
+
ANALYZER_MODULES = [
|
|
30
|
+
"hypergumbo_lang_scip_python.analyzer",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
__all__ = [
|
|
34
|
+
"ANALYZER_MODULES",
|
|
35
|
+
"ScipPythonError",
|
|
36
|
+
"ScipPythonInvocationFailed",
|
|
37
|
+
"ScipPythonNoOutput",
|
|
38
|
+
"ScipPythonNotInstalled",
|
|
39
|
+
"run_scip_python_index",
|
|
40
|
+
"should_use_scip_python_backend",
|
|
41
|
+
"translate_scip_python_to_hg",
|
|
42
|
+
]
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
"""Registered analyzer entry point for the scip-python backend (WI-nanom).
|
|
3
|
+
|
|
4
|
+
Registered as ``scip_python``: a second BACKEND for the language ``python``,
|
|
5
|
+
beside the ast-based ``python`` analyzer (ADR-0057 §10). The registry can
|
|
6
|
+
see that the two are producers for one language and how their records
|
|
7
|
+
pair: this arm's ``Symbol.name`` is the SCIP descriptor name as emitted
|
|
8
|
+
(``area``, never ``Shape.area``) and its ``Symbol.span`` is the identifier
|
|
9
|
+
TOKEN of the Definition occurrence, so the merge pass tests token-inside-item
|
|
10
|
+
against the syntax arm's item span. No measured authority is declared —
|
|
11
|
+
the first two-arm table (lab notebook 2026-09-19) showed the arms agree on
|
|
12
|
+
``kind`` 4,721 times out of 4,723 — and ``executes_analysed_code`` is
|
|
13
|
+
False: pyright infers, it does not run the analysed code, which is why the
|
|
14
|
+
opt-in may live in a config file (ADR-0045 ruling 5).
|
|
15
|
+
|
|
16
|
+
When the gate is False (the default) the analyzer returns a skipped result
|
|
17
|
+
with ``backend_disabled``, so the orchestrator records an honest
|
|
18
|
+
``skipped_passes`` entry. When the binary is missing, exits non-zero or
|
|
19
|
+
writes no index, the corresponding pass-silence code travels with the skip
|
|
20
|
+
(``dependency_unavailable`` / ``pass_crashed`` / ``unreported``) — the same
|
|
21
|
+
discrimination WI-luvud gave the Rust arm. A completed run that produced
|
|
22
|
+
nothing is NOT a skip: the pass ran, and says so.
|
|
23
|
+
"""
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import tempfile
|
|
27
|
+
import warnings
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import Callable, Optional
|
|
30
|
+
|
|
31
|
+
from google.protobuf.message import DecodeError
|
|
32
|
+
|
|
33
|
+
from hypergumbo_core.analyze.base import AnalysisResult
|
|
34
|
+
from hypergumbo_core.analyze.registry import (
|
|
35
|
+
SPAN_ROLE_TOKEN,
|
|
36
|
+
MergeAnchor,
|
|
37
|
+
as_emitted,
|
|
38
|
+
register_analyzer,
|
|
39
|
+
)
|
|
40
|
+
from hypergumbo_core.ir import PASS_VERSION, AnalysisRun, Edge, Symbol
|
|
41
|
+
from hypergumbo_core.pass_silence import (
|
|
42
|
+
BACKEND_DISABLED,
|
|
43
|
+
DEPENDENCY_UNAVAILABLE,
|
|
44
|
+
PASS_CRASHED,
|
|
45
|
+
UNREPORTED,
|
|
46
|
+
silence_reason_for_candidates,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
from hypergumbo_lang_scip_python.gate import should_use_scip_python_backend
|
|
50
|
+
from hypergumbo_lang_scip_python.invoke import (
|
|
51
|
+
ScipPythonError,
|
|
52
|
+
ScipPythonNoOutput,
|
|
53
|
+
ScipPythonNotInstalled,
|
|
54
|
+
run_scip_python_index,
|
|
55
|
+
)
|
|
56
|
+
from hypergumbo_lang_scip_python.translate import translate_scip_python_to_hg
|
|
57
|
+
|
|
58
|
+
InvokeFn = Callable[..., bytes]
|
|
59
|
+
TranslateFn = Callable[..., "tuple[list[Symbol], list[Edge]]"]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _silence_code_for(exc: Exception) -> str:
|
|
63
|
+
"""Map an invocation failure onto the closed pass-silence axis (see the module docstring)."""
|
|
64
|
+
if isinstance(exc, ScipPythonNotInstalled):
|
|
65
|
+
return DEPENDENCY_UNAVAILABLE
|
|
66
|
+
if isinstance(exc, ScipPythonNoOutput):
|
|
67
|
+
return UNREPORTED
|
|
68
|
+
return PASS_CRASHED
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _emit_user_warning(message: str) -> None:
|
|
72
|
+
warnings.warn(message, UserWarning, stacklevel=3)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _repo_has_py_files(repo_root: Path) -> bool:
|
|
76
|
+
try:
|
|
77
|
+
return next(iter(repo_root.rglob("*.py")), None) is not None
|
|
78
|
+
except OSError: # pragma: no cover — pure defensive
|
|
79
|
+
return False
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def analyze_python_with_scip_impl(
|
|
83
|
+
repo_root: Path,
|
|
84
|
+
*,
|
|
85
|
+
invoke: Optional[InvokeFn] = None,
|
|
86
|
+
translate: Optional[TranslateFn] = None,
|
|
87
|
+
gate: Optional[Callable[..., bool]] = None,
|
|
88
|
+
log: Optional[Callable[[str], None]] = None,
|
|
89
|
+
) -> AnalysisResult:
|
|
90
|
+
"""The body of the registered analyzer, with every collaborator injectable."""
|
|
91
|
+
decide = gate if gate is not None else should_use_scip_python_backend
|
|
92
|
+
if not decide(repo_root=repo_root):
|
|
93
|
+
return AnalysisResult(
|
|
94
|
+
skipped=True,
|
|
95
|
+
skip_reason="scip-python backend not enabled",
|
|
96
|
+
skip_reason_code=BACKEND_DISABLED,
|
|
97
|
+
)
|
|
98
|
+
run = AnalysisRun.create( # nosec B106 — a pass NAME, not a password
|
|
99
|
+
pass_id="scip_python", version=PASS_VERSION,
|
|
100
|
+
)
|
|
101
|
+
do_invoke = invoke if invoke is not None else run_scip_python_index
|
|
102
|
+
do_translate = translate if translate is not None else translate_scip_python_to_hg
|
|
103
|
+
emit = log if log is not None else _emit_user_warning
|
|
104
|
+
try:
|
|
105
|
+
with tempfile.TemporaryDirectory(prefix="hypergumbo-scip-python-") as scratch:
|
|
106
|
+
scip_bytes = do_invoke(repo_root, cwd=Path(scratch), project_name=repo_root.name or "project")
|
|
107
|
+
symbols, edges = do_translate(scip_bytes, run_id=run.execution_id)
|
|
108
|
+
except (ScipPythonError, DecodeError) as exc:
|
|
109
|
+
emit(f"scip-python backend did not run on {repo_root}: {exc}")
|
|
110
|
+
return AnalysisResult(
|
|
111
|
+
skipped=True,
|
|
112
|
+
skip_reason=f"scip-python backend: {exc}",
|
|
113
|
+
skip_reason_code=_silence_code_for(exc),
|
|
114
|
+
)
|
|
115
|
+
if not symbols and _repo_has_py_files(repo_root):
|
|
116
|
+
emit(
|
|
117
|
+
f"scip-python backend produced no symbols for {repo_root} despite .py "
|
|
118
|
+
f"files being present — likely a project scip-python could not index.",
|
|
119
|
+
)
|
|
120
|
+
run.silence_reason = silence_reason_for_candidates(symbols)
|
|
121
|
+
return AnalysisResult(run=run, symbols=symbols, edges=edges)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@register_analyzer(
|
|
125
|
+
"scip_python",
|
|
126
|
+
priority=45,
|
|
127
|
+
languages=["python"],
|
|
128
|
+
backend="scip",
|
|
129
|
+
# INV-gabak: the shared SCIP translation stamps this synthetic pass id
|
|
130
|
+
# (ADR-0044, catalog._SYNTHETIC_PASS_IDS) on every record it emits, not
|
|
131
|
+
# this analyzer's registration name. Declared so a consumer joining
|
|
132
|
+
# Edge.origin back to the registry reads a statement, not a coincidence.
|
|
133
|
+
emits_origin="scip",
|
|
134
|
+
executes_analysed_code=False,
|
|
135
|
+
# As with the Rust SCIP arm: nothing to declare, because no tracked
|
|
136
|
+
# attribute's default is a concrete value any more (INV-kubup).
|
|
137
|
+
merge=MergeAnchor(name_key=as_emitted, span_role=SPAN_ROLE_TOKEN, observes=()),
|
|
138
|
+
)
|
|
139
|
+
def analyze_python_with_scip(repo_root: Path) -> AnalysisResult:
|
|
140
|
+
"""Entry point for the SCIP-backed Python analyzer (see the module docstring)."""
|
|
141
|
+
return analyze_python_with_scip_impl(repo_root)
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
"""Opt-in gate for the scip-python backend (WI-nanom).
|
|
3
|
+
|
|
4
|
+
Two conditions, both required: the user asked, and the binary is there.
|
|
5
|
+
|
|
6
|
+
"The user asked" is :func:`hypergumbo_core.backend_selection
|
|
7
|
+
.resolve_scip_python_optin` — the ADR-0045 ruling 4 chain, flag >
|
|
8
|
+
environment > CONFIG tiers. The config tiers are the tier the Rust backend
|
|
9
|
+
cannot have: enabling rust-analyzer executes the analysed crate's
|
|
10
|
+
``build.rs``, so its durable opt-in is a per-repository trust grant;
|
|
11
|
+
pyright executes nothing, so ``[backends] scip_python = true`` in
|
|
12
|
+
``config.toml`` or ``.hypergumbo.toml`` is an ordinary preference (ruling 5).
|
|
13
|
+
The vocabulary and the ordering live in core, shared with the CLI, so the
|
|
14
|
+
two cannot disagree about what ``--backend tree-sitter`` means.
|
|
15
|
+
|
|
16
|
+
"The binary is there" is :func:`hypergumbo_core.scip_python_install
|
|
17
|
+
.is_scip_python_available` — ``which`` plus a ``--version`` smoke test.
|
|
18
|
+
Opted in but not installed returns False silently here; the CLI's
|
|
19
|
+
``--backend scip-python`` path reports the missing binary at exit 2 before
|
|
20
|
+
a run starts, and the analyzer records ``dependency_unavailable`` if the
|
|
21
|
+
binary vanishes between the gate and the invocation.
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import os
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Callable, Mapping, Optional
|
|
28
|
+
|
|
29
|
+
from hypergumbo_core.backend_selection import (
|
|
30
|
+
SCIP_PYTHON_ENV_VAR,
|
|
31
|
+
resolve_scip_python_optin,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
#: Re-exported so callers and tests import the variable name from here.
|
|
35
|
+
ENV_VAR_NAME = SCIP_PYTHON_ENV_VAR
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def should_use_scip_python_backend(
|
|
39
|
+
*,
|
|
40
|
+
backend_flag: Optional[str] = None,
|
|
41
|
+
environ: Optional[Mapping[str, str]] = None,
|
|
42
|
+
is_available: Optional[Callable[[], bool]] = None,
|
|
43
|
+
repo_root: Optional[Path] = None,
|
|
44
|
+
) -> bool:
|
|
45
|
+
"""True iff the scip-python backend should run for ``repo_root``."""
|
|
46
|
+
env = environ if environ is not None else os.environ
|
|
47
|
+
decision = resolve_scip_python_optin(
|
|
48
|
+
flag_choice=backend_flag, environ=env, repo_root=repo_root,
|
|
49
|
+
)
|
|
50
|
+
if decision is not True:
|
|
51
|
+
return False
|
|
52
|
+
if is_available is None:
|
|
53
|
+
from hypergumbo_core.scip_python_install import is_scip_python_available
|
|
54
|
+
|
|
55
|
+
is_available = is_scip_python_available
|
|
56
|
+
return is_available()
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
"""Shell-out wrapper for ``scip-python index`` (WI-nanom).
|
|
3
|
+
|
|
4
|
+
``scip-python index <project> --project-name NAME --output FILE`` writes a
|
|
5
|
+
SCIP index for a Python project; pyright resolves the project's own
|
|
6
|
+
modules and whatever is importable from the invoking interpreter's
|
|
7
|
+
environment (so a project analysed inside its own virtualenv gets its
|
|
8
|
+
dependencies typed too). It executes NOTHING from the project — static
|
|
9
|
+
inference only — which is why this backend's opt-in is a preference and
|
|
10
|
+
not a trust grant.
|
|
11
|
+
|
|
12
|
+
:func:`run_scip_python_index` confirms the binary resolves, runs it with
|
|
13
|
+
stdout/stderr captured and a timeout, and returns the index bytes. Failure
|
|
14
|
+
modes map to three exceptions the analyzer turns into pass-silence codes:
|
|
15
|
+
not installed (``dependency_unavailable``), non-zero exit or timeout
|
|
16
|
+
(``pass_crashed``), and exit-0-with-no-index (``unreported``). ``which`` and
|
|
17
|
+
``runner`` are injectable so tests never touch ``PATH`` or shell out.
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import shutil
|
|
22
|
+
import subprocess # nosec B404 — required for the scip-python invocation
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Any, Callable, Optional
|
|
25
|
+
|
|
26
|
+
from hypergumbo_core.scip_python_install import SCIP_PYTHON_BINARY
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class ScipPythonError(Exception):
|
|
30
|
+
"""Base class for :func:`run_scip_python_index` failures."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class ScipPythonNotInstalled(ScipPythonError):
|
|
34
|
+
"""The ``scip-python`` binary is not resolvable."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class ScipPythonInvocationFailed(ScipPythonError):
|
|
38
|
+
"""The binary ran but exited non-zero or timed out; carries stderr and the exit code."""
|
|
39
|
+
|
|
40
|
+
def __init__(self, message: str, stderr: bytes, *, returncode: Optional[int] = None) -> None:
|
|
41
|
+
super().__init__(message)
|
|
42
|
+
self.stderr = stderr
|
|
43
|
+
self.returncode = returncode
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class ScipPythonNoOutput(ScipPythonError):
|
|
47
|
+
"""The invocation exited 0 but wrote no index."""
|
|
48
|
+
|
|
49
|
+
def __init__(self, message: str, stderr: bytes) -> None:
|
|
50
|
+
super().__init__(message)
|
|
51
|
+
self.stderr = stderr
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def run_scip_python_index(
|
|
55
|
+
project: Path,
|
|
56
|
+
*,
|
|
57
|
+
cwd: Path,
|
|
58
|
+
project_name: str = "hypergumbo-target",
|
|
59
|
+
scip_python_bin: str = SCIP_PYTHON_BINARY,
|
|
60
|
+
timeout_sec: float = 900.0,
|
|
61
|
+
which: Optional[Callable[[str], Optional[str]]] = None,
|
|
62
|
+
runner: Optional[Callable[..., Any]] = None,
|
|
63
|
+
) -> bytes:
|
|
64
|
+
"""Run ``scip-python index`` on ``project`` and return the SCIP bytes.
|
|
65
|
+
|
|
66
|
+
``cwd`` is a caller-owned scratch directory that receives ``index.scip``;
|
|
67
|
+
the project itself is never written to. The timeout defaults to 15
|
|
68
|
+
minutes: scip-python indexed 179 files in 40 s at 2.1 GB RSS, and a
|
|
69
|
+
large monorepo should fail visibly rather than hang the pipeline.
|
|
70
|
+
"""
|
|
71
|
+
resolve = which if which is not None else shutil.which
|
|
72
|
+
resolved = resolve(scip_python_bin)
|
|
73
|
+
if resolved is None:
|
|
74
|
+
raise ScipPythonNotInstalled(
|
|
75
|
+
f"'{scip_python_bin}' is not on PATH; install it with "
|
|
76
|
+
"npm install -g @sourcegraph/scip-python"
|
|
77
|
+
)
|
|
78
|
+
output = cwd / "index.scip"
|
|
79
|
+
argv = [
|
|
80
|
+
resolved, "index", str(project),
|
|
81
|
+
"--project-name", project_name,
|
|
82
|
+
"--output", str(output),
|
|
83
|
+
]
|
|
84
|
+
run = runner if runner is not None else subprocess.run
|
|
85
|
+
try:
|
|
86
|
+
completed = run( # nosec B603 — argv is built here from a resolved binary
|
|
87
|
+
argv, cwd=str(cwd), capture_output=True, timeout=timeout_sec, check=False,
|
|
88
|
+
)
|
|
89
|
+
except subprocess.TimeoutExpired as exc:
|
|
90
|
+
stderr = exc.stderr if isinstance(exc.stderr, bytes) else b""
|
|
91
|
+
raise ScipPythonInvocationFailed(
|
|
92
|
+
f"scip-python index timed out after {timeout_sec:.0f}s on {project}", stderr,
|
|
93
|
+
) from exc
|
|
94
|
+
if completed.returncode != 0:
|
|
95
|
+
raise ScipPythonInvocationFailed(
|
|
96
|
+
f"scip-python index exited {completed.returncode} on {project}",
|
|
97
|
+
completed.stderr or b"", returncode=completed.returncode,
|
|
98
|
+
)
|
|
99
|
+
if not output.is_file():
|
|
100
|
+
raise ScipPythonNoOutput(
|
|
101
|
+
f"scip-python index exited 0 but wrote no {output}", completed.stderr or b"",
|
|
102
|
+
)
|
|
103
|
+
return output.read_bytes()
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
"""SCIP → hypergumbo IR for scip-python output (WI-nanom).
|
|
3
|
+
|
|
4
|
+
Three passes of the shared importer, plus two things scip-python's output
|
|
5
|
+
needs that rust-analyzer's did not — measured on hypergumbo-core's own tree
|
|
6
|
+
before this module was written (lab notebook 2026-09-19 12:15):
|
|
7
|
+
|
|
8
|
+
1. **Drop what the syntax arm never models.** scip-python emits a global
|
|
9
|
+
symbol for every PARAMETER (``f().(x)`` — 4,634 on 179 files) and one
|
|
10
|
+
META ``__init__:`` declaration per module. The ``python`` analyzer emits
|
|
11
|
+
neither, so they could pair with nothing and would sit in the artifact
|
|
12
|
+
as thousands of one-sided records. The module is already the file node.
|
|
13
|
+
|
|
14
|
+
2. **Keep the receiver's module on external callees.** The shared edge
|
|
15
|
+
builders drop a reference whose target is not an in-repo Symbol. That is
|
|
16
|
+
right for rust-analyzer, which resolved nothing outside the crate; it
|
|
17
|
+
throws away exactly what pyright is for. Of the syntax arm's 4,133
|
|
18
|
+
method-call stubs whose receiver module it could not determine,
|
|
19
|
+
scip-python resolved 3,973 (96.1%) — 3,840 of them to the stdlib. Those
|
|
20
|
+
become external ``calls`` edges here, on the id shape the syntax arm
|
|
21
|
+
mints when IT knows the module (``python:pathlib.Path:0-0:glob:unresolved``,
|
|
22
|
+
``ExternalRef(module_path="pathlib.Path", name="glob")``, via
|
|
23
|
+
:func:`hypergumbo_core.ir.format_legacy_dst`), so the two arms are
|
|
24
|
+
comparable at every call site and a consumer keying on the module —
|
|
25
|
+
the I/O-boundary catalogue behind ``verify-claims`` — can read it.
|
|
26
|
+
|
|
27
|
+
Only METHOD-descriptor externals (callables) are kept; an external
|
|
28
|
+
``references`` edge (a type annotation naming ``Path``) is noise for a call
|
|
29
|
+
graph and the syntax arm emits nothing to pair it with. ``call_construct``
|
|
30
|
+
is stamped ``method`` when the descriptor has a class parent and
|
|
31
|
+
``function`` otherwise, the same vocabulary the syntax arm uses.
|
|
32
|
+
"""
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
from typing import Any, Callable, Dict, List, Optional, Tuple
|
|
36
|
+
|
|
37
|
+
from hypergumbo_core.ir import Edge, ExternalRef, Symbol, format_legacy_dst
|
|
38
|
+
from hypergumbo_core.scip._generated import scip_pb2
|
|
39
|
+
from hypergumbo_core.scip.calls import scip_index_to_call_edges
|
|
40
|
+
from hypergumbo_core.scip.descriptor import DescriptorKind, parse_scip_symbol
|
|
41
|
+
from hypergumbo_core.scip.edges import scip_index_to_edges
|
|
42
|
+
from hypergumbo_core.scip.index import scip_index_to_symbols
|
|
43
|
+
|
|
44
|
+
#: Symbol kinds the importer mints for descriptors the syntax arm never models.
|
|
45
|
+
DROPPED_KINDS = frozenset({"parameter", "declaration"})
|
|
46
|
+
|
|
47
|
+
#: The generated protobuf module carries no type information; read it as Any
|
|
48
|
+
#: (the same accommodation the Rust translation makes).
|
|
49
|
+
_generated: Any = scip_pb2
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _external_ref(scip_symbol: str) -> Optional[Tuple[ExternalRef, str]]:
|
|
53
|
+
"""``(ExternalRef, call_construct)`` for an external METHOD descriptor, else None.
|
|
54
|
+
|
|
55
|
+
``scip-python python python-stdlib 3.11 pathlib/Path#glob().`` →
|
|
56
|
+
``ExternalRef("python", "pathlib.Path", "glob")``, ``"method"``;
|
|
57
|
+
``scip-python python python-stdlib 3.11 re/compile().`` →
|
|
58
|
+
``ExternalRef("python", "re", "compile")``, ``"function"``.
|
|
59
|
+
"""
|
|
60
|
+
try:
|
|
61
|
+
parsed = parse_scip_symbol(scip_symbol)
|
|
62
|
+
except ValueError:
|
|
63
|
+
return None
|
|
64
|
+
descriptors = parsed.descriptors
|
|
65
|
+
if not descriptors or descriptors[-1].kind is not DescriptorKind.METHOD:
|
|
66
|
+
return None
|
|
67
|
+
module_path = ".".join(d.name for d in descriptors[:-1])
|
|
68
|
+
if not module_path:
|
|
69
|
+
return None
|
|
70
|
+
has_class_parent = any(d.kind is DescriptorKind.TYPE for d in descriptors[:-1])
|
|
71
|
+
return (
|
|
72
|
+
ExternalRef(lang="python", module_path=module_path, name=descriptors[-1].name),
|
|
73
|
+
"method" if has_class_parent else "function",
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def translate_scip_python_to_hg(
|
|
78
|
+
scip_bytes: bytes, *, run_id: str,
|
|
79
|
+
) -> Tuple[List[Symbol], List[Edge]]:
|
|
80
|
+
"""Parse ``scip_bytes`` and return ``(symbols, edges)``.
|
|
81
|
+
|
|
82
|
+
``run_id`` is the analyzer's own run: every Symbol and Edge names it, so
|
|
83
|
+
the artifact's provenance resolves to a serialized run (WI-didag).
|
|
84
|
+
"""
|
|
85
|
+
index = _generated.Index()
|
|
86
|
+
index.ParseFromString(scip_bytes)
|
|
87
|
+
symbols = [s for s in scip_index_to_symbols(index) if s.kind not in DROPPED_KINDS]
|
|
88
|
+
for symbol in symbols:
|
|
89
|
+
symbol.origin_run_id = run_id
|
|
90
|
+
|
|
91
|
+
id_by_scip_symbol: Dict[str, str] = {
|
|
92
|
+
(sym.meta or {}).get("scip_symbol", ""): sym.id
|
|
93
|
+
for sym in symbols
|
|
94
|
+
if sym.meta and sym.meta.get("scip_symbol")
|
|
95
|
+
}
|
|
96
|
+
external: Dict[str, Tuple[ExternalRef, str]] = {}
|
|
97
|
+
|
|
98
|
+
def _resolve(scip_symbol: str) -> Optional[str]:
|
|
99
|
+
# In-repo: the defining Symbol's id. External callable: the syntax
|
|
100
|
+
# arm's module-qualified unresolved id, remembered so the edge can be
|
|
101
|
+
# given its structured dst_ref below. Anything else: skip the edge.
|
|
102
|
+
internal = id_by_scip_symbol.get(scip_symbol)
|
|
103
|
+
if internal is not None:
|
|
104
|
+
return internal
|
|
105
|
+
if scip_symbol in external:
|
|
106
|
+
return format_legacy_dst(external[scip_symbol][0])
|
|
107
|
+
ref = _external_ref(scip_symbol)
|
|
108
|
+
if ref is None:
|
|
109
|
+
return None
|
|
110
|
+
external[scip_symbol] = ref
|
|
111
|
+
return format_legacy_dst(ref[0])
|
|
112
|
+
|
|
113
|
+
edges: List[Edge] = []
|
|
114
|
+
edges.extend(scip_index_to_edges(index, run_id=run_id, resolve_symbol=_resolve))
|
|
115
|
+
edges.extend(scip_index_to_call_edges(index, run_id=run_id, resolve_symbol=_resolve))
|
|
116
|
+
by_dst = {format_legacy_dst(ref): (ref, construct) for ref, construct in external.values()}
|
|
117
|
+
kept: List[Edge] = []
|
|
118
|
+
for edge in edges:
|
|
119
|
+
hit = by_dst.get(edge.dst)
|
|
120
|
+
if hit is None:
|
|
121
|
+
kept.append(edge)
|
|
122
|
+
continue
|
|
123
|
+
if edge.edge_type != "calls":
|
|
124
|
+
continue # an external non-call reference: nothing to pair, nothing to type
|
|
125
|
+
ref, construct = hit
|
|
126
|
+
edge.dst_ref = ref
|
|
127
|
+
edge.is_resolved = False
|
|
128
|
+
edge.meta = {**(edge.meta or {}), "call_construct": construct}
|
|
129
|
+
kept.append(edge)
|
|
130
|
+
return symbols, kept
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
SourceReader = Callable[[str], Optional[bytes]]
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: hypergumbo-lang-scip-python
|
|
3
|
+
Version: 8.1.0
|
|
4
|
+
Summary: SCIP-backed Python analyzer for hypergumbo (scip-python / pyright integration)
|
|
5
|
+
Author: Hypergumbo contributors
|
|
6
|
+
License: AGPL-3.0-or-later
|
|
7
|
+
Keywords: code-graph,pyright,python,scip,static-analysis
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Requires-Dist: hypergumbo-core==8.1.0
|
|
14
|
+
Requires-Dist: hypergumbo-lang-mainstream==8.1.0
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
<!-- SPDX-License-Identifier: AGPL-3.0-or-later -->
|
|
18
|
+
# hypergumbo-lang-scip-python
|
|
19
|
+
|
|
20
|
+
SCIP-backed Python analyzer for hypergumbo, driven by
|
|
21
|
+
[scip-python](https://github.com/sourcegraph/scip-python) (pyright).
|
|
22
|
+
|
|
23
|
+
This optional package is the second opt-in fidelity backend (after
|
|
24
|
+
`hypergumbo-lang-rust-analyzer`) and the first that **executes nothing** from
|
|
25
|
+
the analysed repository: pyright performs static inference only. It runs
|
|
26
|
+
*beside* the ast-based `python` analyzer, never instead of it — the two
|
|
27
|
+
producers' records for one declaration are folded into one node by the
|
|
28
|
+
ADR-0057 merge pass, and every disagreement stays visible in the provenance
|
|
29
|
+
slot.
|
|
30
|
+
|
|
31
|
+
## What it adds, measured
|
|
32
|
+
|
|
33
|
+
On hypergumbo-core's own 179 source files (2026-09-19, scip-python 0.6.6):
|
|
34
|
+
the syntax arm left 4,133 method-call receivers untyped (`obj.method()` with
|
|
35
|
+
`obj` of unknown module); scip-python typed **3,973 of them (96.1%)** — 3,840
|
|
36
|
+
to the stdlib (`builtins.str`, `dict`, `list`, `pathlib.Path`, `re.Pattern`, …),
|
|
37
|
+
96 in-repo, 37 third-party. That module is what `hypergumbo verify-claims`'s
|
|
38
|
+
I/O-boundary and taint verdicts key on. The Rust backend measured zero
|
|
39
|
+
receivers typed.
|
|
40
|
+
|
|
41
|
+
## Enabling it
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
npm install -g @sourcegraph/scip-python@0.6.6 # the producer (node + npm)
|
|
45
|
+
pipx install 'hypergumbo[scip-python]' # this wrapper
|
|
46
|
+
hypergumbo survey . --backend scip-python # one run
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Durably, as a preference in either ADR-0045 config tier (allowed because the
|
|
50
|
+
backend executes nothing):
|
|
51
|
+
|
|
52
|
+
```toml
|
|
53
|
+
# $XDG_CONFIG_HOME/hypergumbo/config.toml or <repo>/.hypergumbo.toml
|
|
54
|
+
[backends]
|
|
55
|
+
scip_python = true
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
`HYPERGUMBO_SCIP_PYTHON=1` is the environment spelling; `--backend tree-sitter`
|
|
59
|
+
turns every opt-in backend off for one run. Precedence: flag > environment >
|
|
60
|
+
project config > user config.
|
|
61
|
+
|
|
62
|
+
When enabled but the binary or this package is missing, the CLI exits 2 with
|
|
63
|
+
the install line rather than silently falling through.
|
|
64
|
+
|
|
65
|
+
## What the translation does with scip-python's output
|
|
66
|
+
|
|
67
|
+
- Global definitions become `Symbol`s through the shared importer
|
|
68
|
+
(`hypergumbo_core.scip`); scip-python declares no `Document.language` and
|
|
69
|
+
no `SymbolInformation.kind`, so the importer reads the file extension and
|
|
70
|
+
the SCIP descriptor grammar instead.
|
|
71
|
+
- Parameter symbols and the per-module `__init__:` declaration are dropped:
|
|
72
|
+
the syntax arm models neither, and a node nothing can pair with is noise.
|
|
73
|
+
- References to in-repo definitions become `calls` / `references` edges.
|
|
74
|
+
- References to **external** callables (stdlib, third-party) become external
|
|
75
|
+
call edges carrying the receiver's module — `python:pathlib.Path:0-0:glob:unresolved`
|
|
76
|
+
with a structured `dst_ref` — the same shape the syntax arm mints when it
|
|
77
|
+
knows the module, so the two arms are comparable at every call site.
|
|
78
|
+
|
|
79
|
+
## Recorded fixture
|
|
80
|
+
|
|
81
|
+
`recorded/sample_project/` is a three-file package and the `index.scip`
|
|
82
|
+
scip-python 0.6.6 emitted for it (`metadata.project_root` normalised). Every
|
|
83
|
+
cross-backend test reads that recording, never anything derived from the
|
|
84
|
+
syntax arm's own output (ADR-0057 §12).
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
hypergumbo_lang_scip_python/__init__.py,sha256=VaRhpUF0uzQpMlS5OFe0hmBRMBDyRc3X_ebMTt25aeU,1576
|
|
2
|
+
hypergumbo_lang_scip_python/analyzer.py,sha256=LYFy7KDXV4hJw7wAKEKLhUlQ9JhoXzvZDiolxC9iwi8,5880
|
|
3
|
+
hypergumbo_lang_scip_python/gate.py,sha256=8pA-JjXC4FXZEgKvof5qCbCKbPUZVJG0UzMRToljTNs,2261
|
|
4
|
+
hypergumbo_lang_scip_python/invoke.py,sha256=Ea1rARro7FxGJBCTkFGi_WAEPJXKU06rhE7sJFw0b9E,4123
|
|
5
|
+
hypergumbo_lang_scip_python/translate.py,sha256=uazccEipaUm85wGNS8QmYzmrE4BGASO3vwDMPJ0d2xE,6027
|
|
6
|
+
hypergumbo_lang_scip_python-8.1.0.dist-info/METADATA,sha256=q3cWXOjflZCseV6Z7zDK8EWTNMl_-UvDLWeqTLNB_QY,3738
|
|
7
|
+
hypergumbo_lang_scip_python-8.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
8
|
+
hypergumbo_lang_scip_python-8.1.0.dist-info/entry_points.txt,sha256=65bmWVDWBvapMOw-mdY8veEYv0HkhoHZ0ILEJ5lwRb0,82
|
|
9
|
+
hypergumbo_lang_scip_python-8.1.0.dist-info/RECORD,,
|