codeanalyzer-python 1.0.1__tar.gz → 1.0.3__tar.gz

{codeanalyzer_python-1.0.1 → codeanalyzer_python-1.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codeanalyzer-python
-Version: 1.0.1
+Version: 1.0.3
 Summary: Static analysis for Python — canonical schema v2 (symbol table, call graph, and native CFG/PDG/SDG dataflow) as analysis.json or a Neo4j property graph.
 Author-email: Rahul Krishna <i.m.ralk@gmail.com>
 License-File: LICENSE
@@ -18,6 +18,7 @@ Requires-Dist: numpy<2.0.0,>=1.26.0; python_version >= '3.12'
 Requires-Dist: packaging>=25.0
 Requires-Dist: pandas<2.0.0,>=1.3.0; python_version < '3.11'
 Requires-Dist: pandas<3.0.0,>=2.0.0; python_version >= '3.11'
+Requires-Dist: parso>=0.8.5
 Requires-Dist: pycg>=0.0.6
 Requires-Dist: pydantic<2.0.0,>=1.8.0; python_version < '3.11'
 Requires-Dist: pydantic<3.0.0,>=2.0.0; python_version >= '3.11'

{codeanalyzer_python-1.0.1 → codeanalyzer_python-1.0.3}/codeanalyzer/core.py

@@ -163,8 +163,159 @@ class Codeanalyzer:
             stderr=None,
         )
 
+    @classmethod
+    def _get_base_interpreter(cls) -> Path:
+        """The interpreter used to provision the analysis virtualenv.
+
+        jedi parses the *analysis environment's* Python version with parso,
+        which ships one hardcoded grammar file per minor version — an
+        environment newer than the newest shipped grammar makes every file
+        fail with "Python version X.Y is currently not supported" while the
+        run still exits 0 (#107). So the default choice is gated on the
+        installed parso's ceiling: a too-new default is swapped for the
+        newest supported interpreter found on the host, falling back to the
+        default (loudly) only when none exists. An explicit ``SYSTEM_PYTHON``
+        always wins, with a warning when parso cannot parse its version.
+        """
+        # An explicit SYSTEM_PYTHON override wins (consulted only when running
+        # inside a virtualenv, matching the historical behavior).
+        if sys.prefix != sys.base_prefix:
+            system_python = os.getenv("SYSTEM_PYTHON")
+            if system_python:
+                system_python_path = Path(system_python)
+                if system_python_path.exists() and system_python_path.is_file():
+                    ceiling = cls._parso_supported_ceiling()
+                    version = cls._interpreter_version(system_python_path)
+                    if ceiling is not None and version is not None and version > ceiling:
+                        logger.warning(
+                            f"SYSTEM_PYTHON={system_python} is Python "
+                            f"{version[0]}.{version[1]}, newer than the newest grammar "
+                            f"the installed parso ships ({ceiling[0]}.{ceiling[1]}). "
+                            "jedi will likely reject every file in the analysis "
+                            "environment (#107); honoring the explicit override anyway."
+                        )
+                    return system_python_path
+
+        candidate = cls._default_base_interpreter()
+        ceiling = cls._parso_supported_ceiling()
+        if ceiling is None:
+            return candidate
+        version = cls._interpreter_version(candidate)
+        if version is None or version <= ceiling:
+            return candidate
+        logger.warning(
+            f"Default interpreter {candidate} is Python {version[0]}.{version[1]}, "
+            f"newer than the newest grammar the installed parso ships "
+            f"({ceiling[0]}.{ceiling[1]}) — looking for a supported interpreter "
+            "for the analysis environment (#107)."
+        )
+        supported = cls._find_supported_interpreter(ceiling)
+        if supported is not None:
+            logger.info(f"Provisioning the analysis environment with {supported}.")
+            return supported
+        logger.warning(
+            f"No interpreter <= {ceiling[0]}.{ceiling[1]} found on this host; "
+            f"falling back to {candidate}. jedi/parso will likely reject every "
+            "file — install a supported Python or upgrade parso."
+        )
+        return candidate
+
+    @staticmethod
+    def _versions_from_grammar_stems(stems: List[str]) -> List[tuple]:
+        """``grammar313`` → ``(3, 13)``, sorted ascending; malformed stems dropped."""
+        versions = []
+        for stem in stems:
+            digits = stem[len("grammar"):]
+            if len(digits) >= 2 and digits.isdigit():
+                versions.append((int(digits[0]), int(digits[1:])))
+        return sorted(versions)
+
+    @classmethod
+    def _parso_supported_ceiling(cls) -> Optional[tuple]:
+        """Newest ``(major, minor)`` the installed parso ships a grammar for,
+        derived from its ``python/grammar*.txt`` files so the ceiling moves
+        automatically when parso adds a version. ``None`` if undeterminable."""
+        try:
+            import parso
+
+            stems = [
+                p.stem
+                for p in (Path(parso.__file__).parent / "python").glob("grammar*.txt")
+            ]
+            versions = cls._versions_from_grammar_stems(stems)
+            return versions[-1] if versions else None
+        except Exception:
+            return None
+
+    @staticmethod
+    def _interpreter_version(interpreter: Path) -> Optional[tuple]:
+        """``(major, minor)`` of an interpreter, or ``None`` if it can't run."""
+        try:
+            result = subprocess.run(
+                [
+                    str(interpreter),
+                    "-c",
+                    "import sys; print('%d.%d' % sys.version_info[:2])",
+                ],
+                capture_output=True,
+                text=True,
+                timeout=5,
+            )
+            if result.returncode == 0:
+                major, minor = result.stdout.strip().split(".")
+                return (int(major), int(minor))
+        except (subprocess.TimeoutExpired, FileNotFoundError, PermissionError, ValueError):
+            pass
+        return None
+
+    @staticmethod
+    def _pick_supported_interpreter(
+        candidates: List[tuple], ceiling: tuple
+    ) -> Optional[Path]:
+        """Newest candidate whose version is within the ceiling.
+
+        ``candidates`` is ``[(path, (major, minor) | None), ...]``."""
+        supported = [
+            (version, path)
+            for path, version in candidates
+            if version is not None and version <= ceiling
+        ]
+        return max(supported)[1] if supported else None
+
+    @classmethod
+    def _find_supported_interpreter(cls, ceiling: tuple) -> Optional[Path]:
+        """Search the host for the newest interpreter within the parso ceiling:
+        versioned names on PATH (``python3.13``, ``python3.12``, ...) first,
+        then pyenv installs."""
+        paths: List[Path] = []
+        for minor in range(ceiling[1], 7, -1):
+            which = shutil.which(f"python{ceiling[0]}.{minor}")
+            # Skip the current virtualenv's own interpreter (same rule as
+            # _default_base_interpreter): the analysis env must come from a
+            # base installation.
+            if which and not which.startswith(sys.prefix):
+                paths.append(Path(which))
+        for pyenv_root in (os.getenv("PYENV_ROOT"), str(Path.home() / ".pyenv")):
+            if not pyenv_root:
+                continue
+            versions_dir = Path(pyenv_root) / "versions"
+            if versions_dir.is_dir():
+                for install in sorted(versions_dir.iterdir(), reverse=True):
+                    exe = install / "bin" / "python3"
+                    if exe.exists():
+                        paths.append(exe)
+        seen = set()
+        candidates = []
+        for path in paths:
+            key = str(path)
+            if key in seen:
+                continue
+            seen.add(key)
+            candidates.append((path, cls._interpreter_version(path)))
+        return cls._pick_supported_interpreter(candidates, ceiling)
+
     @staticmethod
-    def _get_base_interpreter() -> Path:
+    def _default_base_interpreter() -> Path:
         """Get the base Python interpreter path.
 
         This method finds a suitable base Python interpreter that can be used
@@ -183,13 +334,6 @@ class Codeanalyzer:
 
         # We're inside a virtual environment; need to find the base interpreter
 
-        # First, check if user explicitly set SYSTEM_PYTHON
-        system_python = os.getenv("SYSTEM_PYTHON")
-        if system_python:
-            system_python_path = Path(system_python)
-            if system_python_path.exists() and system_python_path.is_file():
-                return system_python_path
-
         # Try to get the base interpreter from sys.base_executable (Python 3.3+)
         if hasattr(sys, "base_executable") and sys.base_executable:
             base_exec = Path(sys.base_executable)
@@ -778,6 +922,15 @@ class Codeanalyzer:
             if files_from_cache > 0:
                 logger.info(f"Reused {files_from_cache} files from cache, processed {files_processed} new/changed files")
 
+        if py_files and not symbol_table:
+            logger.error(
+                "Every one of the %d discovered Python files failed to process — "
+                "the symbol table is empty. This usually means the analysis "
+                "environment's interpreter is newer than the installed jedi/parso "
+                "stack supports (#107); check the per-file errors above.",
+                len(py_files),
+            )
+
         logger.info(
             "✅ Symbol table: %d modules in %.1fs",
             len(symbol_table), time.perf_counter() - t0_st,

{codeanalyzer_python-1.0.1 → codeanalyzer_python-1.0.3}/codeanalyzer/neo4j/project.py

@@ -290,9 +290,11 @@ def _project_module_body(
     externals: dict, sig_to_id: dict, module_id_by_key: dict,
 ) -> None:
     for fn in (mod.functions or {}).values():
-        _project_callable(b, file_key, mod_ref, "PY_DECLARES", fn, externals, sig_to_id)
+        _project_callable(b, file_key, mod_ref, "PY_DECLARES", fn, externals, sig_to_id,
+                          mod.source)
     for cl in (mod.types or {}).values():
-        _project_class(b, file_key, mod_ref, "PY_DECLARES", cl, externals, sig_to_id)
+        _project_class(b, file_key, mod_ref, "PY_DECLARES", cl, externals, sig_to_id,
+                       mod.source)
     for v in mod.variables or []:
         _project_variable(b, file_key, mod_ref, file_key, v)
     _project_imports(b, mod_ref, mod, module_id_by_key)
@@ -360,10 +362,10 @@ def _project_imports(b: RowBuilder, mod_ref: NodeRef, mod: PyModule,
 
 def _project_class(
     b: RowBuilder, file_key: str, parent: NodeRef, parent_rel: str, cl: PyClass,
-    externals: dict, sig_to_id: dict,
+    externals: dict, sig_to_id: dict, source: str,
 ) -> None:
     ref = b.node(
-        ["PySymbol", "PyClass"], "id", cl.id, _class_props(cl, file_key)
+        ["PySymbol", "PyClass"], "id", cl.id, _class_props(cl, file_key, source)
     )
     b.edge(parent_rel, parent, ref)
 
@@ -372,22 +374,23 @@ def _project_class(
             b.edge_to_symbol("PY_EXTENDS", ref, _symbol_ref(base, externals, sig_to_id))
 
     for m in (cl.callables or {}).values():
-        _project_callable(b, file_key, ref, "PY_HAS_METHOD", m, externals, sig_to_id)
+        _project_callable(b, file_key, ref, "PY_HAS_METHOD", m, externals, sig_to_id,
+                          source)
     for a in (cl.attributes or {}).values():
         _project_attribute(b, file_key, ref, cl.signature, a)
     for ic in (cl.types or {}).values():
-        _project_class(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id)
+        _project_class(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id, source)
 
 
 def _project_callable(
     b: RowBuilder, file_key: str, owner: NodeRef, owner_rel: str, c: PyCallable,
-    externals: dict, sig_to_id: dict,
+    externals: dict, sig_to_id: dict, source: str,
 ) -> None:
     ref = b.node(
         ["PySymbol", "PyCallable"],
         "id",
         c.id,
-        _callable_props(c, file_key),
+        _callable_props(c, file_key, source),
     )
     b.edge(owner_rel, owner, ref)
 
@@ -410,9 +413,10 @@ def _project_callable(
     for v in c.local_variables or []:
         _project_variable(b, file_key, ref, c.signature, v)
     for ic in (c.callables or {}).values():
-        _project_callable(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id)
+        _project_callable(b, file_key, ref, "PY_DECLARES", ic, externals, sig_to_id,
+                          source)
     for cl in (c.types or {}).values():
-        _project_class(b, file_key, ref, "PY_DECLARES", cl, externals, sig_to_id)
+        _project_class(b, file_key, ref, "PY_DECLARES", cl, externals, sig_to_id, source)
 
 
 def _project_attribute(
@@ -459,13 +463,24 @@ def _module_props(mod: PyModule, file_key: str) -> Props:
     )
 
 
-def _class_props(cl: PyClass, file_key: str) -> Props:
+def _span_code(source: str, span) -> str | None:
+    """A declaration's text: the owning module's ``source`` sliced by the node's
+    utf-8 byte span. Schema v2 stores source once per module, so the graph's
+    ``code`` property (declared on :PyClass/:PyCallable and indexed by
+    ``py_code_fts``) is derived here at projection time (#104)."""
+    if span is None or not source:
+        return None
+    lo, hi = span.bytes
+    return source.encode("utf-8")[lo:hi].decode("utf-8")
+
+
+def _class_props(cl: PyClass, file_key: str, source: str) -> Props:
     return prune(
         {
             "id": cl.id,
             "signature": cl.signature,
             "name": cl.name,
-            "code": getattr(cl, "code", None),
+            "code": _span_code(source, cl.span),
             "base_classes": list(cl.base_classes or []),
             "docstring": _docstring_of(cl.comments),
             "start_line": cl.start_line,
@@ -475,7 +490,7 @@ def _class_props(cl: PyClass, file_key: str) -> Props:
     )
 
 
-def _callable_props(c: PyCallable, file_key: str) -> Props:
+def _callable_props(c: PyCallable, file_key: str, source: str) -> Props:
     return prune(
         {
             "id": c.id,
@@ -484,7 +499,7 @@ def _callable_props(c: PyCallable, file_key: str) -> Props:
             "path": c.path,
             "return_type": c.return_type,
             "cyclomatic_complexity": c.cyclomatic_complexity,
-            "code": getattr(c, "code", None),
+            "code": _span_code(source, c.span),
             "code_start_line": c.code_start_line,
             "start_line": c.start_line,
             "end_line": c.end_line,

{codeanalyzer_python-1.0.1 → codeanalyzer_python-1.0.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "codeanalyzer-python"
-version = "1.0.1"
+version = "1.0.3"
 description = "Static analysis for Python — canonical schema v2 (symbol table, call graph, and native CFG/PDG/SDG dataflow) as analysis.json or a Neo4j property graph."
 readme = "README.md"
 authors = [
@@ -12,6 +12,9 @@ dependencies = [
     # jedi
     "jedi>=0.18.0,<0.20.0; python_version < '3.11'",
     "jedi<=0.19.2; python_version >= '3.11'",
+    # parso 0.8.5 is the first release shipping the Python 3.14 grammar; older
+    # resolutions make jedi reject every file in a 3.14 analysis env (#107)
+    "parso>=0.8.5",
     # msgpack
     "msgpack>=1.0.0,<1.0.7; python_version < '3.11'",
     "msgpack>=1.0.7,<2.0.0; python_version >= '3.11'",