codeanalyzer-python 0.3.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/.gitignore +3 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/PKG-INFO +4 -1
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/README.md +3 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/__main__.py +26 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/core.py +34 -3
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/neo4j/__init__.py +1 -1
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/neo4j/emit.py +2 -2
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/neo4j/project.py +84 -18
- codeanalyzer_python-0.3.0/codeanalyzer/neo4j/catalog.py → codeanalyzer_python-0.3.1/codeanalyzer/neo4j/schema.py +58 -18
- codeanalyzer_python-0.3.1/codeanalyzer/provenance.py +61 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/schema/py_schema.py +38 -0
- codeanalyzer_python-0.3.1/codeanalyzer/syntactic_analysis/import_resolver.py +67 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/syntactic_analysis/symbol_table_builder.py +74 -10
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/pyproject.toml +1 -1
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/schema.neo4j.json +19 -10
- codeanalyzer_python-0.3.0/codeanalyzer/neo4j/schema.py +0 -39
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/LICENSE +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/NOTICE +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/__init__.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/config/__init__.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/config/config.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/jedi/__init__.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/jedi/jedi.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/neo4j/bolt.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/neo4j/cypher.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/neo4j/rows.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/options/__init__.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/options/options.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/py.typed +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/schema/__init__.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/semantic_analysis/__init__.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/semantic_analysis/call_graph.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/semantic_analysis/pycg/__init__.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/semantic_analysis/pycg/pycg_analysis.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/semantic_analysis/pycg/pycg_exceptions.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/semantic_analysis/pycg/shard_planner.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/syntactic_analysis/__init__.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/syntactic_analysis/exceptions.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/utils/__init__.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/utils/logging.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/utils/progress_bar.py +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/schema-uml.drawio +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/test/fixtures/whole_applications/flask/README.md +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/test/fixtures/whole_applications/flask/examples/celery/README.md +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/test/fixtures/whole_applications/flask/requirements-skip/README.md +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/test/fixtures/whole_applications/flask/src/flask/sansio/README.md +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/test/fixtures/whole_applications/requests/LICENSE +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/test/fixtures/whole_applications/requests/NOTICE +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/test/fixtures/whole_applications/requests/README.md +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/test/fixtures/whole_applications/requests/docs/_themes/LICENSE +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/test/fixtures/whole_applications/requests/ext/LICENSE +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/test/fixtures/whole_applications/xarray/LICENSE +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/test/fixtures/whole_applications/xarray/README.md +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/test/fixtures/whole_applications/xarray/properties/README.md +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/test/fixtures/whole_applications/xarray/xarray/datatree_/LICENSE +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/test/fixtures/whole_applications/xarray/xarray/datatree_/README.md +0 -0
- {codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/test/fixtures/whole_applications/xarray/xarray/datatree_/docs/README.md +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: codeanalyzer-python
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Static Analysis on Python source code using Jedi, CodeQL and Treesitter — emits analysis.json or a Neo4j property graph.
|
|
5
5
|
Author-email: Rahul Krishna <i.m.ralk@gmail.com>
|
|
6
6
|
License-File: LICENSE
|
|
@@ -181,6 +181,9 @@ $ canpy --help
|
|
|
181
181
|
Static Analysis on Python source code using Jedi, PyCG and Tree sitter.
|
|
182
182
|
|
|
183
183
|
╭─ Options ────────────────────────────────────────────────────────────────────╮
|
|
184
|
+
│ --version Show the canpy │
|
|
185
|
+
│ version and │
|
|
186
|
+
│ exit. │
|
|
184
187
|
│ --input -i PATH Path to the │
|
|
185
188
|
│ project root │
|
|
186
189
|
│ directory (not │
|
|
@@ -144,6 +144,9 @@ $ canpy --help
|
|
|
144
144
|
Static Analysis on Python source code using Jedi, PyCG and Tree sitter.
|
|
145
145
|
|
|
146
146
|
╭─ Options ────────────────────────────────────────────────────────────────────╮
|
|
147
|
+
│ --version Show the canpy │
|
|
148
|
+
│ version and │
|
|
149
|
+
│ exit. │
|
|
147
150
|
│ --input -i PATH Path to the │
|
|
148
151
|
│ project root │
|
|
149
152
|
│ directory (not │
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from importlib.metadata import version as _pkg_version, PackageNotFoundError
|
|
1
2
|
from pathlib import Path
|
|
2
3
|
from typing import Optional, Annotated
|
|
3
4
|
|
|
@@ -10,7 +11,32 @@ from codeanalyzer.schema import model_dump_json
|
|
|
10
11
|
from codeanalyzer.options import AnalysisOptions, EmitTarget, ShardStrategy
|
|
11
12
|
|
|
12
13
|
|
|
14
|
+
def _version_callback(value: bool) -> None:
|
|
15
|
+
"""Print the installed ``codeanalyzer-python`` version and exit.
|
|
16
|
+
|
|
17
|
+
Eager so it fires before the rest of the CLI callback (no -i/--input
|
|
18
|
+
required). Reads the version from package metadata so it always reflects
|
|
19
|
+
what is actually installed rather than a hardcoded string."""
|
|
20
|
+
if not value:
|
|
21
|
+
return
|
|
22
|
+
try:
|
|
23
|
+
installed = _pkg_version("codeanalyzer-python")
|
|
24
|
+
except PackageNotFoundError:
|
|
25
|
+
installed = "unknown"
|
|
26
|
+
typer.echo(f"canpy {installed}")
|
|
27
|
+
raise typer.Exit()
|
|
28
|
+
|
|
29
|
+
|
|
13
30
|
def main(
|
|
31
|
+
version: Annotated[
|
|
32
|
+
Optional[bool],
|
|
33
|
+
typer.Option(
|
|
34
|
+
"--version",
|
|
35
|
+
help="Show the canpy version and exit.",
|
|
36
|
+
callback=_version_callback,
|
|
37
|
+
is_eager=True,
|
|
38
|
+
),
|
|
39
|
+
] = None,
|
|
14
40
|
input: Annotated[
|
|
15
41
|
Optional[Path],
|
|
16
42
|
typer.Option(
|
|
@@ -26,9 +26,11 @@ from codeanalyzer.semantic_analysis.call_graph import (
|
|
|
26
26
|
)
|
|
27
27
|
from codeanalyzer.semantic_analysis.pycg import PyCG, PyCGExceptions
|
|
28
28
|
from codeanalyzer.syntactic_analysis.exceptions import SymbolTableBuilderRayError
|
|
29
|
+
from codeanalyzer.syntactic_analysis.import_resolver import resolve_imports
|
|
29
30
|
from codeanalyzer.syntactic_analysis.symbol_table_builder import SymbolTableBuilder
|
|
30
31
|
from codeanalyzer.utils import ProgressBar
|
|
31
32
|
from codeanalyzer.options import AnalysisOptions
|
|
33
|
+
from codeanalyzer.provenance import analyzer_info, repository_info
|
|
32
34
|
|
|
33
35
|
@ray.remote
|
|
34
36
|
def _process_file_with_ray(py_file: Union[Path, str], project_dir: Union[Path, str], virtualenv: Union[Path, str, None]) -> Dict[str, PyModule]:
|
|
@@ -411,8 +413,8 @@ class Codeanalyzer:
|
|
|
411
413
|
Uses caching to avoid re-analyzing unchanged files.
|
|
412
414
|
"""
|
|
413
415
|
cache_file = self.cache_dir / "analysis_cache.json"
|
|
414
|
-
|
|
415
|
-
# Try to load existing cached analysis
|
|
416
|
+
|
|
417
|
+
# Try to load existing cached analysis
|
|
416
418
|
cached_pyapplication = None
|
|
417
419
|
if not self.rebuild_analysis and cache_file.exists():
|
|
418
420
|
try:
|
|
@@ -422,6 +424,12 @@ class Codeanalyzer:
|
|
|
422
424
|
logger.warning(f"Failed to load cache: {e}. Rebuilding analysis.")
|
|
423
425
|
cached_pyapplication = None
|
|
424
426
|
|
|
427
|
+
if cached_pyapplication is not None and not self._cache_analyzer_matches(
|
|
428
|
+
cached_pyapplication, analyzer_info(self.analysis_level).version
|
|
429
|
+
):
|
|
430
|
+
logger.info("Analysis cache written by a different analyzer version; rebuilding.")
|
|
431
|
+
cached_pyapplication = None
|
|
432
|
+
|
|
425
433
|
# Build symbol table from cached application if available (if no available, the build a new one)
|
|
426
434
|
symbol_table = self._build_symbol_table(cached_pyapplication.symbol_table if cached_pyapplication else {})
|
|
427
435
|
|
|
@@ -454,12 +462,35 @@ class Codeanalyzer:
|
|
|
454
462
|
.external_symbols(external_symbols)
|
|
455
463
|
.build()
|
|
456
464
|
)
|
|
457
|
-
|
|
465
|
+
|
|
466
|
+
# Every run re-resolves import spellings against the analyzed module
|
|
467
|
+
# set -- pure and cheap; cached modules from older caches default to
|
|
468
|
+
# resolved_module=None and get stamped here (issue #82).
|
|
469
|
+
resolve_imports(app, self.project_dir)
|
|
470
|
+
|
|
471
|
+
# Single choke point for provenance: every produced app (fresh symbol
|
|
472
|
+
# table or reused-from-cache) passes through here before being cached
|
|
473
|
+
# or returned, so analyzer/repository always reflect *this* run/checkout
|
|
474
|
+
# even when the symbol table itself came from the on-disk cache.
|
|
475
|
+
app.analyzer = analyzer_info(self.analysis_level)
|
|
476
|
+
app.repository = repository_info(self.project_dir)
|
|
477
|
+
|
|
458
478
|
# Save to cache
|
|
459
479
|
self._save_analysis_cache(app, cache_file)
|
|
460
480
|
|
|
461
481
|
return app
|
|
462
482
|
|
|
483
|
+
@staticmethod
|
|
484
|
+
def _cache_analyzer_matches(cached_app: Optional[PyApplication], current_version: str) -> bool:
|
|
485
|
+
"""A cache written by another analyzer version (or before versions were
|
|
486
|
+
recorded) may lack fields the current models populate — pydantic fills
|
|
487
|
+
silent defaults, which would masquerade as analyzed absence."""
|
|
488
|
+
return (
|
|
489
|
+
cached_app is not None
|
|
490
|
+
and cached_app.analyzer is not None
|
|
491
|
+
and cached_app.analyzer.version == current_version
|
|
492
|
+
)
|
|
493
|
+
|
|
463
494
|
def _load_pyapplication_from_cache(self, cache_file: Path) -> PyApplication:
|
|
464
495
|
"""Load cached analysis from file.
|
|
465
496
|
|
|
@@ -19,7 +19,7 @@ plus the two writers (cypher snapshot / bolt incremental). Nothing here runs
|
|
|
19
19
|
unless ``--emit neo4j`` (or ``--emit schema``) is selected.
|
|
20
20
|
"""
|
|
21
21
|
from codeanalyzer.neo4j.bolt import BoltConfig, bolt_writer
|
|
22
|
-
from codeanalyzer.neo4j.
|
|
22
|
+
from codeanalyzer.neo4j.schema import (
|
|
23
23
|
MARKER_LABELS,
|
|
24
24
|
NODE_LABELS,
|
|
25
25
|
REL_TYPES,
|
|
@@ -28,7 +28,7 @@ from pathlib import Path
|
|
|
28
28
|
from typing import Optional
|
|
29
29
|
|
|
30
30
|
from codeanalyzer.neo4j.bolt import BoltConfig, bolt_writer
|
|
31
|
-
from codeanalyzer.neo4j.
|
|
31
|
+
from codeanalyzer.neo4j.schema import build_schema_document
|
|
32
32
|
from codeanalyzer.neo4j.cypher import render_cypher
|
|
33
33
|
from codeanalyzer.neo4j.project import project
|
|
34
34
|
from codeanalyzer.options import AnalysisOptions
|
|
@@ -38,7 +38,7 @@ from codeanalyzer.utils import logger
|
|
|
38
38
|
|
|
39
39
|
def emit_schema(output: Optional[Path]) -> None:
|
|
40
40
|
"""Emit the Neo4j schema contract (``schema.json``) — a static artifact derived
|
|
41
|
-
from the in-repo
|
|
41
|
+
from the in-repo schema, independent of any analyzed project. With no
|
|
42
42
|
``output`` it prints to stdout."""
|
|
43
43
|
doc = json.dumps(build_schema_document(), indent=2) + "\n"
|
|
44
44
|
if output is None:
|
|
@@ -36,7 +36,7 @@ import json
|
|
|
36
36
|
from pathlib import Path
|
|
37
37
|
from typing import Any, List, Optional
|
|
38
38
|
|
|
39
|
-
from codeanalyzer.neo4j.
|
|
39
|
+
from codeanalyzer.neo4j.schema import SCHEMA_VERSION
|
|
40
40
|
from codeanalyzer.neo4j.rows import GraphRows, NodeRef, Props, RowBuilder, prune
|
|
41
41
|
from codeanalyzer.schema import (
|
|
42
42
|
PyApplication,
|
|
@@ -53,10 +53,26 @@ from codeanalyzer.schema.py_schema import PyCallsite
|
|
|
53
53
|
def project(app: PyApplication, app_name: str) -> GraphRows:
|
|
54
54
|
b = RowBuilder()
|
|
55
55
|
|
|
56
|
-
app_ref = b.node(
|
|
56
|
+
app_ref = b.node(
|
|
57
|
+
["PyApplication"],
|
|
58
|
+
"name",
|
|
59
|
+
app_name,
|
|
60
|
+
prune(
|
|
61
|
+
{
|
|
62
|
+
"schema_version": SCHEMA_VERSION,
|
|
63
|
+
"analyzer_name": app.analyzer.name if app.analyzer else None,
|
|
64
|
+
"analyzer_version": app.analyzer.version if app.analyzer else None,
|
|
65
|
+
"repo_uri": app.repository.uri if app.repository else None,
|
|
66
|
+
"source_revision": app.repository.revision if app.repository else None,
|
|
67
|
+
"repo_dirty": app.repository.dirty if app.repository else None,
|
|
68
|
+
}
|
|
69
|
+
),
|
|
70
|
+
)
|
|
57
71
|
|
|
58
72
|
for file_key, mod in app.symbol_table.items():
|
|
59
|
-
mod_ref = b.node(
|
|
73
|
+
mod_ref = b.node(
|
|
74
|
+
["PyModule"], "file_key", file_key, _module_props(mod, file_key)
|
|
75
|
+
)
|
|
60
76
|
b.edge("PY_HAS_MODULE", app_ref, mod_ref)
|
|
61
77
|
_project_module_body(b, file_key, mod_ref, mod)
|
|
62
78
|
|
|
@@ -66,7 +82,9 @@ def project(app: PyApplication, app_name: str) -> GraphRows:
|
|
|
66
82
|
for e in app.call_graph:
|
|
67
83
|
src = _call_endpoint(b, e.source, externals)
|
|
68
84
|
tgt = _call_endpoint(b, e.target, externals)
|
|
69
|
-
b.edge(
|
|
85
|
+
b.edge(
|
|
86
|
+
"PY_CALLS", src, tgt, _call_edge_props(e.weight, list(e.provenance or []))
|
|
87
|
+
)
|
|
70
88
|
|
|
71
89
|
return b.finish()
|
|
72
90
|
|
|
@@ -86,9 +104,18 @@ def _call_endpoint(b: RowBuilder, signature: str, externals: dict) -> NodeRef:
|
|
|
86
104
|
ext = externals.get(signature)
|
|
87
105
|
if ext is None and b.has_key("PySymbol", signature):
|
|
88
106
|
return _sym(signature)
|
|
89
|
-
name =
|
|
107
|
+
name = (
|
|
108
|
+
ext.name
|
|
109
|
+
if ext is not None
|
|
110
|
+
else (signature.rsplit(".", 1)[-1] if "." in signature else signature)
|
|
111
|
+
)
|
|
90
112
|
module = ext.module if ext is not None else None
|
|
91
|
-
return b.node(
|
|
113
|
+
return b.node(
|
|
114
|
+
["PySymbol", "PyExternal"],
|
|
115
|
+
"signature",
|
|
116
|
+
signature,
|
|
117
|
+
prune({"name": name, "module": module}),
|
|
118
|
+
)
|
|
92
119
|
|
|
93
120
|
|
|
94
121
|
# ----------------------------------------------------------------------------------------------
|
|
@@ -96,7 +123,9 @@ def _call_endpoint(b: RowBuilder, signature: str, externals: dict) -> NodeRef:
|
|
|
96
123
|
# ----------------------------------------------------------------------------------------------
|
|
97
124
|
|
|
98
125
|
|
|
99
|
-
def _project_module_body(
|
|
126
|
+
def _project_module_body(
|
|
127
|
+
b: RowBuilder, file_key: str, mod_ref: NodeRef, mod: PyModule
|
|
128
|
+
) -> None:
|
|
100
129
|
for fn in (mod.functions or {}).values():
|
|
101
130
|
_project_callable(b, file_key, mod_ref, "PY_DECLARES", fn)
|
|
102
131
|
for cl in (mod.classes or {}).values():
|
|
@@ -107,25 +136,47 @@ def _project_module_body(b: RowBuilder, file_key: str, mod_ref: NodeRef, mod: Py
|
|
|
107
136
|
|
|
108
137
|
|
|
109
138
|
def _project_imports(b: RowBuilder, mod_ref: NodeRef, mod: PyModule) -> None:
|
|
110
|
-
#
|
|
111
|
-
#
|
|
139
|
+
# At most one PY_IMPORTS edge per (module, target) pair -- mirrors PY_CALLS,
|
|
140
|
+
# which pre-aggregates for the same reason: both writers MERGE edges on
|
|
141
|
+
# (type, from, to) and SET their props, so a second row for the same pair
|
|
142
|
+
# would silently overwrite the first in Neo4j instead of adding an edge.
|
|
143
|
+
# Buckets key on the edge's target identity (the resolved module, or the
|
|
144
|
+
# spelling itself for unresolved/external imports), so the SAME target
|
|
145
|
+
# imported under different spellings (``from pkg import util``,
|
|
146
|
+
# ``from . import util as u``, ``from .util import helper``) collapses
|
|
147
|
+
# onto one edge; the raw spellings ride along as a ``spellings`` array.
|
|
148
|
+
# Resolved internal imports point at the real :PyModule; externals keep
|
|
149
|
+
# the shared :PyPackage. Unresolved *relative* spellings (".", ".foo")
|
|
150
|
+
# name no package -- they are dropped from the graph (the spelling
|
|
151
|
+
# survives in analysis.json), instead of minting bogus
|
|
152
|
+
# :PyPackage{name: "."} nodes.
|
|
112
153
|
agg: dict = {}
|
|
113
154
|
for im in mod.imports or []:
|
|
114
155
|
if not im.module:
|
|
115
|
-
continue
|
|
116
|
-
|
|
156
|
+
continue
|
|
157
|
+
if im.resolved_module is None and im.module.startswith("."):
|
|
158
|
+
continue
|
|
159
|
+
key = im.resolved_module or im.module
|
|
160
|
+
a = agg.setdefault(
|
|
161
|
+
key, {"spellings": set(), "names": set(), "aliases": set(), "resolved": im.resolved_module}
|
|
162
|
+
)
|
|
163
|
+
a["spellings"].add(im.module)
|
|
117
164
|
if im.name:
|
|
118
165
|
a["names"].add(im.name)
|
|
119
166
|
if im.alias:
|
|
120
167
|
a["aliases"].add(im.alias)
|
|
121
|
-
for
|
|
122
|
-
|
|
168
|
+
for key, a in agg.items():
|
|
169
|
+
if a["resolved"] is not None:
|
|
170
|
+
target = NodeRef("PyModule", "file_key", a["resolved"])
|
|
171
|
+
else:
|
|
172
|
+
target = b.node(["PyPackage"], "name", key, {})
|
|
123
173
|
b.edge(
|
|
124
174
|
"PY_IMPORTS",
|
|
125
175
|
mod_ref,
|
|
126
|
-
|
|
176
|
+
target,
|
|
127
177
|
prune(
|
|
128
178
|
{
|
|
179
|
+
"spellings": sorted(a["spellings"]),
|
|
129
180
|
"imported_names": sorted(a["names"]) or None,
|
|
130
181
|
"aliases": sorted(a["aliases"]) or None,
|
|
131
182
|
}
|
|
@@ -141,7 +192,9 @@ def _project_imports(b: RowBuilder, mod_ref: NodeRef, mod: PyModule) -> None:
|
|
|
141
192
|
def _project_class(
|
|
142
193
|
b: RowBuilder, file_key: str, parent: NodeRef, parent_rel: str, cl: PyClass
|
|
143
194
|
) -> None:
|
|
144
|
-
ref = b.node(
|
|
195
|
+
ref = b.node(
|
|
196
|
+
["PySymbol", "PyClass"], "signature", cl.signature, _class_props(cl, file_key)
|
|
197
|
+
)
|
|
145
198
|
b.edge(parent_rel, parent, ref)
|
|
146
199
|
|
|
147
200
|
for base in cl.base_classes or []:
|
|
@@ -158,7 +211,12 @@ def _project_class(
|
|
|
158
211
|
def _project_callable(
|
|
159
212
|
b: RowBuilder, file_key: str, owner: NodeRef, owner_rel: str, c: PyCallable
|
|
160
213
|
) -> None:
|
|
161
|
-
ref = b.node(
|
|
214
|
+
ref = b.node(
|
|
215
|
+
["PySymbol", "PyCallable"],
|
|
216
|
+
"signature",
|
|
217
|
+
c.signature,
|
|
218
|
+
_callable_props(c, file_key),
|
|
219
|
+
)
|
|
162
220
|
b.edge(owner_rel, owner, ref)
|
|
163
221
|
|
|
164
222
|
for d in c.decorators or []:
|
|
@@ -166,7 +224,9 @@ def _project_callable(
|
|
|
166
224
|
|
|
167
225
|
for s in c.call_sites or []:
|
|
168
226
|
# Key off the relative file (a call site lives in its callable's file) so ids stay portable.
|
|
169
|
-
cs_id =
|
|
227
|
+
cs_id = (
|
|
228
|
+
f"{file_key}#{s.start_line}:{s.start_column}-{s.end_line}:{s.end_column}"
|
|
229
|
+
)
|
|
170
230
|
cs = b.node(["PyCallSite"], "id", cs_id, _call_site_props(s, file_key))
|
|
171
231
|
b.edge("PY_HAS_CALLSITE", ref, cs)
|
|
172
232
|
if s.callee_signature:
|
|
@@ -189,7 +249,11 @@ def _project_attribute(
|
|
|
189
249
|
|
|
190
250
|
|
|
191
251
|
def _project_variable(
|
|
192
|
-
b: RowBuilder,
|
|
252
|
+
b: RowBuilder,
|
|
253
|
+
file_key: str,
|
|
254
|
+
owner: NodeRef,
|
|
255
|
+
owner_id: str,
|
|
256
|
+
v: PyVariableDeclaration,
|
|
193
257
|
) -> None:
|
|
194
258
|
var_id = f"{owner_id}#{v.name}@{v.start_line}"
|
|
195
259
|
ref = b.node(["PyVariable"], "id", var_id, _variable_props(v, var_id, file_key))
|
|
@@ -258,6 +322,7 @@ def _attribute_props(a: PyClassAttribute, attr_id: str, file_key: str) -> Props:
|
|
|
258
322
|
"id": attr_id,
|
|
259
323
|
"name": a.name,
|
|
260
324
|
"type": a.type,
|
|
325
|
+
"initializer": a.initializer,
|
|
261
326
|
"docstring": _docstring_of(a.comments),
|
|
262
327
|
"start_line": a.start_line,
|
|
263
328
|
"end_line": a.end_line,
|
|
@@ -290,6 +355,7 @@ def _call_site_props(s: PyCallsite, file_key: str) -> Props:
|
|
|
290
355
|
"receiver_expr": s.receiver_expr,
|
|
291
356
|
"receiver_type": s.receiver_type,
|
|
292
357
|
"argument_types": list(s.argument_types or []),
|
|
358
|
+
"arguments_json": _stringify_if(s.arguments),
|
|
293
359
|
"return_type": s.return_type,
|
|
294
360
|
"callee_signature": s.callee_signature,
|
|
295
361
|
"is_constructor_call": s.is_constructor_call,
|
|
@@ -14,27 +14,28 @@
|
|
|
14
14
|
# limitations under the License.
|
|
15
15
|
################################################################################
|
|
16
16
|
|
|
17
|
-
"""The declarative Neo4j schema catalog — the single in-repo source of truth for
|
|
18
|
-
the graph contract (node labels, their keys and typed properties, relationship
|
|
19
|
-
types and their endpoints). ``--emit schema`` serializes this (with the DDL from
|
|
20
|
-
:mod:`codeanalyzer.neo4j.schema`) to a machine-readable ``schema.json``, and the
|
|
21
|
-
conformance test (``test/test_neo4j_schema.py``) asserts the real emitter never
|
|
22
|
-
produces a label / relationship / property that isn't declared here — so this
|
|
23
|
-
file cannot silently drift from :mod:`codeanalyzer.neo4j.project`.
|
|
24
|
-
|
|
25
|
-
SCHEMA_VERSION is the contract version: bump MAJOR on a breaking change
|
|
26
|
-
(renamed/removed label, relationship or key), MINOR on an additive change (new
|
|
27
|
-
label/rel/property). It is stamped onto the ``:PyApplication`` node of every
|
|
28
|
-
emitted graph so any consumer can detect a producer/consumer mismatch at runtime.
|
|
29
17
|
"""
|
|
18
|
+
The declarative Neo4j schema — the single in-repo source of truth for the graph contract: node
|
|
19
|
+
labels with their keys and typed properties, relationship types and their endpoints, and the
|
|
20
|
+
Cypher DDL (uniqueness constraints + indexes). The constraints are DERIVED from the node labels
|
|
21
|
+
(one per distinct mergeLabel/key) so a new label brings its own constraint — there is no second
|
|
22
|
+
list to keep in sync. `--emit schema` serializes all of this to a machine-readable schema.json,
|
|
23
|
+
and the conformance test (``test/test_neo4j_schema.py``) asserts the real emitter never produces a
|
|
24
|
+
label / relationship / property that isn't declared here — so this file cannot silently drift
|
|
25
|
+
from :mod:`codeanalyzer.neo4j.project`.
|
|
26
|
+
|
|
27
|
+
SCHEMA_VERSION is the contract version: bump MAJOR on a breaking change (renamed/removed label,
|
|
28
|
+
relationship or key), MINOR on an additive change (new label/rel/property). It is stamped onto
|
|
29
|
+
the :PyApplication node of every emitted graph so any consumer can detect a producer/consumer
|
|
30
|
+
mismatch at runtime.
|
|
31
|
+
"""
|
|
32
|
+
|
|
30
33
|
from __future__ import annotations
|
|
31
34
|
|
|
32
35
|
from dataclasses import dataclass, field
|
|
33
36
|
from typing import Dict, List
|
|
34
37
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
SCHEMA_VERSION = "1.1.0"
|
|
38
|
+
SCHEMA_VERSION = "1.2.0"
|
|
38
39
|
|
|
39
40
|
# PropType ∈ {"string", "integer", "float", "boolean", "string[]", "integer[]"}.
|
|
40
41
|
|
|
@@ -66,7 +67,15 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
66
67
|
"PyApplication",
|
|
67
68
|
"PyApplication",
|
|
68
69
|
"name",
|
|
69
|
-
{
|
|
70
|
+
{
|
|
71
|
+
"name": "string",
|
|
72
|
+
"schema_version": "string",
|
|
73
|
+
"analyzer_name": "string",
|
|
74
|
+
"analyzer_version": "string",
|
|
75
|
+
"repo_uri": "string",
|
|
76
|
+
"source_revision": "string",
|
|
77
|
+
"repo_dirty": "boolean",
|
|
78
|
+
},
|
|
70
79
|
),
|
|
71
80
|
NodeLabel(
|
|
72
81
|
"PyModule",
|
|
@@ -138,6 +147,7 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
138
147
|
"receiver_expr": "string",
|
|
139
148
|
"receiver_type": "string",
|
|
140
149
|
"argument_types": "string[]",
|
|
150
|
+
"arguments_json": "string",
|
|
141
151
|
"return_type": "string",
|
|
142
152
|
"callee_signature": "string",
|
|
143
153
|
"is_constructor_call": "boolean",
|
|
@@ -156,6 +166,7 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
156
166
|
"id": "string",
|
|
157
167
|
"name": "string",
|
|
158
168
|
"type": "string",
|
|
169
|
+
"initializer": "string",
|
|
159
170
|
"docstring": "string",
|
|
160
171
|
**_SPAN,
|
|
161
172
|
"_module": "string",
|
|
@@ -198,13 +209,42 @@ REL_TYPES: List[RelType] = [
|
|
|
198
209
|
RelType(
|
|
199
210
|
"PY_IMPORTS",
|
|
200
211
|
["PyModule"],
|
|
201
|
-
["PyPackage"],
|
|
202
|
-
{"imported_names": "string[]", "aliases": "string[]"},
|
|
212
|
+
["PyModule", "PyPackage"],
|
|
213
|
+
{"spellings": "string[]", "imported_names": "string[]", "aliases": "string[]"},
|
|
203
214
|
),
|
|
204
215
|
RelType("PY_DECORATED_BY", ["PyCallable"], ["PyDecorator"]),
|
|
205
216
|
]
|
|
206
217
|
|
|
207
218
|
|
|
219
|
+
def uniqueness_constraints() -> list[str]:
|
|
220
|
+
"""One uniqueness constraint per distinct (merge_label, key)."""
|
|
221
|
+
seen: set[tuple[str, str]] = set()
|
|
222
|
+
out: list[str] = []
|
|
223
|
+
|
|
224
|
+
for node in NODE_LABELS:
|
|
225
|
+
identifier = (node.merge_label, node.key)
|
|
226
|
+
if identifier in seen:
|
|
227
|
+
continue
|
|
228
|
+
|
|
229
|
+
seen.add(identifier)
|
|
230
|
+
out.append(
|
|
231
|
+
f"CREATE CONSTRAINT {node.merge_label.lower()}_{node.key} "
|
|
232
|
+
f"IF NOT EXISTS FOR (x:{node.merge_label}) "
|
|
233
|
+
f"REQUIRE x.{node.key} IS UNIQUE"
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
return out
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
CONSTRAINTS: List[str] = uniqueness_constraints()
|
|
240
|
+
|
|
241
|
+
INDEXES: List[str] = [
|
|
242
|
+
"CREATE INDEX py_callable_name IF NOT EXISTS FOR (c:PyCallable) ON (c.name)",
|
|
243
|
+
"CREATE INDEX py_class_name IF NOT EXISTS FOR (c:PyClass) ON (c.name)",
|
|
244
|
+
"CREATE FULLTEXT INDEX py_code_fts IF NOT EXISTS FOR (c:PyCallable) ON EACH [c.code, c.docstring]",
|
|
245
|
+
]
|
|
246
|
+
|
|
247
|
+
|
|
208
248
|
@dataclass
|
|
209
249
|
class SchemaDocument:
|
|
210
250
|
schema_version: str
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Provenance capture: repository (git) and analyzer identity for a snapshot.
|
|
2
|
+
|
|
3
|
+
Runs git only as a subprocess query against the analyzed project directory —
|
|
4
|
+
never mutates anything. Absence of git (no repo, no binary) degrades to None.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import subprocess
|
|
9
|
+
from importlib.metadata import PackageNotFoundError, version as _pkg_version
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Optional, Union
|
|
12
|
+
from urllib.parse import urlsplit, urlunsplit
|
|
13
|
+
|
|
14
|
+
from codeanalyzer.schema.py_schema import PyAnalyzerInfo, PyRepositoryInfo
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _strip_userinfo(uri: str) -> str:
|
|
18
|
+
"""Drop credentials from URL-style remotes; scp-style git@host:path is a
|
|
19
|
+
username, not a secret, and parses with no netloc — left intact."""
|
|
20
|
+
parts = urlsplit(uri)
|
|
21
|
+
if parts.netloc and "@" in parts.netloc:
|
|
22
|
+
return urlunsplit(parts._replace(netloc=parts.netloc.rpartition("@")[2]))
|
|
23
|
+
return uri
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _git(project_dir: Union[Path, str], *args: str) -> Optional[str]:
|
|
27
|
+
"""One git query; None on any failure (no repo, no git, timeout)."""
|
|
28
|
+
try:
|
|
29
|
+
result = subprocess.run(
|
|
30
|
+
["git", "-C", str(project_dir), *args],
|
|
31
|
+
capture_output=True, text=True, timeout=10, check=False,
|
|
32
|
+
)
|
|
33
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
34
|
+
return None
|
|
35
|
+
if result.returncode != 0:
|
|
36
|
+
return None
|
|
37
|
+
return result.stdout.strip()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def repository_info(project_dir: Union[Path, str]) -> Optional[PyRepositoryInfo]:
|
|
41
|
+
"""Git provenance of ``project_dir``, or None when it isn't a git checkout."""
|
|
42
|
+
revision = _git(project_dir, "rev-parse", "HEAD")
|
|
43
|
+
if not revision:
|
|
44
|
+
return None
|
|
45
|
+
raw_uri = _git(project_dir, "remote", "get-url", "origin")
|
|
46
|
+
uri = _strip_userinfo(raw_uri) if raw_uri else None
|
|
47
|
+
status = _git(project_dir, "status", "--porcelain", "--untracked-files=no")
|
|
48
|
+
return PyRepositoryInfo(uri=uri, revision=revision, dirty=bool(status))
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def analyzer_info(analysis_level: int) -> PyAnalyzerInfo:
|
|
52
|
+
"""Identity + configuration of this analyzer run."""
|
|
53
|
+
try:
|
|
54
|
+
version = _pkg_version("codeanalyzer-python")
|
|
55
|
+
except PackageNotFoundError:
|
|
56
|
+
version = "unknown"
|
|
57
|
+
return PyAnalyzerInfo(
|
|
58
|
+
name="codeanalyzer-python",
|
|
59
|
+
version=version,
|
|
60
|
+
config={"analysis_level": analysis_level},
|
|
61
|
+
)
|
|
@@ -176,6 +176,7 @@ class PyImport(BaseModel):
|
|
|
176
176
|
module: str
|
|
177
177
|
name: str
|
|
178
178
|
alias: Optional[str] = None
|
|
179
|
+
resolved_module: Optional[str] = None
|
|
179
180
|
start_line: int = -1
|
|
180
181
|
end_line: int = -1
|
|
181
182
|
start_column: int = -1
|
|
@@ -240,6 +241,19 @@ class PyCallableParameter(BaseModel):
|
|
|
240
241
|
end_column: int = -1
|
|
241
242
|
|
|
242
243
|
|
|
244
|
+
@builder
|
|
245
|
+
@msgpk
|
|
246
|
+
class PyCallArgument(BaseModel):
|
|
247
|
+
"""One call-site argument: AST category + inferred type, kept separate.
|
|
248
|
+
|
|
249
|
+
The legacy ``PyCallsite.argument_types`` mixed these two vocabularies
|
|
250
|
+
in one list; this model is the disambiguated replacement (#86).
|
|
251
|
+
"""
|
|
252
|
+
|
|
253
|
+
ast_kind: str
|
|
254
|
+
inferred_type: Optional[str] = None
|
|
255
|
+
|
|
256
|
+
|
|
243
257
|
@builder
|
|
244
258
|
@msgpk
|
|
245
259
|
class PyCallsite(BaseModel):
|
|
@@ -249,6 +263,7 @@ class PyCallsite(BaseModel):
|
|
|
249
263
|
receiver_expr: Optional[str] = None
|
|
250
264
|
receiver_type: Optional[str] = None
|
|
251
265
|
argument_types: List[str] = []
|
|
266
|
+
arguments: List[PyCallArgument] = []
|
|
252
267
|
return_type: Optional[str] = None
|
|
253
268
|
callee_signature: Optional[str] = None
|
|
254
269
|
is_constructor_call: bool = False
|
|
@@ -295,6 +310,7 @@ class PyClassAttribute(BaseModel):
|
|
|
295
310
|
|
|
296
311
|
name: str
|
|
297
312
|
type: Optional[str] = None
|
|
313
|
+
initializer: Optional[str] = None
|
|
298
314
|
comments: List[PyComment] = []
|
|
299
315
|
start_line: int = -1
|
|
300
316
|
end_line: int = -1
|
|
@@ -369,6 +385,26 @@ class PyExternalSymbol(BaseModel):
|
|
|
369
385
|
module: Optional[str] = None # best-effort owning module, e.g. "requests"
|
|
370
386
|
|
|
371
387
|
|
|
388
|
+
@builder
|
|
389
|
+
@msgpk
|
|
390
|
+
class PyRepositoryInfo(BaseModel):
|
|
391
|
+
"""Where the analyzed source came from: git provenance captured at analysis time."""
|
|
392
|
+
|
|
393
|
+
uri: Optional[str] = None
|
|
394
|
+
revision: str
|
|
395
|
+
dirty: bool = False
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
@builder
|
|
399
|
+
@msgpk
|
|
400
|
+
class PyAnalyzerInfo(BaseModel):
|
|
401
|
+
"""Which analyzer produced this snapshot, and how it was configured."""
|
|
402
|
+
|
|
403
|
+
name: str
|
|
404
|
+
version: str
|
|
405
|
+
config: Dict[str, Any] = {}
|
|
406
|
+
|
|
407
|
+
|
|
372
408
|
@builder
|
|
373
409
|
@msgpk
|
|
374
410
|
class PyApplication(BaseModel):
|
|
@@ -380,3 +416,5 @@ class PyApplication(BaseModel):
|
|
|
380
416
|
# builtin members), keyed by signature. Populated by the analyzer so every
|
|
381
417
|
# backend (JSON and Neo4j) shares one authoritative external-symbol set.
|
|
382
418
|
external_symbols: Dict[str, PyExternalSymbol] = {}
|
|
419
|
+
analyzer: Optional[PyAnalyzerInfo] = None
|
|
420
|
+
repository: Optional[PyRepositoryInfo] = None
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Static resolution of import spellings against the analyzed module set.
|
|
2
|
+
|
|
3
|
+
Pure post-pass over a built ``PyApplication``: no filesystem access, no
|
|
4
|
+
sys.path semantics — a spelling resolves iff it names a module that was
|
|
5
|
+
itself analyzed (issue #82). External/library imports stay unresolved by
|
|
6
|
+
design and keep their :PyPackage projection.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Dict, Optional, Union
|
|
13
|
+
|
|
14
|
+
from codeanalyzer.schema.py_schema import PyApplication
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _dotted_candidates(app: PyApplication, project_dir: Union[Path, str]) -> Dict[str, str]:
|
|
18
|
+
"""dotted module path -> file_key, for every analyzed module.
|
|
19
|
+
|
|
20
|
+
``pkg/util.py`` -> ``pkg.util``; ``pkg/__init__.py`` -> ``pkg``.
|
|
21
|
+
file_keys share project_dir's form (both come from the same CLI arg),
|
|
22
|
+
so os.path.relpath keeps mixed absolute/relative setups consistent.
|
|
23
|
+
"""
|
|
24
|
+
mapping: Dict[str, str] = {}
|
|
25
|
+
for file_key in app.symbol_table:
|
|
26
|
+
rel = os.path.relpath(file_key, str(project_dir))
|
|
27
|
+
if rel.startswith(".."):
|
|
28
|
+
continue
|
|
29
|
+
parts = Path(rel).with_suffix("").parts
|
|
30
|
+
if parts and parts[-1] == "__init__":
|
|
31
|
+
parts = parts[:-1]
|
|
32
|
+
if parts:
|
|
33
|
+
mapping[".".join(parts)] = file_key
|
|
34
|
+
return mapping
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _resolve_one(
|
|
38
|
+
spelling: str, original_name: str, importer_rel_parts: tuple, candidates: Dict[str, str]
|
|
39
|
+
) -> Optional[str]:
|
|
40
|
+
if spelling.startswith("."):
|
|
41
|
+
level = len(spelling) - len(spelling.lstrip("."))
|
|
42
|
+
suffix = spelling.lstrip(".")
|
|
43
|
+
# level 1 = the importer's own package; each extra dot walks one up.
|
|
44
|
+
package_parts = importer_rel_parts[:-1] # drop the filename
|
|
45
|
+
if level - 1 > len(package_parts):
|
|
46
|
+
return None
|
|
47
|
+
base = package_parts[: len(package_parts) - (level - 1)]
|
|
48
|
+
stems = list(base) + (suffix.split(".") if suffix else [])
|
|
49
|
+
else:
|
|
50
|
+
stems = spelling.split(".")
|
|
51
|
+
dotted = ".".join(stems)
|
|
52
|
+
with_name = f"{dotted}.{original_name}" if dotted else original_name
|
|
53
|
+
return candidates.get(with_name) or candidates.get(dotted)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def resolve_imports(app: PyApplication, project_dir: Union[Path, str]) -> None:
|
|
57
|
+
"""Stamp ``resolved_module`` on every import of every module, in place."""
|
|
58
|
+
candidates = _dotted_candidates(app, project_dir)
|
|
59
|
+
for file_key, module in app.symbol_table.items():
|
|
60
|
+
rel = os.path.relpath(file_key, str(project_dir))
|
|
61
|
+
importer_parts = Path(rel).parts
|
|
62
|
+
for im in module.imports or []:
|
|
63
|
+
if not im.module:
|
|
64
|
+
im.resolved_module = None
|
|
65
|
+
continue
|
|
66
|
+
original = im.alias or im.name
|
|
67
|
+
im.resolved_module = _resolve_one(im.module, original, importer_parts, candidates)
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import ast
|
|
2
2
|
import hashlib
|
|
3
|
+
import os
|
|
3
4
|
import tokenize
|
|
4
5
|
from ast import AST, ClassDef
|
|
5
6
|
from io import StringIO
|
|
@@ -13,6 +14,7 @@ from jedi.api.project import Project
|
|
|
13
14
|
from codeanalyzer.schema.py_schema import (
|
|
14
15
|
PyCallable,
|
|
15
16
|
PyCallableParameter,
|
|
17
|
+
PyCallArgument,
|
|
16
18
|
PyCallsite,
|
|
17
19
|
PyClass,
|
|
18
20
|
PyClassAttribute,
|
|
@@ -28,7 +30,11 @@ class SymbolTableBuilder:
|
|
|
28
30
|
"""A class for building a symbol table for a Python project."""
|
|
29
31
|
|
|
30
32
|
def __init__(self, project_dir: Union[Path, str], virtualenv: Union[Path, str, None]) -> None:
|
|
31
|
-
|
|
33
|
+
# Jedi reports absolute Script paths, so a relative project_dir would
|
|
34
|
+
# crash every relative_to() fallback below. abspath (not resolve())
|
|
35
|
+
# keeps symlinks intact, matching Jedi's own absolute()-style
|
|
36
|
+
# normalization under symlinked roots like macOS /tmp.
|
|
37
|
+
self.project_dir = Path(os.path.abspath(project_dir))
|
|
32
38
|
if virtualenv is None:
|
|
33
39
|
# If no virtual environment is provided, create a jedi project without an environment.
|
|
34
40
|
self.jedi_project: Project = jedi.Project(path=self.project_dir)
|
|
@@ -39,6 +45,16 @@ class SymbolTableBuilder:
|
|
|
39
45
|
environment_path=Path(virtualenv) / "bin" / "python",
|
|
40
46
|
)
|
|
41
47
|
|
|
48
|
+
def _fallback_signature(self, script_path: Union[Path, str], name: str) -> str:
|
|
49
|
+
"""Path-derived qualified name used when Jedi can't name a definition.
|
|
50
|
+
|
|
51
|
+
Strips only the terminal ``.py`` suffix — a bare ``str.replace``
|
|
52
|
+
would also eat interior ``.py`` substrings and corrupt the module
|
|
53
|
+
prefix (``odoo/tools/pycompat.py`` → ``odoo.toolscompat``).
|
|
54
|
+
"""
|
|
55
|
+
relative = Path(script_path).relative_to(self.project_dir)
|
|
56
|
+
return ".".join(relative.with_suffix("").parts) + f".{name}"
|
|
57
|
+
|
|
42
58
|
@staticmethod
|
|
43
59
|
def _infer_type(script: Script, line: int, column: int) -> str:
|
|
44
60
|
"""Tries to infer the type at a given position using Jedi."""
|
|
@@ -97,6 +113,44 @@ class SymbolTableBuilder:
|
|
|
97
113
|
except Exception:
|
|
98
114
|
return None, False
|
|
99
115
|
|
|
116
|
+
@staticmethod
|
|
117
|
+
def _callee_anchor(node: ast.Call) -> Tuple[int, int]:
|
|
118
|
+
"""Position of the callee *name* for Jedi inference.
|
|
119
|
+
|
|
120
|
+
An ``ast.Call``'s own ``lineno``/``col_offset`` is the first
|
|
121
|
+
character of the whole call expression — for an attribute call
|
|
122
|
+
``receiver.method(...)`` that is the receiver token, and Jedi
|
|
123
|
+
would infer the receiver's type instead of the invoked method
|
|
124
|
+
(issue #80). Anchor attribute calls inside the attribute name —
|
|
125
|
+
its last character, so one-character names stay in range; other
|
|
126
|
+
callee shapes keep the call-expression start.
|
|
127
|
+
"""
|
|
128
|
+
func_expr = node.func
|
|
129
|
+
if isinstance(func_expr, ast.Attribute):
|
|
130
|
+
return func_expr.end_lineno, func_expr.end_col_offset - 1
|
|
131
|
+
return node.lineno, node.col_offset
|
|
132
|
+
|
|
133
|
+
@staticmethod
|
|
134
|
+
def _infer_call_return_type(script: Script, line: int, column: int) -> Optional[str]:
|
|
135
|
+
"""Inferred type of the call's *result*, not of the callee itself.
|
|
136
|
+
|
|
137
|
+
``Script.infer`` at the callee name yields the function/class
|
|
138
|
+
being called; executing that definition yields what the call
|
|
139
|
+
evaluates to — a function's inferred return type, or the
|
|
140
|
+
instance for a constructor call. Returns ``None`` when Jedi
|
|
141
|
+
can't tell, so an unknown stays absent instead of masquerading
|
|
142
|
+
as the callee's own name.
|
|
143
|
+
"""
|
|
144
|
+
try:
|
|
145
|
+
definitions = script.infer(line=line, column=column)
|
|
146
|
+
if definitions:
|
|
147
|
+
results = definitions[0].execute()
|
|
148
|
+
if results:
|
|
149
|
+
return results[0].name
|
|
150
|
+
except Exception:
|
|
151
|
+
pass
|
|
152
|
+
return None
|
|
153
|
+
|
|
100
154
|
def build_pymodule_from_file(self, py_file: Path) -> PyModule:
|
|
101
155
|
"""Builds a PyModule from a Python file.
|
|
102
156
|
|
|
@@ -203,10 +257,10 @@ class SymbolTableBuilder:
|
|
|
203
257
|
definitions = script.goto(line=start_line, column=child.col_offset)
|
|
204
258
|
signature = next(
|
|
205
259
|
(d.full_name for d in definitions if d.type == "class"),
|
|
206
|
-
|
|
260
|
+
self._fallback_signature(script.path, class_name),
|
|
207
261
|
)
|
|
208
262
|
except Exception:
|
|
209
|
-
signature =
|
|
263
|
+
signature = self._fallback_signature(script.path, class_name)
|
|
210
264
|
py_class = (
|
|
211
265
|
PyClass.builder()
|
|
212
266
|
.name(class_name)
|
|
@@ -258,8 +312,7 @@ class SymbolTableBuilder:
|
|
|
258
312
|
|
|
259
313
|
# If Jedi didn't provide a signature, build one relative to project_dir
|
|
260
314
|
if not signature:
|
|
261
|
-
|
|
262
|
-
signature = f"{str(relative_path).replace('/', '.').replace('.py', '')}.{method_name}"
|
|
315
|
+
signature = self._fallback_signature(script.path, method_name)
|
|
263
316
|
py_callable = (
|
|
264
317
|
PyCallable.builder()
|
|
265
318
|
.name(method_name) # Use the actual method name, not the full signature
|
|
@@ -380,6 +433,7 @@ class SymbolTableBuilder:
|
|
|
380
433
|
script, target.lineno, target.col_offset
|
|
381
434
|
)
|
|
382
435
|
)
|
|
436
|
+
.initializer(ast.unparse(stmt.value) if stmt.value else None)
|
|
383
437
|
.start_line(getattr(target, "lineno", -1))
|
|
384
438
|
.end_line(getattr(stmt, "end_lineno", stmt.lineno))
|
|
385
439
|
.build()
|
|
@@ -398,6 +452,7 @@ class SymbolTableBuilder:
|
|
|
398
452
|
script, target.lineno, target.col_offset
|
|
399
453
|
)
|
|
400
454
|
)
|
|
455
|
+
.initializer(ast.unparse(stmt.value) if stmt.value else None)
|
|
401
456
|
.start_line(getattr(target, "lineno", -1))
|
|
402
457
|
.end_line(getattr(stmt, "end_lineno", stmt.lineno))
|
|
403
458
|
.build()
|
|
@@ -588,10 +643,11 @@ class SymbolTableBuilder:
|
|
|
588
643
|
func_expr = node.func
|
|
589
644
|
|
|
590
645
|
method_name = "<unknown>"
|
|
646
|
+
anchor_line, anchor_col = self._callee_anchor(node)
|
|
591
647
|
callee_signature, is_constructor = self._infer_callee(
|
|
592
|
-
script,
|
|
648
|
+
script, anchor_line, anchor_col
|
|
593
649
|
)
|
|
594
|
-
return_type = self.
|
|
650
|
+
return_type = self._infer_call_return_type(script, anchor_line, anchor_col)
|
|
595
651
|
|
|
596
652
|
receiver_expr = None
|
|
597
653
|
receiver_type = None
|
|
@@ -604,18 +660,26 @@ class SymbolTableBuilder:
|
|
|
604
660
|
elif isinstance(func_expr, ast.Name):
|
|
605
661
|
method_name = func_expr.id
|
|
606
662
|
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
663
|
+
arguments = [
|
|
664
|
+
PyCallArgument(
|
|
665
|
+
ast_kind=type(arg).__name__,
|
|
666
|
+
inferred_type=self._infer_type(script, arg.lineno, arg.col_offset),
|
|
667
|
+
)
|
|
610
668
|
for arg in node.args
|
|
611
669
|
]
|
|
612
670
|
|
|
671
|
+
# Legacy field, derived from the structured arguments above rather
|
|
672
|
+
# than re-inferring: byte-identical to the old
|
|
673
|
+
# `self._infer_type(...) or type(arg).__name__` per argument.
|
|
674
|
+
argument_types = [a.inferred_type or a.ast_kind for a in arguments]
|
|
675
|
+
|
|
613
676
|
call_sites.append(
|
|
614
677
|
PyCallsite.builder()
|
|
615
678
|
.method_name(method_name)
|
|
616
679
|
.receiver_expr(receiver_expr)
|
|
617
680
|
.receiver_type(receiver_type)
|
|
618
681
|
.argument_types(argument_types)
|
|
682
|
+
.arguments(arguments)
|
|
619
683
|
.return_type(return_type)
|
|
620
684
|
.callee_signature(callee_signature)
|
|
621
685
|
.is_constructor_call(is_constructor)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"schema_version": "1.
|
|
2
|
+
"schema_version": "1.2.0",
|
|
3
3
|
"generator": "codeanalyzer-python",
|
|
4
4
|
"marker_labels": [],
|
|
5
5
|
"node_labels": [
|
|
@@ -9,7 +9,12 @@
|
|
|
9
9
|
"key": "name",
|
|
10
10
|
"properties": {
|
|
11
11
|
"name": "string",
|
|
12
|
-
"schema_version": "string"
|
|
12
|
+
"schema_version": "string",
|
|
13
|
+
"analyzer_name": "string",
|
|
14
|
+
"analyzer_version": "string",
|
|
15
|
+
"repo_uri": "string",
|
|
16
|
+
"source_revision": "string",
|
|
17
|
+
"repo_dirty": "boolean"
|
|
13
18
|
}
|
|
14
19
|
},
|
|
15
20
|
{
|
|
@@ -97,6 +102,7 @@
|
|
|
97
102
|
"receiver_expr": "string",
|
|
98
103
|
"receiver_type": "string",
|
|
99
104
|
"argument_types": "string[]",
|
|
105
|
+
"arguments_json": "string",
|
|
100
106
|
"return_type": "string",
|
|
101
107
|
"callee_signature": "string",
|
|
102
108
|
"is_constructor_call": "boolean",
|
|
@@ -115,6 +121,7 @@
|
|
|
115
121
|
"id": "string",
|
|
116
122
|
"name": "string",
|
|
117
123
|
"type": "string",
|
|
124
|
+
"initializer": "string",
|
|
118
125
|
"docstring": "string",
|
|
119
126
|
"start_line": "integer",
|
|
120
127
|
"end_line": "integer",
|
|
@@ -244,9 +251,11 @@
|
|
|
244
251
|
"PyModule"
|
|
245
252
|
],
|
|
246
253
|
"to": [
|
|
254
|
+
"PyModule",
|
|
247
255
|
"PyPackage"
|
|
248
256
|
],
|
|
249
257
|
"properties": {
|
|
258
|
+
"spellings": "string[]",
|
|
250
259
|
"imported_names": "string[]",
|
|
251
260
|
"aliases": "string[]"
|
|
252
261
|
}
|
|
@@ -263,14 +272,14 @@
|
|
|
263
272
|
}
|
|
264
273
|
],
|
|
265
274
|
"constraints": [
|
|
266
|
-
"CREATE CONSTRAINT
|
|
267
|
-
"CREATE CONSTRAINT
|
|
268
|
-
"CREATE CONSTRAINT
|
|
269
|
-
"CREATE CONSTRAINT
|
|
270
|
-
"CREATE CONSTRAINT
|
|
271
|
-
"CREATE CONSTRAINT
|
|
272
|
-
"CREATE CONSTRAINT
|
|
273
|
-
"CREATE CONSTRAINT
|
|
275
|
+
"CREATE CONSTRAINT pyapplication_name IF NOT EXISTS FOR (x:PyApplication) REQUIRE x.name IS UNIQUE",
|
|
276
|
+
"CREATE CONSTRAINT pymodule_file_key IF NOT EXISTS FOR (x:PyModule) REQUIRE x.file_key IS UNIQUE",
|
|
277
|
+
"CREATE CONSTRAINT pysymbol_signature IF NOT EXISTS FOR (x:PySymbol) REQUIRE x.signature IS UNIQUE",
|
|
278
|
+
"CREATE CONSTRAINT pypackage_name IF NOT EXISTS FOR (x:PyPackage) REQUIRE x.name IS UNIQUE",
|
|
279
|
+
"CREATE CONSTRAINT pydecorator_name IF NOT EXISTS FOR (x:PyDecorator) REQUIRE x.name IS UNIQUE",
|
|
280
|
+
"CREATE CONSTRAINT pycallsite_id IF NOT EXISTS FOR (x:PyCallSite) REQUIRE x.id IS UNIQUE",
|
|
281
|
+
"CREATE CONSTRAINT pyattribute_id IF NOT EXISTS FOR (x:PyAttribute) REQUIRE x.id IS UNIQUE",
|
|
282
|
+
"CREATE CONSTRAINT pyvariable_id IF NOT EXISTS FOR (x:PyVariable) REQUIRE x.id IS UNIQUE"
|
|
274
283
|
],
|
|
275
284
|
"indexes": [
|
|
276
285
|
"CREATE INDEX py_callable_name IF NOT EXISTS FOR (c:PyCallable) ON (c.name)",
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
################################################################################
|
|
2
|
-
# Copyright IBM Corporation 2025
|
|
3
|
-
#
|
|
4
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
-
# you may not use this file except in compliance with the License.
|
|
6
|
-
# You may obtain a copy of the License at
|
|
7
|
-
#
|
|
8
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
-
#
|
|
10
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
-
# See the License for the specific language governing permissions and
|
|
14
|
-
# limitations under the License.
|
|
15
|
-
################################################################################
|
|
16
|
-
|
|
17
|
-
"""The Cypher DDL — uniqueness constraints and indexes — shared by both writers.
|
|
18
|
-
Run BEFORE any load so MERGE uses an index seek (not a label scan) and the
|
|
19
|
-
identity invariant is enforced by the database. Every statement is idempotent
|
|
20
|
-
(``IF NOT EXISTS``).
|
|
21
|
-
"""
|
|
22
|
-
from typing import List
|
|
23
|
-
|
|
24
|
-
CONSTRAINTS: List[str] = [
|
|
25
|
-
"CREATE CONSTRAINT py_symbol_sig IF NOT EXISTS FOR (s:PySymbol) REQUIRE s.signature IS UNIQUE",
|
|
26
|
-
"CREATE CONSTRAINT py_app_name IF NOT EXISTS FOR (a:PyApplication) REQUIRE a.name IS UNIQUE",
|
|
27
|
-
"CREATE CONSTRAINT py_module_key IF NOT EXISTS FOR (m:PyModule) REQUIRE m.file_key IS UNIQUE",
|
|
28
|
-
"CREATE CONSTRAINT py_package_name IF NOT EXISTS FOR (p:PyPackage) REQUIRE p.name IS UNIQUE",
|
|
29
|
-
"CREATE CONSTRAINT py_decorator_name IF NOT EXISTS FOR (d:PyDecorator) REQUIRE d.name IS UNIQUE",
|
|
30
|
-
"CREATE CONSTRAINT py_callsite_id IF NOT EXISTS FOR (c:PyCallSite) REQUIRE c.id IS UNIQUE",
|
|
31
|
-
"CREATE CONSTRAINT py_attribute_id IF NOT EXISTS FOR (a:PyAttribute) REQUIRE a.id IS UNIQUE",
|
|
32
|
-
"CREATE CONSTRAINT py_variable_id IF NOT EXISTS FOR (v:PyVariable) REQUIRE v.id IS UNIQUE",
|
|
33
|
-
]
|
|
34
|
-
|
|
35
|
-
INDEXES: List[str] = [
|
|
36
|
-
"CREATE INDEX py_callable_name IF NOT EXISTS FOR (c:PyCallable) ON (c.name)",
|
|
37
|
-
"CREATE INDEX py_class_name IF NOT EXISTS FOR (c:PyClass) ON (c.name)",
|
|
38
|
-
"CREATE FULLTEXT INDEX py_code_fts IF NOT EXISTS FOR (c:PyCallable) ON EACH [c.code, c.docstring]",
|
|
39
|
-
]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/semantic_analysis/__init__.py
RENAMED
|
File without changes
|
{codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/semantic_analysis/call_graph.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{codeanalyzer_python-0.3.0 → codeanalyzer_python-0.3.1}/codeanalyzer/syntactic_analysis/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|