codeanalyzer-python 1.2.0__py3-none-any.whl → 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +29 -0
- codeanalyzer/artifacts/__init__.py +20 -0
- codeanalyzer/artifacts/config_keys.py +588 -0
- codeanalyzer/artifacts/config_use.py +597 -0
- codeanalyzer/artifacts/config_use_rules.yml +58 -0
- codeanalyzer/artifacts/dependencies.py +237 -0
- codeanalyzer/artifacts/discovery.py +167 -0
- codeanalyzer/artifacts/parsers.py +248 -0
- codeanalyzer/core.py +91 -0
- codeanalyzer/dataflow/builder.py +15 -1
- codeanalyzer/neo4j/project.py +192 -0
- codeanalyzer/neo4j/schema.py +57 -0
- codeanalyzer/options/options.py +5 -0
- codeanalyzer/schema/ids.py +21 -0
- codeanalyzer/schema/py_schema.py +118 -0
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +11 -0
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/METADATA +109 -5
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/RECORD +22 -15
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/WHEEL +0 -0
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/core.py
CHANGED
|
@@ -37,6 +37,21 @@ from codeanalyzer.utils import ProgressBar
|
|
|
37
37
|
from codeanalyzer.options import AnalysisOptions
|
|
38
38
|
from codeanalyzer.provenance import analyzer_info, repository_info
|
|
39
39
|
|
|
40
|
+
def _artifact_full_text(project_dir: Path, path: str, art) -> str:
|
|
41
|
+
"""Mirrors ``artifacts.dependencies._full_text`` verbatim (not imported
|
|
42
|
+
-- that name is module-private to ``dependencies.py``): config-key
|
|
43
|
+
extraction (#152) must never depend on the stored ``source`` -- capped
|
|
44
|
+
by ``text_max_bytes`` and emptied by ``capture_text=False`` (payload-size
|
|
45
|
+
controls, not extraction controls). Read the real file fresh instead;
|
|
46
|
+
fall back to ``art.source`` only if it's gone (e.g. a synthetic artifact
|
|
47
|
+
in a unit test, or the file vanished mid-run). Keep the two in sync if
|
|
48
|
+
this logic changes."""
|
|
49
|
+
try:
|
|
50
|
+
return (project_dir / path).read_bytes().decode("utf-8")
|
|
51
|
+
except (OSError, UnicodeDecodeError):
|
|
52
|
+
return art.source
|
|
53
|
+
|
|
54
|
+
|
|
40
55
|
def _ensure_ray() -> None:
|
|
41
56
|
"""Initialize Ray with the driver's pinned hash seed in the workers.
|
|
42
57
|
|
|
@@ -648,6 +663,59 @@ class Codeanalyzer:
|
|
|
648
663
|
|
|
649
664
|
detect_entrypoints(app, self.project_dir, self.options.entrypoint_rules)
|
|
650
665
|
|
|
666
|
+
# Artifacts + dependencies: L1 data, every level, never varies with -a
|
|
667
|
+
# (spec 2026-08-27). Deterministic by default; venv probing is opt-in.
|
|
668
|
+
from codeanalyzer.artifacts import (
|
|
669
|
+
build_dependency_view, discover_artifacts, extract_config_keys,
|
|
670
|
+
is_config_eligible,
|
|
671
|
+
)
|
|
672
|
+
|
|
673
|
+
app.artifacts = discover_artifacts(
|
|
674
|
+
self.project_dir, app_name,
|
|
675
|
+
capture_text=self.options.artifact_text,
|
|
676
|
+
text_max_bytes=self.options.artifact_text_max_bytes,
|
|
677
|
+
)
|
|
678
|
+
app.dependencies, app.unresolved_imports = build_dependency_view(
|
|
679
|
+
app.artifacts,
|
|
680
|
+
app.symbol_table,
|
|
681
|
+
self.project_dir,
|
|
682
|
+
self.virtualenv if self.options.resolve_installed else None,
|
|
683
|
+
self.options.resolve_installed,
|
|
684
|
+
)
|
|
685
|
+
|
|
686
|
+
# Config keys (#152): L1 data, layered onto the same artifacts, every
|
|
687
|
+
# level. Namespace-eligible artifacts only (env-family by basename,
|
|
688
|
+
# else by format). `extraction` combines with any prior
|
|
689
|
+
# dependency-manifest pass on the same artifact: a parse failure here
|
|
690
|
+
# always downgrades to "partial" (never drops the artifact); a clean
|
|
691
|
+
# parse upgrades an untouched "none" to "full", but never overwrites
|
|
692
|
+
# an existing "partial" (e.g. from a failed dependency-manifest parse
|
|
693
|
+
# on the same artifact) -- a successful pass here must not silently
|
|
694
|
+
# erase an unrelated failure already recorded on the artifact.
|
|
695
|
+
for path in sorted(app.artifacts):
|
|
696
|
+
art = app.artifacts[path]
|
|
697
|
+
if not is_config_eligible(art):
|
|
698
|
+
continue
|
|
699
|
+
full_text = _artifact_full_text(self.project_dir, path, art)
|
|
700
|
+
keys, ok = extract_config_keys(art, full_text, self.options.artifact_text)
|
|
701
|
+
art.config_keys = keys
|
|
702
|
+
if not ok:
|
|
703
|
+
art.extraction = "partial"
|
|
704
|
+
elif art.extraction == "none":
|
|
705
|
+
art.extraction = "full"
|
|
706
|
+
|
|
707
|
+
# config_use (#162) detection: needs callees resolved (backfill_
|
|
708
|
+
# callees above, gated the same >= 2) and config_keys just extracted
|
|
709
|
+
# above, so this runs after both rather than immediately following
|
|
710
|
+
# the callee backfill call itself. Resolution is deferred past the
|
|
711
|
+
# L3/L4 blocks below (the dataflow tiers need DDG and call_graph/
|
|
712
|
+
# caller-argument substrate those blocks populate) and gated by
|
|
713
|
+
# level there, literal tier included.
|
|
714
|
+
if self.analysis_level >= 2:
|
|
715
|
+
from codeanalyzer.artifacts import detect_config_reads
|
|
716
|
+
|
|
717
|
+
config_reads = detect_config_reads(app)
|
|
718
|
+
|
|
651
719
|
# L3: intraprocedural dataflow (CFG/CDG/DDG) emitted onto the v2 tree.
|
|
652
720
|
if self.analysis_level >= 3:
|
|
653
721
|
from codeanalyzer.dataflow.builder import (
|
|
@@ -688,6 +756,29 @@ class Codeanalyzer:
|
|
|
688
756
|
# ``infos`` are the syntactic (L3) PDGs from the >=3 block above.
|
|
689
757
|
emit_ddg_pointsto_delta(app, infos, ir, sig_to_id)
|
|
690
758
|
|
|
759
|
+
# config_use (#162) resolution: literal tier always runs at >= 2;
|
|
760
|
+
# the dataflow tiers widen the set as substrate comes online -- intra
|
|
761
|
+
# needs the DDG the L3 block above just emitted, interproc needs
|
|
762
|
+
# call_graph + caller PyCallArgument values (already present since
|
|
763
|
+
# L2) plus, for its one-hop caller-side closure, the caller's own
|
|
764
|
+
# DDG -- populated for every callable by the same L3 block, which
|
|
765
|
+
# always runs before this when analysis_level >= 4. A read resolved
|
|
766
|
+
# at a lower tier is never recomputed, so `-a 2 ⊆ -a 3 ⊆ -a 4` holds
|
|
767
|
+
# by construction.
|
|
768
|
+
if self.analysis_level >= 2:
|
|
769
|
+
from codeanalyzer.artifacts import (
|
|
770
|
+
dataflow_intra_tier, dataflow_interproc_tier, resolve_uses,
|
|
771
|
+
)
|
|
772
|
+
|
|
773
|
+
tier_fns = []
|
|
774
|
+
if self.analysis_level >= 3:
|
|
775
|
+
tier_fns.append(dataflow_intra_tier)
|
|
776
|
+
if self.analysis_level >= 4:
|
|
777
|
+
tier_fns.append(dataflow_interproc_tier)
|
|
778
|
+
app.config_uses, app.config_reads_unresolved = resolve_uses(
|
|
779
|
+
config_reads, app, tier_fns=tier_fns
|
|
780
|
+
)
|
|
781
|
+
|
|
691
782
|
# Build the v2 envelope, then persist it (the cache stores the full
|
|
692
783
|
# ``Analysis`` envelope so a reused cache round-trips schema_version).
|
|
693
784
|
# k_limit is an L3+ envelope key: below the dataflow levels it stays
|
codeanalyzer/dataflow/builder.py
CHANGED
|
@@ -353,6 +353,10 @@ def build_program_graphs(
|
|
|
353
353
|
"""
|
|
354
354
|
class_idx = _class_index(app)
|
|
355
355
|
callable_idx = _callable_index(app)
|
|
356
|
+
# Reverse of callable_idx: a callsite's already-L2-backfilled BodyNode.callee
|
|
357
|
+
# (a can:// id, folding in the defuse linker's resolution -- not just Jedi's
|
|
358
|
+
# own callee_signature side channel) resolved back to a dotted signature.
|
|
359
|
+
id_to_sig = {c.id: sig for sig, c in callable_idx.items()}
|
|
356
360
|
|
|
357
361
|
infos, func_asts = build_function_pdgs(app, k, oracle_factory=oracle_factory)
|
|
358
362
|
|
|
@@ -373,7 +377,17 @@ def build_program_graphs(
|
|
|
373
377
|
calls_by_line.setdefault(call.lineno, (node.id, call))
|
|
374
378
|
|
|
375
379
|
for site in pycallable.call_sites or []:
|
|
376
|
-
|
|
380
|
+
# Prefer the callsite's body-backfilled callee over Jedi's own
|
|
381
|
+
# callee_signature side channel: under cross-test parso/Jedi cache
|
|
382
|
+
# pressure that inference can silently degrade (full-suite-only
|
|
383
|
+
# loss of SDG summary/global stitching -- test_dataflow_sdg.py),
|
|
384
|
+
# while BodyNode.callee is the deterministic L2 path (Jedi filtered
|
|
385
|
+
# the same way, PLUS the defuse linker's fallback -- l2_callees.py).
|
|
386
|
+
# Only a resolved INTERNAL target counts (id_to_sig misses on an
|
|
387
|
+
# external/unresolved callee); falls through to callee_signature
|
|
388
|
+
# exactly as before whenever the body doesn't have an answer.
|
|
389
|
+
body_node = pycallable.body.get(f"{site.start_line}:{site.start_column}")
|
|
390
|
+
target = (id_to_sig.get(body_node.callee) if body_node else None) or site.callee_signature
|
|
377
391
|
if not target:
|
|
378
392
|
continue
|
|
379
393
|
if target in class_idx and target not in infos:
|
codeanalyzer/neo4j/project.py
CHANGED
|
@@ -47,6 +47,7 @@ from codeanalyzer.schema import (
|
|
|
47
47
|
PyModule,
|
|
48
48
|
PyVariableDeclaration,
|
|
49
49
|
)
|
|
50
|
+
from codeanalyzer.schema.ids import application_id, purl_pypi
|
|
50
51
|
from codeanalyzer.schema.py_schema import PyDecorator
|
|
51
52
|
|
|
52
53
|
|
|
@@ -100,6 +101,16 @@ def project(app: PyApplication, app_name: str, sig_to_id: dict,
|
|
|
100
101
|
# MERGE — a no-op when no callable carries L3 fields (levels 1/2).
|
|
101
102
|
_project_program_graphs(b, app, externals, sig_to_id)
|
|
102
103
|
|
|
104
|
+
# Neutral artifact/dependency subgraph (Task 6). L1 data — always present,
|
|
105
|
+
# full-depth-always regardless of -a.
|
|
106
|
+
_project_artifacts(b, app, app_name, app_ref)
|
|
107
|
+
|
|
108
|
+
# config_use (#162): the resolved-read bridge (PyBodyNode from
|
|
109
|
+
# _project_program_graphs above) into the config-key subgraph
|
|
110
|
+
# (ConfigKey from _project_artifacts above), plus first-class unresolved
|
|
111
|
+
# reads.
|
|
112
|
+
_project_config_uses(b, app, app_ref, externals, sig_to_id)
|
|
113
|
+
|
|
103
114
|
return b.finish()
|
|
104
115
|
|
|
105
116
|
|
|
@@ -248,6 +259,187 @@ def _project_program_graphs(
|
|
|
248
259
|
)
|
|
249
260
|
|
|
250
261
|
|
|
262
|
+
# ----------------------------------------------------------------------------------------------
|
|
263
|
+
# Artifact / dependency subgraph (spec 2026-08-27, Task 6)
|
|
264
|
+
# ----------------------------------------------------------------------------------------------
|
|
265
|
+
|
|
266
|
+
_LOCK_BASENAMES = ("poetry.lock", "uv.lock", "Pipfile.lock")
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _import_ghost(b: RowBuilder, app_can_id: str, name: str) -> NodeRef:
|
|
270
|
+
"""A ``:PyExternal`` ghost for a bare imported module name (``PY_PROVIDES``'s
|
|
271
|
+
``provides_imports`` entries, ``PY_UNRESOLVED_IMPORT``'s ``module``).
|
|
272
|
+
|
|
273
|
+
``app.external_symbols`` only homes call-graph endpoints (``_home_external_
|
|
274
|
+
symbols`` walks ``app.call_graph``), so a module that is imported but never
|
|
275
|
+
called — the overwhelmingly common case for ``provides_imports`` and the
|
|
276
|
+
*only* case for an unresolved import — has no existing ghost to MERGE onto.
|
|
277
|
+
This builds one with the same id shape ``_call_endpoint``/``_home_external_
|
|
278
|
+
symbols`` use for a dot-less (no ``.`` in the signature) call target:
|
|
279
|
+
``<app can:// id>/@external/<name>``, ``module=None``. Same two-label
|
|
280
|
+
``["PySymbol", "PyExternal"]`` idiom as ``_call_endpoint`` -- the schema
|
|
281
|
+
declares :PyExternal's merge label as PySymbol, and RowBuilder MERGEs by
|
|
282
|
+
``(labels[0], value)``, so if a call to that same bare name is ever
|
|
283
|
+
projected too, both rows collapse onto this one node — correctly, since
|
|
284
|
+
they name the same real-world symbol."""
|
|
285
|
+
return b.node(
|
|
286
|
+
["PySymbol", "PyExternal"], "id", f"{app_can_id}/@external/{name}", {"name": name}
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _project_artifacts(b: RowBuilder, app: PyApplication, app_name: str, app_ref: NodeRef) -> None:
|
|
291
|
+
"""Non-code artifacts, declared dependencies and undeclared imports (Tasks
|
|
292
|
+
1-5) -- neutral ``Artifact``/``Package`` nodes with no ``Py`` prefix
|
|
293
|
+
(deliberate: cross-language merge targets, unlike everything else this
|
|
294
|
+
module projects). Always emitted regardless of ``-a`` -- this section is
|
|
295
|
+
L1 data, identical at every analysis level (mirrors ``analysis.json``)."""
|
|
296
|
+
app_can_id = application_id(app_name)
|
|
297
|
+
|
|
298
|
+
for path in sorted(app.artifacts or {}):
|
|
299
|
+
art = app.artifacts[path]
|
|
300
|
+
art_ref = b.node(
|
|
301
|
+
["Artifact"],
|
|
302
|
+
"id",
|
|
303
|
+
art.id,
|
|
304
|
+
prune(
|
|
305
|
+
{
|
|
306
|
+
"path": art.path,
|
|
307
|
+
"format": art.format,
|
|
308
|
+
"roles": art.roles,
|
|
309
|
+
"size_bytes": art.size_bytes,
|
|
310
|
+
"sha256": art.sha256,
|
|
311
|
+
"source": art.source,
|
|
312
|
+
"text_truncated": art.text_truncated,
|
|
313
|
+
"extraction": art.extraction,
|
|
314
|
+
}
|
|
315
|
+
),
|
|
316
|
+
)
|
|
317
|
+
b.edge("HAS_ARTIFACT", app_ref, art_ref)
|
|
318
|
+
|
|
319
|
+
# Config keys flattened out of this artifact (#152) -- sorted by key
|
|
320
|
+
# for deterministic row order, matching the JSON side's L1 determinism.
|
|
321
|
+
for ck in sorted(art.config_keys or [], key=lambda k: k.key):
|
|
322
|
+
ck_ref = b.node(
|
|
323
|
+
["ConfigKey"],
|
|
324
|
+
"id",
|
|
325
|
+
ck.id,
|
|
326
|
+
prune(
|
|
327
|
+
{
|
|
328
|
+
"key": ck.key,
|
|
329
|
+
"namespace": ck.namespace,
|
|
330
|
+
"value": ck.value,
|
|
331
|
+
"references": list(ck.references or []),
|
|
332
|
+
"start_line": ck.span.start[0] if ck.span else None,
|
|
333
|
+
"end_line": ck.span.end[0] if ck.span else None,
|
|
334
|
+
}
|
|
335
|
+
),
|
|
336
|
+
)
|
|
337
|
+
b.edge("DEFINES_CONFIG", art_ref, ck_ref)
|
|
338
|
+
|
|
339
|
+
# Every lock artifact present LOCKS every dependency it pinned. The pins
|
|
340
|
+
# from all lock files are already merged into one `locked_version` per
|
|
341
|
+
# dependency upstream (Task 5 `build_dependency_view`) -- there is no
|
|
342
|
+
# per-lock-file attribution to split on, so (like the JSON projection) a
|
|
343
|
+
# dependency locked with N lock artifacts present gets N LOCKS edges.
|
|
344
|
+
lock_ids = [
|
|
345
|
+
app.artifacts[p].id
|
|
346
|
+
for p in sorted(app.artifacts or {})
|
|
347
|
+
if p.rsplit("/", 1)[-1] in _LOCK_BASENAMES
|
|
348
|
+
]
|
|
349
|
+
|
|
350
|
+
# app.dependencies has one PyDependency per DECLARING MANIFEST, so a
|
|
351
|
+
# package declared in 2+ manifests (e.g. requirements.txt +
|
|
352
|
+
# requirements-dev.txt both listing "requests") walks this loop once per
|
|
353
|
+
# manifest. DECLARES_DEPENDENCY is correctly one row per declaration (its
|
|
354
|
+
# `from_ref` is the manifest, so those rows are already distinct) -- but
|
|
355
|
+
# LOCKS/PY_PROVIDES/PY_UNRESOLVED_IMPORT are per-PACKAGE facts, and
|
|
356
|
+
# RowBuilder.edge() is append-only (unlike node(), it does not MERGE-dedup)
|
|
357
|
+
# -- so without a guard they'd be emitted once per declaring manifest
|
|
358
|
+
# instead of once, violating GraphRows' documented deduped-bag contract.
|
|
359
|
+
seen: set = set()
|
|
360
|
+
|
|
361
|
+
for d in app.dependencies or []:
|
|
362
|
+
pkg_id = purl_pypi(d.name)
|
|
363
|
+
pkg_ref = b.node(["Package"], "id", pkg_id, {"ecosystem": "pypi", "name": d.name})
|
|
364
|
+
# kind-discriminated: the same manifest may declare one package twice
|
|
365
|
+
# under different kinds (e.g. requests in [project.dependencies] AND
|
|
366
|
+
# again under [project.optional-dependencies]) -- same endpoint pair,
|
|
367
|
+
# so a plain MERGE would collapse the two declarations into one row.
|
|
368
|
+
b.edge(
|
|
369
|
+
"DECLARES_DEPENDENCY",
|
|
370
|
+
NodeRef("Artifact", "id", d.declared_in),
|
|
371
|
+
pkg_ref,
|
|
372
|
+
prune({"spec": d.spec, "kind": d.kind, "extras": d.extras, "prov": d.prov, "direct": d.direct}),
|
|
373
|
+
key=d.kind,
|
|
374
|
+
)
|
|
375
|
+
if d.locked_version:
|
|
376
|
+
for lock_id in lock_ids:
|
|
377
|
+
key = ("LOCKS", lock_id, pkg_id)
|
|
378
|
+
if key not in seen:
|
|
379
|
+
seen.add(key)
|
|
380
|
+
b.edge(
|
|
381
|
+
"LOCKS",
|
|
382
|
+
NodeRef("Artifact", "id", lock_id),
|
|
383
|
+
pkg_ref,
|
|
384
|
+
{"version": d.locked_version},
|
|
385
|
+
)
|
|
386
|
+
for top in d.provides_imports:
|
|
387
|
+
ghost_ref = _import_ghost(b, app_can_id, top)
|
|
388
|
+
key = ("PY_PROVIDES", pkg_id, ghost_ref.value)
|
|
389
|
+
if key not in seen:
|
|
390
|
+
seen.add(key)
|
|
391
|
+
b.edge("PY_PROVIDES", pkg_ref, ghost_ref)
|
|
392
|
+
|
|
393
|
+
for u in app.unresolved_imports or []:
|
|
394
|
+
ghost_ref = _import_ghost(b, app_can_id, u.module)
|
|
395
|
+
key = ("PY_UNRESOLVED_IMPORT", app_ref.value, ghost_ref.value)
|
|
396
|
+
if key not in seen:
|
|
397
|
+
seen.add(key)
|
|
398
|
+
b.edge("PY_UNRESOLVED_IMPORT", app_ref, ghost_ref, prune({"prov": u.prov}))
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def _project_config_uses(
|
|
402
|
+
b: RowBuilder, app: PyApplication, app_ref: NodeRef, externals: dict, sig_to_id: dict,
|
|
403
|
+
) -> None:
|
|
404
|
+
"""config_use (#162): PY_USES_CONFIG (`app.config_uses`) and
|
|
405
|
+
PY_READS_CONFIG_UNRESOLVED (`app.config_reads_unresolved`).
|
|
406
|
+
|
|
407
|
+
`PyConfigUseEdge.src`/`.dst` are already a GLOBAL ordinal id and a
|
|
408
|
+
ConfigKey id (both resolved upstream by `resolve_uses`), so — like
|
|
409
|
+
`param_in`/`param_out` — they address existing PyBodyNode/ConfigKey rows
|
|
410
|
+
directly with a plain :class:`NodeRef`; no defer-and-gate needed. Call
|
|
411
|
+
nodes enter a callable's `body` at L1 (before any config_use tier runs),
|
|
412
|
+
so the src PyBodyNode is always already projected by
|
|
413
|
+
`_project_program_graphs`, whatever level this ran at.
|
|
414
|
+
|
|
415
|
+
`PyConfigRead.callee` is already the full external `can://…/@external/…`
|
|
416
|
+
id (not a bare module name), so its ghost goes through `_call_endpoint`
|
|
417
|
+
(which looks it up in `externals` directly) rather than `_import_ghost`
|
|
418
|
+
(built for a bare imported name) — same inline node()+edge() shape
|
|
419
|
+
`PY_UNRESOLVED_IMPORT` uses. `_k` discriminates by (key, reason): the same
|
|
420
|
+
external callee (e.g. `os.getenv`) legitimately reads several distinct
|
|
421
|
+
undeclared/dynamic keys across a codebase, and without a discriminant a
|
|
422
|
+
plain endpoint-pair MERGE would collapse those onto one relationship.
|
|
423
|
+
"""
|
|
424
|
+
for e in app.config_uses:
|
|
425
|
+
b.edge(
|
|
426
|
+
"PY_USES_CONFIG",
|
|
427
|
+
NodeRef("PyBodyNode", "id", e.src),
|
|
428
|
+
NodeRef("ConfigKey", "id", e.dst),
|
|
429
|
+
prune({"prov": list(e.prov) if e.prov else None}),
|
|
430
|
+
)
|
|
431
|
+
for r in app.config_reads_unresolved:
|
|
432
|
+
ghost_ref = _call_endpoint(b, r.callee, externals, sig_to_id)
|
|
433
|
+
b.edge(
|
|
434
|
+
"PY_READS_CONFIG_UNRESOLVED",
|
|
435
|
+
app_ref,
|
|
436
|
+
ghost_ref,
|
|
437
|
+
prune({"key": r.key, "reason": r.reason, "prov": list(r.prov) if r.prov else None}),
|
|
438
|
+
# _k=(key,reason) does not per-site discriminate the non-literal bucket (accepted prop-list ceiling).
|
|
439
|
+
key=f"{r.key or ''}|{r.reason}",
|
|
440
|
+
)
|
|
441
|
+
|
|
442
|
+
|
|
251
443
|
def _sym(can_id: str) -> NodeRef:
|
|
252
444
|
return NodeRef("PySymbol", "id", can_id)
|
|
253
445
|
|
codeanalyzer/neo4j/schema.py
CHANGED
|
@@ -201,6 +201,30 @@ NODE_LABELS: List[NodeLabel] = [
|
|
|
201
201
|
"_module": "string",
|
|
202
202
|
},
|
|
203
203
|
),
|
|
204
|
+
# Neutral artifact/dependency subgraph (spec 2026-08-27, Task 6). No `Py`
|
|
205
|
+
# prefix -- deliberate: `Artifact`/`Package` are cross-language merge
|
|
206
|
+
# targets, so a sibling-language analyzer over the same repo lands on the
|
|
207
|
+
# same nodes instead of a per-language duplicate. `PY_PROVIDES` /
|
|
208
|
+
# `PY_UNRESOLVED_IMPORT` stay PY_-namespaced (this analyzer's own claim
|
|
209
|
+
# about what an import resolves to) and target `:PyExternal`.
|
|
210
|
+
NodeLabel("Artifact", "Artifact", "id", {
|
|
211
|
+
"id": "string", "path": "string", "format": "string",
|
|
212
|
+
"roles": "string[]", "size_bytes": "integer", "sha256": "string",
|
|
213
|
+
"source": "string", "text_truncated": "boolean", "extraction": "string",
|
|
214
|
+
}),
|
|
215
|
+
NodeLabel("Package", "Package", "id", {
|
|
216
|
+
"id": "string", "ecosystem": "string", "name": "string",
|
|
217
|
+
}),
|
|
218
|
+
# A configuration key flattened out of a config-bearing Artifact (#152).
|
|
219
|
+
# Neutral vocabulary like Artifact/Package -- a yaml/env/ini key is not a
|
|
220
|
+
# Python concept. `value` is omitted (not null) when the source model's
|
|
221
|
+
# value is None (--no-artifact-text, or a namespace with no value at that
|
|
222
|
+
# path); `references` is always present, possibly empty.
|
|
223
|
+
NodeLabel("ConfigKey", "ConfigKey", "id", {
|
|
224
|
+
"id": "string", "key": "string", "namespace": "string",
|
|
225
|
+
"value": "string", "references": "string[]",
|
|
226
|
+
**_SPAN,
|
|
227
|
+
}),
|
|
204
228
|
]
|
|
205
229
|
|
|
206
230
|
_DECL_TARGETS = ["PyClass", "PyCallable"]
|
|
@@ -250,6 +274,39 @@ REL_TYPES: List[RelType] = [
|
|
|
250
274
|
RelType("PY_PARAM_IN", ["PyBodyNode"], ["PyBodyNode"], {"var": "string"}),
|
|
251
275
|
RelType("PY_PARAM_OUT", ["PyBodyNode"], ["PyBodyNode"], {"var": "string"}),
|
|
252
276
|
RelType("PY_SUMMARY", ["PyBodyNode"], ["PyBodyNode"]),
|
|
277
|
+
# Neutral artifact/dependency subgraph (Task 6).
|
|
278
|
+
RelType("HAS_ARTIFACT", ["PyApplication"], ["Artifact"]),
|
|
279
|
+
# A config key nests under exactly one owning artifact (its id is
|
|
280
|
+
# `<artifact-id>@key/<dotted.key>`) -- a plain containment edge, no
|
|
281
|
+
# per-edge properties or discriminant needed (#152).
|
|
282
|
+
RelType("DEFINES_CONFIG", ["Artifact"], ["ConfigKey"]),
|
|
283
|
+
# ``_k`` (merges per ``kind``): the same manifest may declare one package
|
|
284
|
+
# twice under different kinds (e.g. a runtime dep re-listed under an
|
|
285
|
+
# optional extra) -- same endpoint pair, so without the discriminant the
|
|
286
|
+
# plain MERGE collapses the two declarations into one row.
|
|
287
|
+
RelType("DECLARES_DEPENDENCY", ["Artifact"], ["Package"], {
|
|
288
|
+
"spec": "string", "kind": "string", "extras": "string[]", "prov": "string[]",
|
|
289
|
+
"direct": "boolean", "_k": "string",
|
|
290
|
+
}),
|
|
291
|
+
RelType("LOCKS", ["Artifact"], ["Package"], {"version": "string"}),
|
|
292
|
+
RelType("PY_PROVIDES", ["Package"], ["PyExternal"]),
|
|
293
|
+
RelType("PY_UNRESOLVED_IMPORT", ["PyApplication"], ["PyExternal"], {"prov": "string[]"}),
|
|
294
|
+
# config_use (#162): the resolved-read bridge from a call site's body node
|
|
295
|
+
# to the PyConfigKey it reads. `src`/`dst` are already GLOBAL ordinal /
|
|
296
|
+
# ConfigKey ids (resolved upstream by `resolve_uses`), so no discriminant
|
|
297
|
+
# is needed -- one call site reads one key per edge.
|
|
298
|
+
RelType("PY_USES_CONFIG", ["PyBodyNode"], ["ConfigKey"], {"prov": "string[]"}),
|
|
299
|
+
# A detector-matched read that never closed on exactly one declared key --
|
|
300
|
+
# first-class per #162, PyApplication -> PyExternal ghost of the callee
|
|
301
|
+
# (mirrors PY_UNRESOLVED_IMPORT's shape). `_k` discriminates by (key,
|
|
302
|
+
# reason): the same external callee (e.g. `os.getenv`) legitimately reads
|
|
303
|
+
# several distinct undeclared/dynamic keys across a codebase -- without a
|
|
304
|
+
# discriminant a plain endpoint-pair MERGE would collapse those onto one
|
|
305
|
+
# relationship and silently drop every key but the last one SET.
|
|
306
|
+
RelType(
|
|
307
|
+
"PY_READS_CONFIG_UNRESOLVED", ["PyApplication"], ["PyExternal"],
|
|
308
|
+
{"key": "string", "reason": "string", "prov": "string[]", "_k": "string"},
|
|
309
|
+
),
|
|
253
310
|
]
|
|
254
311
|
|
|
255
312
|
|
codeanalyzer/options/options.py
CHANGED
|
@@ -37,8 +37,13 @@ class AnalysisOptions:
|
|
|
37
37
|
rebuild_analysis: bool = False
|
|
38
38
|
skip_tests: bool = True
|
|
39
39
|
no_venv: bool = False
|
|
40
|
+
resolve_installed: bool = False
|
|
40
41
|
file_name: Optional[Path] = None
|
|
41
42
|
cache_dir: Optional[Path] = None
|
|
42
43
|
clear_cache: bool = False
|
|
43
44
|
verbosity: int = 0
|
|
44
45
|
entrypoint_rules: Tuple[Path, ...] = ()
|
|
46
|
+
# Artifact text-capture controls (#157 follow-up): whether to capture
|
|
47
|
+
# `source` at all, and the per-file byte cap before it truncates.
|
|
48
|
+
artifact_text: bool = True
|
|
49
|
+
artifact_text_max_bytes: int = 262144
|
codeanalyzer/schema/ids.py
CHANGED
|
@@ -21,3 +21,24 @@ def callable_sig_segment(name: str, param_names: List[str]) -> str:
|
|
|
21
21
|
|
|
22
22
|
def ordinal_id(callable_id: str, tag: str) -> str:
|
|
23
23
|
return f"{callable_id}@{tag}"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def artifact_id(app_name: str, rel_path: str) -> str:
|
|
27
|
+
"""Language-neutral artifact id: ``can://artifact/<app>/<rel-path>``.
|
|
28
|
+
|
|
29
|
+
The first segment is a namespace (a language for code nodes, the literal
|
|
30
|
+
``artifact`` for files), so sibling analyzers over the same repo emit the
|
|
31
|
+
same id for the same file."""
|
|
32
|
+
return f"can://artifact/{app_name}/{rel_path}"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def config_key_id(artifact_id: str, dotted_key: str) -> str:
|
|
36
|
+
"""A ``PyConfigKey`` extracted from an artifact: ``<artifact-id>@key/<dotted.key>``.
|
|
37
|
+
``dotted_key`` uses numeric segments for array indices (e.g.
|
|
38
|
+
``services.web.ports.0``); ids are opaque, do not re-split them."""
|
|
39
|
+
return f"{artifact_id}@key/{dotted_key}"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def purl_pypi(name: str) -> str:
|
|
43
|
+
"""Package URL for a (PEP 503 normalized) PyPI distribution name."""
|
|
44
|
+
return f"pkg:pypi/{name}"
|
codeanalyzer/schema/py_schema.py
CHANGED
|
@@ -306,6 +306,17 @@ class PyCallArgument(BaseModel):
|
|
|
306
306
|
|
|
307
307
|
ast_kind: str
|
|
308
308
|
inferred_type: Optional[str] = None
|
|
309
|
+
# Literal capture (#162): populated only for an `ast.Constant` argument
|
|
310
|
+
# whose value is str/int/float/bool/None, JSON-encoded (`json.dumps`) --
|
|
311
|
+
# decode with `json.loads` to recover the Python constant. `None` for
|
|
312
|
+
# every non-constant argument (and for a Constant of another type, e.g.
|
|
313
|
+
# bytes/complex/Ellipsis).
|
|
314
|
+
value: Optional[str] = None
|
|
315
|
+
# Bare-identifier capture (#162): the `id` of an `ast.Name` argument
|
|
316
|
+
# (e.g. `f(KEY)` -> `"KEY"`) -- ships alongside `value` as the dataflow
|
|
317
|
+
# tier's join point back to the variable's own definitions. `None` for
|
|
318
|
+
# every other argument shape.
|
|
319
|
+
name: Optional[str] = None
|
|
309
320
|
|
|
310
321
|
|
|
311
322
|
# BodyNode.arguments forward-references PyCallArgument (defined later);
|
|
@@ -470,6 +481,104 @@ class PyExternalSymbol(BaseModel):
|
|
|
470
481
|
module: Optional[str] = None # best-effort owning module, e.g. "requests"
|
|
471
482
|
|
|
472
483
|
|
|
484
|
+
@builder
|
|
485
|
+
class PyConfigKey(BaseModel):
|
|
486
|
+
"""A configuration key flattened out of a config-bearing ``PyArtifact``
|
|
487
|
+
(#152). Graph vocabulary stays neutral (label ``ConfigKey``, edge
|
|
488
|
+
``DEFINES_CONFIG``) -- the ``Py`` prefix here is only the ``PyArtifact``
|
|
489
|
+
naming precedent, not a Python-specific claim. L1 data, identical at
|
|
490
|
+
every analysis level; nested under the owning artifact, containment
|
|
491
|
+
mirrors ``DEFINES_CONFIG``."""
|
|
492
|
+
|
|
493
|
+
id: str = "" # <artifact-id>@key/<dotted.key>
|
|
494
|
+
key: str # dotted path; numeric segments for arrays, e.g. "services.web.ports.0"
|
|
495
|
+
namespace: str # env|yaml|json|toml|ini|properties|dockerfile
|
|
496
|
+
value: Optional[str] = None # populated only when options.artifact_text is on
|
|
497
|
+
span: Optional[Span] = None # into the artifact's source; best-effort for yaml/json/toml
|
|
498
|
+
references: List[str] = [] # raw recognized tokens, order of appearance, deduplicated
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
@builder
|
|
502
|
+
class PyArtifact(BaseModel):
|
|
503
|
+
"""Any non-`.py` project file (config, manifest, CI, container spec, or
|
|
504
|
+
plain data/binary) -- never dropped from the walk. Captured broadly (node
|
|
505
|
+
+ verbatim ``source``); *meaning* is extracted narrowly -- only
|
|
506
|
+
``dependency-manifest`` roles feed ``dependencies`` today. ``id`` is
|
|
507
|
+
language-neutral (``can://artifact/<app>/<path>``)."""
|
|
508
|
+
|
|
509
|
+
id: str = ""
|
|
510
|
+
kind: str = "artifact"
|
|
511
|
+
path: str # repo-relative POSIX path (also the map key)
|
|
512
|
+
format: str # toml|yaml|json|ini|properties|requirements|dockerfile|text|binary
|
|
513
|
+
roles: List[str] = []
|
|
514
|
+
size_bytes: int = 0
|
|
515
|
+
sha256: str = "" # always the full file's hash, even when source is truncated/empty
|
|
516
|
+
source: str = "" # verbatim by default; "" for binary or when capture is disabled
|
|
517
|
+
text_truncated: bool = False # True when `source` is a prefix, not the full file
|
|
518
|
+
extraction: str = "none" # none|partial|full
|
|
519
|
+
config_keys: List[PyConfigKey] = [] # flattened config keys (#152); [] when not namespace-eligible
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
@builder
|
|
523
|
+
class PyConfigUseEdge(BaseModel):
|
|
524
|
+
"""One resolved config read (#162): a detector-matched call's key
|
|
525
|
+
argument closed on exactly one string literal that matches a declared
|
|
526
|
+
``PyConfigKey``. ``src`` is the call's GLOBAL ordinal id
|
|
527
|
+
(``<callable-id>@<local-id>``); ``dst`` is the matched ``PyConfigKey.id``
|
|
528
|
+
-- application scope, mirroring ``param_in`` (endpoints span callables/
|
|
529
|
+
artifacts). Superset-monotonic across levels, same additive contract as
|
|
530
|
+
the DDG's ``prov`` widening: literal (``-a 2``+) subset of +dataflow
|
|
531
|
+
(``-a 3``/``-a 4``)."""
|
|
532
|
+
|
|
533
|
+
src: str
|
|
534
|
+
dst: str
|
|
535
|
+
prov: List[Literal["literal", "dataflow"]] = []
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
@builder
|
|
539
|
+
class PyConfigRead(BaseModel):
|
|
540
|
+
"""A detector-matched call whose key did not close on exactly one string
|
|
541
|
+
literal -- first-class so a config read nobody can trace is as visible
|
|
542
|
+
as one that resolves (#162). ``key`` is the decoded literal text only
|
|
543
|
+
when it IS a literal but matches no declared ``PyConfigKey``
|
|
544
|
+
(``reason="undefined-key"``); ``None`` for a key that never closed on a
|
|
545
|
+
literal at all (``reason="non-literal"``). ``prov`` lists every tier
|
|
546
|
+
that was attempted before giving up."""
|
|
547
|
+
|
|
548
|
+
site: str # GLOBAL ordinal id
|
|
549
|
+
callee: str # external id (can://.../@external/<module>/<name>)
|
|
550
|
+
key: Optional[str] = None
|
|
551
|
+
reason: Literal["non-literal", "undefined-key"]
|
|
552
|
+
prov: List[Literal["literal", "dataflow"]] = []
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
@builder
|
|
556
|
+
class PyDependency(BaseModel):
|
|
557
|
+
"""One declared third-party dependency, evidence-tagged via ``prov``."""
|
|
558
|
+
|
|
559
|
+
name: str # PEP 503 normalized
|
|
560
|
+
ecosystem: str = "pypi" # SDK symmetry with purl (#152 rider); the only ecosystem this analyzer emits
|
|
561
|
+
spec: str = ""
|
|
562
|
+
kind: str = "runtime" # runtime|dev|optional|build
|
|
563
|
+
extras: List[str] = []
|
|
564
|
+
declared_in: str = "" # PyArtifact id
|
|
565
|
+
# False for lockfile-only (transitive) dependencies -- pinned in a lock
|
|
566
|
+
# with no manifest declaration (#152 reconciliation).
|
|
567
|
+
direct: bool = True
|
|
568
|
+
locked_version: Optional[str] = None
|
|
569
|
+
provides_imports: List[str] = []
|
|
570
|
+
prov: List[str] = [] # declared|lockfile|installed-metadata|heuristic
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
@builder
|
|
574
|
+
class PyImportBinding(BaseModel):
|
|
575
|
+
"""A top-level import no declared dependency accounts for."""
|
|
576
|
+
|
|
577
|
+
module: str
|
|
578
|
+
bound_to: Optional[str] = None # best-effort distribution name
|
|
579
|
+
prov: List[str] = []
|
|
580
|
+
|
|
581
|
+
|
|
473
582
|
@builder
|
|
474
583
|
class PyRepositoryInfo(BaseModel):
|
|
475
584
|
"""Where the analyzed source came from: git provenance captured at analysis time."""
|
|
@@ -502,6 +611,11 @@ class PyApplication(BaseModel):
|
|
|
502
611
|
# builtin members), keyed by signature. Populated by the analyzer so every
|
|
503
612
|
# backend (JSON and Neo4j) shares one authoritative external-symbol set.
|
|
504
613
|
external_symbols: Dict[str, PyExternalSymbol] = {}
|
|
614
|
+
# Non-code artifacts, declared dependencies, and undeclared imports
|
|
615
|
+
# (spec 2026-08-27). L1 data: identical at every analysis level.
|
|
616
|
+
artifacts: Dict[str, PyArtifact] = {}
|
|
617
|
+
dependencies: List[PyDependency] = []
|
|
618
|
+
unresolved_imports: List[PyImportBinding] = []
|
|
505
619
|
# Coverage/failure record for the entrypoint pass; see PyEntrypointReport (#27).
|
|
506
620
|
entrypoint_report: PyEntrypointReport = PyEntrypointReport()
|
|
507
621
|
# Git provenance of the analyzed checkout, captured at analysis time.
|
|
@@ -509,6 +623,10 @@ class PyApplication(BaseModel):
|
|
|
509
623
|
# Interprocedural parameter-passing edges (formal↔actual); populated at L4.
|
|
510
624
|
param_in: List[ParamEdge] = []
|
|
511
625
|
param_out: List[ParamEdge] = []
|
|
626
|
+
# config_use (#162): PY_USES_CONFIG edges + first-class unresolved reads.
|
|
627
|
+
# Literal tier from L2; dataflow tiers widen the set at L3/L4 (additive).
|
|
628
|
+
config_uses: List[PyConfigUseEdge] = []
|
|
629
|
+
config_reads_unresolved: List[PyConfigRead] = []
|
|
512
630
|
|
|
513
631
|
|
|
514
632
|
@builder
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import ast
|
|
2
2
|
import hashlib
|
|
3
|
+
import json
|
|
3
4
|
import os
|
|
4
5
|
import tokenize
|
|
5
6
|
from ast import AST, ClassDef
|
|
@@ -786,6 +787,16 @@ class SymbolTableBuilder:
|
|
|
786
787
|
PyCallArgument(
|
|
787
788
|
ast_kind=type(arg).__name__,
|
|
788
789
|
inferred_type=self._infer_type(script, arg.lineno, arg.col_offset),
|
|
790
|
+
# Literal/name capture (#162): value JSON-encoded for a
|
|
791
|
+
# Constant of a JSON-safe type; name for a bare Name --
|
|
792
|
+
# never both, per the AST shapes involved.
|
|
793
|
+
value=(
|
|
794
|
+
json.dumps(arg.value)
|
|
795
|
+
if isinstance(arg, ast.Constant)
|
|
796
|
+
and isinstance(arg.value, (str, int, float, bool, type(None)))
|
|
797
|
+
else None
|
|
798
|
+
),
|
|
799
|
+
name=arg.id if isinstance(arg, ast.Name) else None,
|
|
789
800
|
)
|
|
790
801
|
for arg in node.args
|
|
791
802
|
]
|