codeanalyzer-python 0.3.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

codeanalyzer/__main__.py

@@ -1,9 +1,40 @@
+import os
+import sys
 from importlib.metadata import version as _pkg_version, PackageNotFoundError
 from pathlib import Path
 from typing import Optional, Annotated
 
 import typer
 
+
+def _pin_hash_seed() -> None:
+    """Re-exec once with ``PYTHONHASHSEED=0`` unless the caller pinned one.
+
+    PyCG's capped fixpoint (``--pycg-max-iter``) iterates hash-ordered sets
+    keyed on module/access-path strings, so an unpinned per-interpreter hash
+    seed makes the emitted L2+ call graph vary run to run (issue #99). The
+    seed cannot be set after interpreter start, hence the exec. Export
+    PYTHONHASHSEED (any value) to opt out or pin a different seed.
+
+    Only fires when this process really is the CLI (canpy / python -m
+    codeanalyzer): in-process invocations — e.g. Typer's CliRunner in the
+    test suite, or a host app calling the callback — must never have their
+    own process exec'd out from under them."""
+    if os.environ.get("PYTHONHASHSEED") is not None:
+        return
+    argv0 = os.path.basename(sys.argv[0]) if sys.argv else ""
+    is_cli = argv0 in ("canpy", "codeanalyzer") or sys.argv[0].endswith(
+        os.path.join("codeanalyzer", "__main__.py")
+    )
+    if not is_cli:
+        return
+    env = dict(os.environ, PYTHONHASHSEED="0")
+    os.execvpe(
+        sys.executable,
+        [sys.executable, "-m", "codeanalyzer", *sys.argv[1:]],
+        env,
+    )
+
 from codeanalyzer.core import Codeanalyzer
 from codeanalyzer.utils import _set_log_level, logger
 from codeanalyzer.config import OutputFormat
@@ -114,11 +145,32 @@ def main(
         typer.Option(
             "-a",
             "--analysis-level",
-            help="Analysis depth: 1=symbol table+Jedi call graph, 2=+PyCG call graph.",
+            help="Analysis depth: 1=symbol table+Jedi call graph, 2=+PyCG call "
+            "graph, 3=+native intraprocedural dataflow (CFG/PDG), "
+            "4=+interprocedural SDG (param/summary edges, alias-aware DDG).",
             min=1,
-            max=2,
+            max=4,
         ),
     ] = 1,
+    graphs: Annotated[
+        str,
+        typer.Option(
+            "--graphs",
+            help="Level 3+ only: comma-separated program-graph sections to emit "
+            "(cfg, dfg, pdg, sdg). Default: cfg,dfg,pdg. `dfg` emits the PDG's data "
+            "edges only; `sdg` requires -a 4.",
+        ),
+    ] = "cfg,dfg,pdg",
+    graph_field_depth: Annotated[
+        int,
+        typer.Option(
+            "--graph-field-depth",
+            help="Level 3 only: k-limit on access-path depth (x.f.g.h with "
+            "k=3 becomes x.f.g.*). Mandatory bound — it is what guarantees "
+            "the interprocedural fixpoint terminates.",
+            min=1,
+        ),
+    ] = 3,
     using_ray: Annotated[
         bool,
         typer.Option("--ray/--no-ray", help="Enable Ray for distributed analysis."),
@@ -243,6 +295,34 @@ def main(
         ),
     ] = 50,
 ):
+    # Determinism: pin the interpreter hash seed before any analysis (no-op
+    # when PYTHONHASHSEED is already set; --version exits before this).
+    _pin_hash_seed()
+
+    # Flag validation (strict: unrecognized values error out, never fall back).
+    selected_graphs = [g.strip() for g in graphs.split(",") if g.strip()]
+    from codeanalyzer.dataflow.builder import VALID_GRAPHS
+
+    unknown_graphs = [g for g in selected_graphs if g not in VALID_GRAPHS]
+    if unknown_graphs:
+        logger.error(
+            f"Unrecognized --graphs value(s): {', '.join(unknown_graphs)} "
+            f"(valid: {', '.join(VALID_GRAPHS)})."
+        )
+        raise typer.Exit(code=2)
+    if not selected_graphs:
+        logger.error("--graphs requires at least one of: " + ", ".join(VALID_GRAPHS))
+        raise typer.Exit(code=2)
+    if "sdg" in selected_graphs and analysis_level < 4:
+        logger.error("--graphs sdg requires -a 4 (interprocedural SDG).")
+        raise typer.Exit(code=2)
+    if analysis_level < 3 and graphs != "cfg,dfg,pdg":
+        logger.error("--graphs is a level-3 option; pass -a 3 to emit program graphs.")
+        raise typer.Exit(code=2)
+    if analysis_level < 3 and graph_field_depth != 3:
+        logger.error("--graph-field-depth is a level-3 option; pass -a 3.")
+        raise typer.Exit(code=2)
+
     options = AnalysisOptions(
         input=input,
         output=output,
@@ -254,6 +334,8 @@ def main(
         neo4j_password=neo4j_password,
         neo4j_database=neo4j_database,
         analysis_level=analysis_level,
+        graphs=",".join(selected_graphs),
+        graph_field_depth=graph_field_depth,
         using_ray=using_ray,
         rebuild_analysis=rebuild_analysis,
         skip_tests=skip_tests,
@@ -310,7 +392,7 @@ def main(
 
             emit_neo4j(artifacts, options)
         elif options.output is None:
-            print(model_dump_json(artifacts, separators=(",", ":")))
+            print(model_dump_json(artifacts, exclude_none=True))
         else:
             options.output.mkdir(parents=True, exist_ok=True)
             _write_output(artifacts, options.output, options.format)
@@ -321,7 +403,7 @@ def _write_output(artifacts, output_dir: Path, format: OutputFormat):
     if format == OutputFormat.JSON:
         output_file = output_dir / "analysis.json"
         # Use Pydantic's model_dump_json() for compact output
-        json_str = model_dump_json(artifacts, indent=None)
+        json_str = model_dump_json(artifacts, indent=None, exclude_none=True)
         with output_file.open("w") as f:
             f.write(json_str)
         logger.info(f"Analysis saved to {output_file}")

codeanalyzer/core.py

@@ -11,12 +11,17 @@ import time
 import ray
 from codeanalyzer.utils import logger
 from codeanalyzer.schema import (
+    Analysis,
     PyApplication,
     PyExternalSymbol,
     PyModule,
     model_dump_json,
     model_validate_json,
 )
+from codeanalyzer.schema.assign_ids import assign_ids
+from codeanalyzer.schema.l1_body import populate_l1_body
+from codeanalyzer.schema.l2_callees import backfill_callees
+from codeanalyzer.schema.call_graph_ids import reidentify_call_graph
 from codeanalyzer.schema.py_schema import PyCallEdge
 from codeanalyzer.semantic_analysis.call_graph import (
     filter_external_edges,
@@ -32,6 +37,22 @@ from codeanalyzer.utils import ProgressBar
 from codeanalyzer.options import AnalysisOptions
 from codeanalyzer.provenance import analyzer_info, repository_info
 
+def _ensure_ray() -> None:
+    """Initialize Ray with the driver's pinned hash seed in the workers.
+
+    An implicit auto-init would not carry PYTHONHASHSEED into worker
+    interpreters, so PyCG shards (and Jedi inference) run there with random
+    set-iteration order and the emitted edges vary run to run (issue #99)."""
+    if not ray.is_initialized():
+        ray.init(
+            runtime_env={
+                "env_vars": {
+                    "PYTHONHASHSEED": os.environ.get("PYTHONHASHSEED", "0")
+                }
+            },
+        )
+
+
 @ray.remote
 def _process_file_with_ray(py_file: Union[Path, str], project_dir: Union[Path, str], virtualenv: Union[Path, str, None]) -> Dict[str, PyModule]:
     """Processes files in the project directory using Ray for distributed processing.
@@ -49,7 +70,7 @@ def _process_file_with_ray(py_file: Union[Path, str], project_dir: Union[Path, s
     try:
         py_file = Path(py_file)
         symbol_table_builder = SymbolTableBuilder(project_dir, virtualenv)
-        module_map[str(py_file)] = symbol_table_builder.build_pymodule_from_file(py_file)
+        module_map[str(py_file.relative_to(Path(project_dir)))] = symbol_table_builder.build_pymodule_from_file(py_file)
     except Exception as e:
         console.log(f"❌ Failed to process {py_file}: {e}")
         raise SymbolTableBuilderRayError(f"Ray processing error for {py_file}: {e}")
@@ -371,67 +392,52 @@ class Codeanalyzer:
             shutil.rmtree(self.cache_dir)
 
     @staticmethod
-    def _compute_external_symbols(symbol_table, call_graph):
-        """Build the external-symbol map: every call-graph endpoint whose signature
-        is not a declared class/callable in the symbol table is an external (an
-        imported library or builtin member). ``name``/``module`` are derived from
-        the signature (best effort: split on the last dot)."""
-        declared = set()
-
-        def walk_callable(c):
-            declared.add(c.signature)
-            for ic in (c.inner_callables or {}).values():
-                walk_callable(ic)
-            for cl in (c.inner_classes or {}).values():
-                walk_class(cl)
-
-        def walk_class(cl):
-            declared.add(cl.signature)
-            for m in (cl.methods or {}).values():
-                walk_callable(m)
-            for ic in (cl.inner_classes or {}).values():
-                walk_class(ic)
-
-        for mod in symbol_table.values():
-            for c in (mod.functions or {}).values():
-                walk_callable(c)
-            for cl in (mod.classes or {}).values():
-                walk_class(cl)
-
+    def _home_external_symbols(app, app_id, sig_to_id):
+        """Home every call-graph endpoint that is not a declared class/callable
+        onto a ``can://…/@external/<module>/<name>`` id (the keystone edge-endpoint
+        id home). Registers each homed id in ``sig_to_id`` so callee backfill and
+        call-graph re-identity map the dotted signature to it, and returns the
+        id-keyed external-symbol map. ``name``/``module`` are derived from the
+        signature (best effort: split on the last dot)."""
         externals: Dict[str, PyExternalSymbol] = {}
-        for edge in call_graph:
-            for sig in (edge.source, edge.target):
-                if sig in declared or sig in externals:
+        for edge in app.call_graph:
+            for sig in (edge.src, edge.dst):
+                if sig in sig_to_id:
                     continue
-                module, name = sig.rsplit(".", 1) if "." in sig else (sig, sig)
-                externals[sig] = PyExternalSymbol(name=name, module=module)
+                module, name = sig.rsplit(".", 1) if "." in sig else (None, sig)
+                ext_id = f"{app_id}/@external/{module}/{name}" if module else \
+                    f"{app_id}/@external/{name}"
+                sig_to_id[sig] = ext_id
+                externals[ext_id] = PyExternalSymbol(
+                    id=ext_id, name=name, module=module
+                )
         return externals
 
-    def analyze(self) -> PyApplication:
-        """Analyze the project and return a PyApplication with symbol table.
-        
+    def analyze(self) -> Analysis:
+        """Analyze the project and return the v2 ``Analysis`` envelope.
+
         Uses caching to avoid re-analyzing unchanged files.
         """
         cache_file = self.cache_dir / "analysis_cache.json"
 
         # Try to load existing cached analysis
-        cached_pyapplication = None
+        cached = None
         if not self.rebuild_analysis and cache_file.exists():
             try:
-                cached_pyapplication = self._load_pyapplication_from_cache(cache_file)
-                logger.info("Loaded cached analysis")
+                cached = self._load_pyapplication_from_cache(cache_file)
+                if cached is not None:
+                    logger.info("Loaded cached analysis")
             except Exception as e:
                 logger.warning(f"Failed to load cache: {e}. Rebuilding analysis.")
-                cached_pyapplication = None
+                cached = None
 
-        if cached_pyapplication is not None and not self._cache_analyzer_matches(
-            cached_pyapplication, analyzer_info(self.analysis_level).version
-        ):
-            logger.info("Analysis cache written by a different analyzer version; rebuilding.")
-            cached_pyapplication = None
+        if not self._cache_analyzer_matches(cached, analyzer_info(self.analysis_level).version):
+            if cached is not None:
+                logger.info("Analysis cache written by a different analyzer version; rebuilding.")
+            cached = None
 
         # Build symbol table from cached application if available (if no available, the build a new one)
-        symbol_table = self._build_symbol_table(cached_pyapplication.symbol_table if cached_pyapplication else {})
+        symbol_table = self._build_symbol_table(cached.application.symbol_table if cached else {})
 
         resolve_unresolved_constructors(symbol_table)
 
@@ -448,18 +454,17 @@ class Codeanalyzer:
             call_graph = merge_edges(call_graph, pycg_edges)
 
         call_graph = filter_external_edges(call_graph, symbol_table)
-
-        # Classify call-graph endpoints that are not declared in the symbol table
-        # (imported library / builtin members) once, so the JSON and Neo4j backends
-        # share one authoritative external-symbol set.
-        external_symbols = self._compute_external_symbols(symbol_table, call_graph)
+        # Canonical edge order: backend iteration order (PyCG dicts, Counter
+        # insertion) is not a contract — sort so identical edge SETS always
+        # serialize identically (issue #99 determinism gate), and so the
+        # external-symbol homing below assigns ids in a stable order.
+        call_graph.sort(key=lambda e: (e.src, e.dst))
 
         # Recreate pyapplication
         app = (
             PyApplication.builder()
             .symbol_table(symbol_table)
             .call_graph(call_graph)
-            .external_symbols(external_symbols)
             .build()
         )
 
@@ -470,52 +475,139 @@ class Codeanalyzer:
 
         # Single choke point for provenance: every produced app (fresh symbol
         # table or reused-from-cache) passes through here before being cached
-        # or returned, so analyzer/repository always reflect *this* run/checkout
-        # even when the symbol table itself came from the on-disk cache.
-        app.analyzer = analyzer_info(self.analysis_level)
+        # or returned, so repository provenance always reflects *this* checkout
+        # even when the symbol table itself came from the on-disk cache. The
+        # analyzer identity rides the envelope below (keystone home).
         app.repository = repository_info(self.project_dir)
 
-        # Save to cache
-        self._save_analysis_cache(app, cache_file)
-        
-        return app
+        app_name = self.options.app_name or self.project_dir.name
+        sig_to_id = assign_ids(app, app_name)
+        # Home call-graph endpoints that are not declared in the symbol table
+        # (imported library / builtin members) onto @external ids once, so the
+        # JSON and Neo4j backends share one authoritative external-symbol set
+        # and every edge endpoint joins the id space (no dangling endpoints).
+        app.external_symbols = self._home_external_symbols(app, app.id, sig_to_id)
+        populate_l1_body(app)
+        if self.analysis_level >= 2:
+            backfill_callees(app, sig_to_id)
+        reidentify_call_graph(app, sig_to_id)
+
+        # L3: intraprocedural dataflow (CFG/CDG/DDG) emitted onto the v2 tree.
+        if self.analysis_level >= 3:
+            from codeanalyzer.dataflow.builder import (
+                build_function_pdgs,
+                emit_l3_body,
+            )
+            from codeanalyzer.dataflow.syntactic import SyntacticOracle
+
+            infos, _func_asts = build_function_pdgs(
+                app,
+                k=self.options.graph_field_depth,
+                oracle_factory=lambda c, fast: SyntacticOracle(),
+            )
+            emit_l3_body(app, infos, sig_to_id, set(self.options.graphs.split(",")))
+
+        # L4: interprocedural dataflow (param vertices + summary + param_in/out)
+        # layered on top of the L3 syntactic overlay (L3 ⊆ L4). Scalpel is the
+        # primary may-alias oracle, with the type-based total fallback.
+        if self.analysis_level >= 4:
+            from codeanalyzer.dataflow.builder import (
+                _base_types,
+                build_program_graphs,
+                emit_ddg_pointsto_delta,
+                emit_l4,
+            )
+            from codeanalyzer.dataflow.scalpel_oracle import make_alias_oracle
+
+            ir = build_program_graphs(
+                app,
+                k=self.options.graph_field_depth,
+                oracle_factory=lambda c, fast: make_alias_oracle(
+                    c, fast, _base_types(c)
+                ),
+            )
+            emit_l4(app, ir, sig_to_id)
+            # Semantic ddg delta: the alias-derived def-use edges the real
+            # oracle adds beyond the L3 syntactic set, tagged prov=["points-to"].
+            # ``infos`` are the syntactic (L3) PDGs from the >=3 block above.
+            emit_ddg_pointsto_delta(app, infos, ir, sig_to_id)
+
+        # Build the v2 envelope, then persist it (the cache stores the full
+        # ``Analysis`` envelope so a reused cache round-trips schema_version).
+        # k_limit is an L3+ envelope key: below the dataflow levels it stays
+        # None and exclude_none drops it from the payload.
+        analysis = Analysis(
+            max_level=self.analysis_level,
+            k_limit=self.options.graph_field_depth if self.analysis_level >= 3 else None,
+            analyzer=analyzer_info(self.analysis_level),
+            application=app,
+        )
+        self._save_analysis_cache(analysis, cache_file)
+
+        return analysis
 
     @staticmethod
-    def _cache_analyzer_matches(cached_app: Optional[PyApplication], current_version: str) -> bool:
+    def _cache_analyzer_matches(cached: Optional[Analysis], current_version: str) -> bool:
         """A cache written by another analyzer version (or before versions were
         recorded) may lack fields the current models populate — pydantic fills
-        silent defaults, which would masquerade as analyzed absence."""
+        silent defaults, which would masquerade as analyzed absence. The
+        analyzer identity lives on the envelope (keystone home)."""
         return (
-            cached_app is not None
-            and cached_app.analyzer is not None
-            and cached_app.analyzer.version == current_version
+            cached is not None
+            and cached.analyzer is not None
+            and cached.analyzer.version == current_version
         )
 
-    def _load_pyapplication_from_cache(self, cache_file: Path) -> PyApplication:
-        """Load cached analysis from file.
-        
+    def _load_pyapplication_from_cache(self, cache_file: Path) -> Optional[Analysis]:
+        """Load a cached v2 ``Analysis`` envelope from file.
+
+        A cache written by an older (v1) analyzer stored a bare
+        ``PyApplication`` with no ``schema_version``; such a payload no longer
+        validates as an ``Analysis`` (or carries the wrong ``schema_version``).
+        In that case we log and return ``None`` so the caller treats it as a
+        cache miss and rebuilds from scratch — rather than crashing.
+
         Args:
             cache_file: Path to the cache file
-            
+
         Returns:
-            PyApplication: The cached application data
+            Optional[Analysis]: The cached envelope, or ``None`` if the cache is
+            stale/incompatible and should be rebuilt.
         """
         with cache_file.open('r') as f:
             data = f.read()
-        return model_validate_json(PyApplication, data)
-    
-    def _save_analysis_cache(self, app: PyApplication, cache_file: Path) -> None:
-        """Save analysis to cache file.
-        
+        try:
+            cached = model_validate_json(Analysis, data)
+        except Exception:
+            logger.info("stale/incompatible analysis cache — rebuilding")
+            return None
+        if getattr(cached, "schema_version", None) != "2.0.0":
+            logger.info("stale/incompatible analysis cache (schema_version) — rebuilding")
+            return None
+        # The cache keys only on file hash/mtime/size, not on level, so a cache
+        # built at a different analysis_level would leak higher-level body/edge
+        # content (or omit content when the cached level is lower). Reject the
+        # mismatch and force a full rebuild at the requested level.
+        if cached.max_level != self.analysis_level:
+            logger.info(
+                f"cache built at level {cached.max_level} != requested "
+                f"{self.analysis_level} — rebuilding"
+            )
+            return None
+        return cached
+
+    def _save_analysis_cache(self, analysis: Analysis, cache_file: Path) -> None:
+        """Save the v2 ``Analysis`` envelope to the cache file.
+
         Args:
-            app: The PyApplication to cache
+            analysis: The Analysis envelope to cache
             cache_file: Path to save the cache file
         """
         # Ensure cache directory exists
         cache_file.parent.mkdir(parents=True, exist_ok=True)
-        
+
         with cache_file.open('w') as f:
-            f.write(model_dump_json(app, indent=2))
+            f.write(model_dump_json(analysis, indent=2))
 
         logger.info(f"Analysis cached to {cache_file}")
 
@@ -583,9 +675,9 @@ class Codeanalyzer:
         if self.file_name is not None:
             single_file = self.project_dir / self.file_name
             logger.info(f"Analyzing single file: {single_file}")
-            
+
             # Check if file is in cache and unchanged
-            file_key = str(single_file)
+            file_key = str(single_file.relative_to(self.project_dir))
             if file_key in cached_symbol_table and not self.rebuild_analysis:
                 # Compute file checksum to see if it changed
                 if self._file_unchanged(single_file, cached_symbol_table[file_key]):
@@ -635,7 +727,7 @@ class Codeanalyzer:
             # Separate files into cached and new/changed
             files_to_process = []
             for py_file in py_files:
-                file_key = str(py_file)
+                file_key = str(py_file.relative_to(self.project_dir))
                 if file_key in cached_symbol_table and not self.rebuild_analysis:
                     if self._file_unchanged(py_file, cached_symbol_table[file_key]):
                         # Use cached version
@@ -645,6 +737,7 @@ class Codeanalyzer:
             
             # Process only new/changed files with Ray
             if files_to_process:
+                _ensure_ray()
                 futures = [_process_file_with_ray.remote(py_file, self.project_dir, str(self.virtualenv) if self.virtualenv else None) for py_file in files_to_process]
                 
                 with ProgressBar(len(futures), "Building symbol table (parallel)") as progress:
@@ -663,7 +756,7 @@ class Codeanalyzer:
             
             with ProgressBar(len(py_files), "Building symbol table") as progress:
                 for py_file in py_files:
-                    file_key = str(py_file)
+                    file_key = str(py_file.relative_to(self.project_dir))
                     
                     # Check if file is cached and unchanged
                     if file_key in cached_symbol_table and not self.rebuild_analysis:
@@ -699,7 +792,7 @@ class Codeanalyzer:
         """Build PyCG-resolved call edges.
 
         Runs PyCG's iterative name-pointer analysis over the whole project
-        and returns edges with ``provenance=["pycg"]``.  Falls back to an
+        and returns edges with ``prov=["pycg"]``.  Falls back to an
         empty list and logs a warning on any failure so the caller can
         continue with Jedi-only edges.
 

codeanalyzer/dataflow/__init__.py

@@ -0,0 +1,35 @@
+################################################################################
+# Copyright IBM Corporation 2025
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+"""Level-3 native dataflow graphs: CFG, PDG (CDG + DDG), and the SDG.
+
+One pass per module, mirroring the construction ladder:
+
+- :mod:`cfg` — stage 1, exceptional statement-level CFG per callable;
+- :mod:`dominance` — stage 2, post-dominators and control dependence;
+- :mod:`access_paths` — stage 3a, the k-limited access-path variable model;
+- :mod:`defuse` — stage 3b, reaching definitions → DDG edges;
+- :mod:`alias` — stage 5, the type-based may-alias oracle (MVP stub);
+- :mod:`scc` — stage 5, Tarjan SCC condensation of the call graph;
+- :mod:`summaries` — stage 6, bottom-up formal-in → formal-out summaries;
+- :mod:`sdg` — stage 7, parameter nodes and CALL/PARAM_IN/PARAM_OUT/SUMMARY
+  edges;
+- :mod:`slicing` — stage 8, the two-phase context-sensitive backward slice;
+- :mod:`builder` — the orchestrator ``build_program_graphs`` wired into
+  ``Codeanalyzer.analyze`` at ``-a 3``.
+"""
+
+from codeanalyzer.dataflow.cfg import build_cfg  # noqa: F401