coretrace-python-analyzer 0.7.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

coretrace_python/__init__.py

@@ -1,4 +1,4 @@
 """CoreTrace's Python static analysis frontend."""
 
-__version__ = "0.7.0"
+__version__ = "0.8.0"
 

coretrace_python/cache.py

@@ -25,6 +25,7 @@ from coretrace_python.findings import Confidence, Finding, Severity
 from coretrace_python.interprocedural import (
     Arguments,
     CallSite,
+    Cleared,
     ExternalCall,
     ExternalSymbol,
     FunctionSummary,
@@ -41,7 +42,7 @@ from coretrace_python.interprocedural import (
 from coretrace_python.semantic.symbols import SymbolId
 from coretrace_python.source import SourceId, SourceSpan
 
-CACHE_FORMAT = 9
+CACHE_FORMAT = 10
 
 
 @dataclass(frozen=True)
@@ -237,6 +238,8 @@ def _encode_call(call: ExternalCall) -> dict[str, Any]:
         "location": _encode_span(call.location),
         "call_site": None if call.call_site is None else _encode_span(call.call_site),
         "given": _encode_arguments(call.arguments),
+        "argument_cleared": [_encode_cleared(c) for c in call.argument_cleared],
+        "keyword_cleared": [_encode_cleared(c) for c in call.keyword_cleared],
     }
 
 
@@ -249,9 +252,19 @@ def _decode_call(data: Mapping[str, Any]) -> ExternalCall:
         _decode_span(data["location"]),
         None if site is None else _decode_span(site),
         _decode_arguments(data["given"]),
+        tuple(_decode_cleared(c) for c in data["argument_cleared"]),
+        tuple(_decode_cleared(c) for c in data["keyword_cleared"]),
     )
 
 
+def _encode_cleared(cleared: Cleared) -> list[list[int]]:
+    return [[parameter, bits] for parameter, bits in cleared]
+
+
+def _decode_cleared(data: Any) -> Cleared:
+    return tuple((_integer(parameter), _integer(bits)) for parameter, bits in data)
+
+
 def _encode_summary(summary: FunctionSummary) -> dict[str, Any]:
     return {
         "name": summary.name,
@@ -275,6 +288,7 @@ def _encode_summary(summary: FunctionSummary) -> dict[str, Any]:
             for w in summary.nonlocal_writes
         ],
         "static": summary.static,
+        "return_cleared": _encode_cleared(summary.return_cleared),
     }
 
 
@@ -303,6 +317,7 @@ def _decode_summary(data: Mapping[str, Any]) -> FunctionSummary:
             for w in data["nonlocal_writes"]
         ),
         bool(data["static"]),
+        _decode_cleared(data["return_cleared"]),
     )
 
 

coretrace_python/cli.py

@@ -27,6 +27,7 @@ from coretrace_python.reporters import FORMATS, render
 from coretrace_python.semantic.imports import ImportResolutionError
 from coretrace_python.semantic.scopes import ScopeError
 from coretrace_python.source import SourceManager
+from coretrace_python.taint import ModelError
 
 EXIT_CLEAN = 0
 EXIT_FINDINGS = 1
@@ -45,6 +46,7 @@ _ANALYSIS_ERRORS = (
     AnalysisError,
     ManifestError,
     IncompatiblePluginError,
+    ModelError,
 )
 
 

coretrace_python/engine.py

@@ -70,6 +70,7 @@ from coretrace_python.interprocedural import (
     CallGraph,
     CallGraphAnalysis,
     CallSite,
+    ClearingAnalysis,
     FunctionSummary,
     ModuleFunction,
     ModuleGraph,
@@ -149,6 +150,7 @@ ALL_ANALYSES: tuple[AnyAnalysis, ...] = (
     DependencyAnalysis,
     RegisteredRoutes,
     EscapedTemplates,
+    ClearingAnalysis,
 )
 
 
@@ -188,6 +190,7 @@ class ResultsEvicted(TransformationPass):
             DependencyAnalysis,
             RegisteredRoutes,
             EscapedTemplates,
+            ClearingAnalysis,
         }
     )
 
@@ -246,7 +249,9 @@ def build_manager(
     """A manager with every engine analysis registered and the engine inputs provided."""
 
     manager = _register_all(module)
-    manager.provide(SecurityModelAnalysis, (models or SecurityModelRegistry()).freeze())
+    table = (models or SecurityModelRegistry()).freeze()
+    manager.provide(SecurityModelAnalysis, table)
+    manager.provide(ClearingAnalysis, table.clearing())
     manager.provide(ProjectSummaries, SummaryIndex())
     manager.provide(RegisteredRoutes, _routes_of(manager))
     return manager
@@ -276,10 +281,14 @@ def load_plugins(plugin_roots: Sequence[Path], manager: AnalysisManager) -> Plug
     return registry
 
 
-def plugin_models(plugins: Iterable[Plugin]) -> ModelTable:
+def plugin_models(plugins: Iterable[Plugin], root: Path | None = None) -> ModelTable:
+    """The models of ``plugins``, with those they read from the project at ``root``."""
+
     models = SecurityModelRegistry()
     for plugin in plugins:
         models.register(*plugin.models, origin=plugin.name)
+        if root is not None:
+            models.register(*plugin.project_models(root), origin=plugin.name)
     return models.freeze()
 
 
@@ -294,7 +303,9 @@ def analyze_file(source: SourceFile, plugin_roots: Sequence[Path]) -> FileAnalys
 
     manager = _register_all(build_hir(source))
     registry = load_plugins(plugin_roots, manager)
-    manager.provide(SecurityModelAnalysis, plugin_models(loaded.plugin for loaded in registry))
+    table = plugin_models(loaded.plugin for loaded in registry)
+    manager.provide(SecurityModelAnalysis, table)
+    manager.provide(ClearingAnalysis, table.clearing())
     manager.provide(ProjectSummaries, SummaryIndex())
     manager.provide(RegisteredRoutes, _routes_of(manager))
     findings, supported = _check_module(manager, tuple(loaded.plugin for loaded in registry))
@@ -385,7 +396,7 @@ def analyze_project(
         (a for plugin in all_plugins for a in plugin.advisories), file_advisories
     )
     affected = affected_symbols(dependencies, advisories)
-    models = plugin_models(all_plugins).extended(*advisory_sinks(affected))
+    models = plugin_models(all_plugins, root).extended(*advisory_sinks(affected))
     for manager in managers.values():
         manager.provide(SecurityModelAnalysis, models)
         manager.provide(DependencyAnalysis, dependencies)
@@ -408,9 +419,11 @@ def analyze_project(
         for symbol, registered in _routes_of(analysable[name]).items():
             routes.setdefault(symbol, registered)
     escaped = escaped_templates(root)
+    clearing = models.clearing(escaped)
     for manager in analysable.values():
         manager.provide(RegisteredRoutes, routes)
         manager.provide(EscapedTemplates, escaped)
+        manager.provide(ClearingAnalysis, clearing)
 
     configuration = _configuration_key(registry, plugins, models, advisories, dependencies, routes, escaped)
     keys = module_keys(
@@ -620,13 +633,15 @@ def _analyse_batch(batch: _Batch) -> dict[str, dict[str, Any]]:
         (a for plugin in all_plugins for a in plugin.advisories), file_advisories
     )
     affected = affected_symbols(dependencies, advisories)
-    models = plugin_models(all_plugins).extended(*advisory_sinks(affected))
+    models = plugin_models(all_plugins, batch.root).extended(*advisory_sinks(affected))
     routes = _decode_routes(batch.routes)
+    escaped = frozenset(batch.escaped)
     for manager in managers.values():
         manager.provide(SecurityModelAnalysis, models)
         manager.provide(DependencyAnalysis, dependencies)
         manager.provide(RegisteredRoutes, routes)
-        manager.provide(EscapedTemplates, frozenset(batch.escaped))
+        manager.provide(EscapedTemplates, escaped)
+        manager.provide(ClearingAnalysis, models.clearing(escaped))
     module_plugins = tuple(p for p in all_plugins if not isinstance(p, ProjectPlugin))
     results = _analyse_managers(managers, decode_index(batch.seed), module_plugins, affected)
     return {name: encode(entry) for name, entry in results.items()}

coretrace_python/findings/refutation.py

@@ -29,7 +29,12 @@ from coretrace_python.abstract import ConstantPropagation, RangeAnalysis, RangeF
 from coretrace_python.analysis import AnalysisContext, AnyAnalysis, FunctionAnalysis
 from coretrace_python.cfg import CFG, BlockId, CFGAnalysis, DominanceAnalysis, DominatorTree
 from coretrace_python.hir import nodes
-from coretrace_python.interprocedural import CallGraphAnalysis
+from coretrace_python.interprocedural import (
+    CallGraph,
+    CallGraphAnalysis,
+    KnownFunction,
+    project_symbol,
+)
 from coretrace_python.ir.model import (
     BoolOp,
     Branch,
@@ -55,6 +60,7 @@ from coretrace_python.taint import (
     TaintAnalysis,
     TaintFacts,
     TaintFlow,
+    TaintKind,
 )
 
 VALIDATORS = frozenset(
@@ -214,15 +220,16 @@ class _Judge:
 
     def interpret(
         self, condition: Value, truth: bool | None
-    ) -> tuple[dict[Value, str], set[Value]]:
-        """Values the condition proves safe (with the reason), and values it mentions
-        without proving anything. ``truth`` is ``None`` when it is not fixed at the sink.
+    ) -> tuple[dict[Value, tuple[str, TaintKind]], set[Value]]:
+        """Values the condition proves safe (with the reason and the kinds it proves them
+        safe for), and values it mentions without proving anything. ``truth`` is ``None``
+        when it is not fixed at the sink.
 
         A recognised check evaluated the wrong way (``isdigit()`` known false) yields
         nothing: it is neither a proof nor a reassuring guard."""
 
         definition = self.defs.get(condition)
-        proven: dict[Value, str] = {}
+        proven: dict[Value, tuple[str, TaintKind]] = {}
         mentioned: set[Value] = set()
         if isinstance(definition, UnaryOp) and definition.operator == "not":
             return self.interpret(definition.operand, None if truth is None else not truth)
@@ -236,27 +243,29 @@ class _Judge:
         recognised = self.recognise(definition)
         if recognised is None:
             return proven, set(self.closure(condition)) | {condition}
-        tested, reason, when_true = recognised
+        tested, reason, when_true, kinds = recognised
         if truth is None:
             mentioned = set(self.closure(condition)) | {condition}
         elif truth == when_true:
             # The check proves ``tested``; whatever else it reads is merely mentioned.
-            proven[tested] = reason
+            proven[tested] = (reason, kinds)
             mentioned = set(self.closure(condition)) | {condition}
         return proven, mentioned
 
-    def recognise(self, definition: Instruction | None) -> tuple[Value, str, bool] | None:
-        """``(validated value, reason, truth that validates)`` for known check shapes."""
+    def recognise(self, definition: Instruction | None) -> tuple[Value, str, bool, TaintKind] | None:
+        """``(validated value, reason, truth that validates, kinds it proves)`` for known
+        check shapes. A string check, a constant allowlist or an equality with a constant
+        proves every kind; a ``Validator`` only the kinds it declares."""
 
         if isinstance(definition, Call) and not definition.arguments:
             callee = self.defs.get(definition.callee)
             if isinstance(callee, GetAttr) and callee.attribute in VALIDATORS:
-                return callee.object, f"guarded by {callee.attribute}()", True
+                return callee.object, f"guarded by {callee.attribute}()", True, TaintKind.ALL
         if isinstance(definition, Call):
             symbol = self.symbols.get(definition.callee)
             validator = self.models.validator(symbol) if symbol is not None else None
             if validator is not None and validator.argument < len(definition.arguments):
-                return definition.arguments[validator.argument], f"validated by {symbol}", True
+                return definition.arguments[validator.argument], f"validated by {symbol}", True, validator.kinds
         if isinstance(definition, Compare):
             left, right = definition.left, definition.right
             if definition.operator in ("in", "not_in") and self.is_constant_collection(right):
@@ -264,11 +273,12 @@ class _Judge:
                     left,
                     "allowlisted by a membership check on constants",
                     definition.operator == "in",
+                    TaintKind.ALL,
                 )
             if definition.operator in ("eq", "not_eq"):
                 for tested, other in ((left, right), (right, left)):
                     if isinstance(self.defs.get(other), Constant):
-                        return tested, "equals a constant", definition.operator == "eq"
+                        return tested, "equals a constant", definition.operator == "eq", TaintKind.ALL
         return None
 
     def examined(
@@ -387,8 +397,9 @@ class _Judge:
         )
         for guard in self.guards(sink):
             found, mentioned = self.interpret(guard.condition, guard.truth)
-            for value, reason in found.items():
-                if value in chain:
+            for value, (reason, kinds) in found.items():
+                # A check proving some kinds only leaves the others reaching the sink.
+                if value in chain and not flow.kinds & ~kinds:
                     proven.setdefault(value, reason)
             examined = self.examined(guard.condition, mentioned, flow.argument, chain)
             for origin in origins:
@@ -478,6 +489,7 @@ class RefutationAnalysis(FunctionAnalysis[Verdicts]):
         constants = ctx.get(ConstantPropagation, function)
         graph = ctx.get(CallGraphAnalysis)
         models = ctx.get(SecurityModelAnalysis)
+        name = graph.name_of(function)
         return judge_flows(
             ssa,
             ctx.get(CFGAnalysis, function),
@@ -486,7 +498,21 @@ class RefutationAnalysis(FunctionAnalysis[Verdicts]):
             frozenset(b.id for b in ssa.blocks if constants.reachable(b.id)),
             ctx.get(RangeAnalysis, function),
             models,
-            graph.symbols(graph.name_of(function)),
+            _with_project_callees(ssa, graph, name, ctx.module.name),
             authorization_of(function, models, ctx.get(ScopeAnalysis), ctx.get(SymbolAnalysis)),
         )
 
+
+def _with_project_callees(function: FunctionIR, graph: CallGraph, name: str, module: str) -> dict[Value, SymbolId]:
+    """The symbols of ``name``'s values, with each call to a function of its module named
+    by its project symbol, so a model declared on it (``Validator``) applies there too."""
+
+    symbols = dict(graph.symbols(name))
+    for block in function.blocks:
+        for instruction in block.instructions:
+            if isinstance(instruction, Call):
+                target = graph.target_at(name, instruction.location)
+                if isinstance(target, KnownFunction):
+                    symbols.setdefault(instruction.callee, project_symbol(module, target.name))
+    return symbols
+

coretrace_python/interprocedural/__init__.py

@@ -20,6 +20,9 @@ from coretrace_python.interprocedural.modulegraph import (
     project_symbol,
 )
 from coretrace_python.interprocedural.summaries import (
+    Cleared,
+    Clearing,
+    ClearingAnalysis,
     ExternalCall,
     FunctionSummary,
     Mutation,
@@ -28,6 +31,7 @@ from coretrace_python.interprocedural.summaries import (
     SummaryAnalysis,
     SummaryIndex,
     SummaryTable,
+    cleared_by,
 )
 
 __all__ = [
@@ -35,6 +39,9 @@ __all__ = [
     "CallGraph",
     "CallGraphAnalysis",
     "CallSite",
+    "Cleared",
+    "Clearing",
+    "ClearingAnalysis",
     "ExternalCall",
     "ExternalSymbol",
     "FunctionSummary",
@@ -51,6 +58,7 @@ __all__ = [
     "Target",
     "UnknownTarget",
     "build_module_graph",
+    "cleared_by",
     "discover_files",
     "discover_sources",
     "project_symbol",

coretrace_python/interprocedural/summaries.py

@@ -4,16 +4,19 @@ A summary says which parameters a function's return value depends on and which e
 symbols its parameters reach, directly or through calls to other known functions. It is
 computed as a dependence data-flow problem over the SSA form and iterated to a fixpoint
 over the call graph, so recursion converges to the least solution. Summaries carry no
-security knowledge: the taint engine decides which external symbols matter.
+security knowledge of their own: the taint engine decides which external symbols
+matter. They only keep, per parameter, the bits a ``Clearing`` the engine provides says
+the calls on the way cleared, on every path, so a caller's data reaches a sink or comes
+back without the kinds a sanitizer inside the function removed.
 """
 
 from __future__ import annotations
 
 import dataclasses
-from collections.abc import Mapping
+from collections.abc import Iterable, Mapping
 from dataclasses import dataclass
 from types import MappingProxyType
-from typing import ClassVar
+from typing import ClassVar, TypeVar
 
 from coretrace_python.abstract import (
     ATTRIBUTES,
@@ -69,22 +72,90 @@ from coretrace_python.source import SourceSpan
 
 Dependencies = frozenset[int]
 NONE: Dependencies = frozenset()
+_T = TypeVar("_T")
+# Per parameter, the taint kind bits cleared on every path from it, sorted, none empty.
+Cleared = tuple[tuple[int, int], ...]
 
 
 @dataclass(frozen=True)
 class Dep:
-    """What a value depends on: parameter indices and results of external symbols."""
+    """What a value depends on: parameter indices, results of external symbols, and for
+    each parameter the bits cleared on every path from it to the value."""
 
     parameters: Dependencies = NONE
     externals: frozenset[SymbolId] = frozenset()
+    cleared: Cleared = ()
 
     def __or__(self, other: Dep) -> Dep:
-        return Dep(self.parameters | other.parameters, self.externals | other.externals)
+        parameters = self.parameters | other.parameters
+        if not self.cleared and not other.cleared:
+            return Dep(parameters, self.externals | other.externals)
+        own, theirs = dict(self.cleared), dict(other.cleared)
+        # A parameter both sides depend on keeps what both paths cleared.
+        cleared = {
+            p: (own.get(p, 0) & theirs.get(p, 0))
+            if p in self.parameters and p in other.parameters
+            else own.get(p, 0) | theirs.get(p, 0)
+            for p in parameters
+        }
+        return Dep(parameters, self.externals | other.externals, _cleared(cleared))
+
+    def clearing(self, bits: int) -> Dep:
+        """This dependence after a call clearing ``bits`` from what it returns."""
+
+        if not bits or not self.parameters:
+            return self
+        own = dict(self.cleared)
+        return Dep(self.parameters, self.externals, _cleared({p: own.get(p, 0) | bits for p in self.parameters}))
+
+
+def _cleared(bits: Mapping[int, int]) -> Cleared:
+    return tuple(sorted((p, b) for p, b in bits.items() if b))
+
+
+def cleared_by(cleared: Cleared, parameter: int) -> int:
+    """The bits ``cleared`` records for ``parameter``, none when it has no entry."""
+
+    return next((bits for p, bits in cleared if p == parameter), 0)
 
 
 EMPTY = Dep()
 
 
+@dataclass(frozen=True)
+class Clearing:
+    """What an external call clears from the data it returns, as taint kind bits the
+    summaries carry without reading them. The engine derives it from the security models:
+    a sanitizer clears its kinds; a template render clears the bits of its entry when the
+    template it names is one the project shows escaping (``escaped`` holds the names the
+    way call sites record a constant)."""
+
+    sanitizers: Mapping[SymbolId, int] = dataclasses.field(default_factory=dict)
+    renders: Mapping[SymbolId, tuple[int, str, int]] = dataclasses.field(default_factory=dict)
+    escaped: frozenset[str] = frozenset()
+
+    def clears(self, symbol: SymbolId, arguments: Arguments) -> int:
+        bits = self.sanitizers.get(symbol, 0)
+        render = self.renders.get(symbol)
+        if render is not None:
+            position, keyword, rendered = render
+            named = arguments.given(keyword, position)
+            if named is not None and named[0] and named[1] in self.escaped:
+                bits |= rendered
+        return bits
+
+
+class ClearingAnalysis(Analysis[Clearing]):
+    """What calls clear, provided by the engine from the security models; nothing on its
+    own, so a summary then keeps every kind."""
+
+    name: ClassVar[str] = "interprocedural.clearing"
+
+    @classmethod
+    def compute(cls, ctx: AnalysisContext) -> Clearing:
+        return Clearing()
+
+
 @dataclass(frozen=True)
 class ExternalCall:
     """An external symbol reached from this function; dependencies are parameter indices.
@@ -98,6 +169,19 @@ class ExternalCall:
     call_site: SourceSpan | None
     # What the arguments of the call at ``location`` denote, where it is made.
     arguments: Arguments = dataclasses.field(default_factory=Arguments)
+    # What was cleared on the way to each argument and keyword, in the same order.
+    argument_cleared: tuple[Cleared, ...] = ()
+    keyword_cleared: tuple[Cleared, ...] = ()
+
+    def positional(self) -> Iterable[tuple[Dependencies, Cleared]]:
+        """Each positional argument's dependencies, with what was cleared on their way."""
+
+        return _aligned(self.argument_dependencies, self.argument_cleared)
+
+    def named(self) -> Iterable[tuple[str | None, Dependencies, Cleared]]:
+        """Each keyword's name and dependencies, with what was cleared on their way."""
+
+        return ((name, deps, c) for (name, deps), c in _aligned(self.keyword_dependencies, self.keyword_cleared))
 
 
 @dataclass(frozen=True)
@@ -134,6 +218,8 @@ class FunctionSummary:
     nonlocal_writes: tuple[NonlocalWrite, ...] = ()
     # A ``@staticmethod``: ``Class.method(...)`` passes no receiver.
     static: bool = False
+    # What was cleared on the way from each parameter to the return value.
+    return_cleared: Cleared = ()
 
 
 class SummaryIndex:
@@ -191,6 +277,7 @@ class _DependenceProblem(DataflowProblem[State]):
         table: Mapping[str, FunctionSummary],
         project: Mapping[SymbolId, FunctionSummary] | None = None,
         heap: HeapFacts | None = None,
+        clearing: Clearing | None = None,
     ) -> None:
         self.name = name
         self.function = function
@@ -198,6 +285,7 @@ class _DependenceProblem(DataflowProblem[State]):
         self.table = table
         self.project = project or {}
         self.heap = heap or HeapFacts({})
+        self.clearing = clearing or Clearing()
         self.blocks = {block.id: block for block in function.blocks}
         self.defs: dict[Value, Instruction] = {
             i.result: i for block in function.blocks for i in block.instructions if i.result
@@ -338,15 +426,10 @@ class _DependenceProblem(DataflowProblem[State]):
                 bound = () if project.static else self.receiver(call, project.name)
                 arguments = (*(self.deep(r, state) for r in bound), *arguments)
                 return self.known(project, arguments, keywords, everything, call, state, bound)
-            self.record(
-                target.symbol,
-                tuple(a.parameters for a in arguments),
-                tuple((name, deps.parameters) for name, deps in named),
-                call.location,
-                None,
-                self.graph.arguments_at(self.name, call.location),
-            )
-            return everything | Dep(externals=frozenset({target.symbol}))
+            given = self.graph.arguments_at(self.name, call.location)
+            self.record(target.symbol, arguments, named, call.location, None, given)
+            cleared = everything.clearing(self.clearing.clears(target.symbol, given))
+            return cleared | Dep(externals=frozenset({target.symbol}))
         if isinstance(target, KnownFunction):
             made = self.defs.get(call.callee)
             if isinstance(made, MakeFunction):
@@ -414,17 +497,18 @@ class _DependenceProblem(DataflowProblem[State]):
         if callee.unsupported:
             return everything
 
-        def mapped(indices: Dependencies) -> Dep:
+        def mapped(indices: Dependencies, cleared: Cleared = ()) -> Dep:
             result = EMPTY
             for index in indices:
-                result |= arguments[index] if index < len(arguments) else keywords
+                part = arguments[index] if index < len(arguments) else keywords
+                result |= part.clearing(cleared_by(cleared, index))
             return result
 
         for reached in callee.external_calls:
             self.record(
                 reached.symbol,
-                tuple(mapped(d).parameters for d in reached.argument_dependencies),
-                tuple((name, mapped(d).parameters) for name, d in reached.keyword_dependencies),
+                tuple(mapped(d, c) for d, c in reached.positional()),
+                tuple((name, mapped(d, c)) for name, d, c in reached.named()),
                 reached.location,
                 call.location,
                 reached.arguments,
@@ -435,13 +519,13 @@ class _DependenceProblem(DataflowProblem[State]):
             if mutation.parameter < len(values):
                 deps = mapped(mutation.dependencies) | Dep(externals=mutation.externals)
                 self.store(state, values[mutation.parameter], mutation.field, deps)
-        return mapped(callee.return_dependencies) | Dep(externals=callee.return_externals)
+        return mapped(callee.return_dependencies, callee.return_cleared) | Dep(externals=callee.return_externals)
 
     def record(
         self,
         symbol: SymbolId,
-        arguments: tuple[Dependencies, ...],
-        keywords: tuple[tuple[str | None, Dependencies], ...],
+        arguments: tuple[Dep, ...],
+        keywords: tuple[tuple[str | None, Dep], ...],
         location: SourceSpan,
         call_site: SourceSpan | None,
         given: Arguments,
@@ -450,14 +534,30 @@ class _DependenceProblem(DataflowProblem[State]):
         previous = self.external.get(key)
         if previous is not None:
             arguments = tuple(
-                a | b for a, b in zip(previous.argument_dependencies, arguments, strict=False)
+                a | Dep(b, cleared=c) for a, (b, c) in zip(arguments, previous.positional(), strict=False)
             )
             # The same call records the same keywords, in the same order.
             keywords = tuple(
-                (name, deps | old)
-                for (name, deps), (_, old) in zip(keywords, previous.keyword_dependencies, strict=False)
+                (name, deps | Dep(old, cleared=c))
+                for (name, deps), (_, old, c) in zip(keywords, previous.named(), strict=False)
             )
-        self.external[key] = ExternalCall(symbol, arguments, keywords, location, call_site, given)
+        self.external[key] = ExternalCall(
+            symbol,
+            tuple(a.parameters for a in arguments),
+            tuple((name, deps.parameters) for name, deps in keywords),
+            location,
+            call_site,
+            given,
+            tuple(a.cleared for a in arguments),
+            tuple(deps.cleared for _, deps in keywords),
+        )
+
+
+def _aligned(dependencies: tuple[_T, ...], cleared: tuple[Cleared, ...]) -> Iterable[tuple[_T, Cleared]]:
+    """Each dependency with what was cleared on its way; nothing when not recorded."""
+
+    missing = len(dependencies) - len(cleared)
+    return zip(dependencies, (*cleared, *(() for _ in range(missing))), strict=True)
 
 
 def summarize(
@@ -469,8 +569,9 @@ def summarize(
     project: Mapping[SymbolId, FunctionSummary] | None = None,
     heap: HeapFacts | None = None,
     static: bool = False,
+    clearing: Clearing | None = None,
 ) -> FunctionSummary:
-    problem = _DependenceProblem(name, function, graph, table, project, heap)
+    problem = _DependenceProblem(name, function, graph, table, project, heap, clearing)
     solution = solve(problem, cfg)
     problem.external = {}
     problem.returns = EMPTY
@@ -502,19 +603,21 @@ def summarize(
             if deps.parameters or deps.externals
         ),
         static=static,
+        return_cleared=problem.returns.cleared,
     )
 
 
 class SummaryAnalysis(Analysis[SummaryTable]):
     name: ClassVar[str] = "interprocedural.summaries"
     requires: ClassVar[frozenset[AnyAnalysis]] = frozenset(
-        {CallGraphAnalysis, SSAAnalysis, CFGAnalysis, ProjectSummaries, HeapAnalysis}
+        {CallGraphAnalysis, SSAAnalysis, CFGAnalysis, ProjectSummaries, HeapAnalysis, ClearingAnalysis}
     )
 
     @classmethod
     def compute(cls, ctx: AnalysisContext) -> SummaryTable:
         graph = ctx.get(CallGraphAnalysis)
         project = ctx.get(ProjectSummaries).summaries
+        clearing = ctx.get(ClearingAnalysis)
         table: dict[str, FunctionSummary] = {}
         supported: dict[str, tuple[FunctionIR, CFG, HeapFacts]] = {}
         for name, function in graph.definitions.items():
@@ -533,7 +636,7 @@ class SummaryAnalysis(Analysis[SummaryTable]):
             changed = False
             for name, (ssa, cfg, heap) in supported.items():
                 updated = summarize(
-                    name, ssa, cfg, graph, table, project, heap, _is_staticmethod(graph.definitions[name])
+                    name, ssa, cfg, graph, table, project, heap, _is_staticmethod(graph.definitions[name]), clearing
                 )
                 if updated != table[name]:
                     table[name] = updated

coretrace_python/plugins/api.py

@@ -51,6 +51,14 @@ class Plugin(ABC):
     def analyze(self, ctx: PluginContext) -> Sequence[Finding]:
         raise NotImplementedError
 
+    def project_models(self, root: Path) -> Sequence[Model]:
+        """Models read from the project under ``root``, such as validators it declares
+        in a file there. Called once per directory check, in every process analysing
+        it, before any module; a single-file check reads no project. A model declared
+        wrongly raises ``ModelError``, which stops the check with its message."""
+
+        return ()
+
 
 class ModelPlugin(Plugin):
     """A plugin that only contributes security models or advisories (§15 providers)."""

coretrace_python/taint/engine.py

@@ -30,6 +30,8 @@ from coretrace_python.interprocedural import (
     Arguments,
     CallGraph,
     CallGraphAnalysis,
+    Cleared,
+    Clearing,
     ExternalSymbol,
     FunctionSummary,
     KnownFunction,
@@ -38,6 +40,7 @@ from coretrace_python.interprocedural import (
     SummaryAnalysis,
     SummaryIndex,
     SummaryTable,
+    cleared_by,
     project_symbol,
 )
 from coretrace_python.ir.lowering import analyzable_functions, is_body_function
@@ -153,12 +156,11 @@ class _TaintProblem(DataflowProblem[State]):
         heap: HeapFacts | None = None,
         seeds: Mapping[HeapLocation, Taint] | None = None,
         module: str = "",
-        escaped: frozenset[str] = frozenset(),
+        clearing: Clearing | None = None,
     ) -> None:
         self.name = name
         self.module = module
-        # As call sites record a constant argument: the way Python writes it.
-        self.escaped = frozenset(repr(name) for name in escaped)
+        self.clearing = clearing or Clearing()
         self.function = function
         self.models = models
         self.graph = graph
@@ -411,11 +413,9 @@ class _TaintProblem(DataflowProblem[State]):
                 self.report(
                     flows, sink, self.rendered(argument, state), argument, call, None, None, None, given, keyword=name
                 )
-        sanitizer = self.models.sanitizer(symbol)
-        if sanitizer is not None:
-            return everything.without(sanitizer.kinds)
-        if self.renders_escaped(symbol, given):
-            return everything.without(TaintKind.HTML)
+        cleared = self.clearing.clears(symbol, given)
+        if cleared:
+            return everything.without(TaintKind(cleared))
         # A method on a tainted object returns tainted data (``request.args.get``).
         everything = everything.join(state.get(call.callee, Taint.none()))
         source = self.models.source_covering(symbol)
@@ -423,14 +423,6 @@ class _TaintProblem(DataflowProblem[State]):
             return everything.join(Taint(source.kinds, frozenset({source})))
         return everything
 
-    def renders_escaped(self, symbol: SymbolId, given: Arguments) -> bool:
-        """Whether the call renders, by a name written in the call, a template the
-        project shows escaping everything it renders."""
-
-        render = self.models.template_render(symbol)
-        named = given.given(render.keyword, render.position) if render is not None else None
-        return named is not None and named[0] and named[1] in self.escaped
-
     def guarded(self, sink: Sink | None, arguments: Arguments) -> Sink | None:
         """The sink as this call makes it: an argument declared safe (``yaml.load`` with
         ``Loader=SafeLoader``) takes off the kinds it covers, and no kind left, no sink."""
@@ -501,11 +493,11 @@ class _TaintProblem(DataflowProblem[State]):
         spread = (*call.starred, *(value for _, value in call.keywords))
         values = (*receiver, *call.arguments, *captured)
 
-        def mapped(deps: frozenset[int]) -> tuple[Taint, Value | None]:
+        def mapped(deps: frozenset[int], cleared: Cleared = ()) -> tuple[Taint, Value | None]:
             taint, witness = Taint.none(), None
             for index in sorted(deps):
                 positional = index < len(arguments)
-                part = arguments[index] if positional else keywords
+                part = (arguments[index] if positional else keywords).without(TaintKind(cleared_by(cleared, index)))
                 if part and witness is None and (positional or spread):
                     witness = values[index] if positional else spread[0]
                 taint = taint.join(part)
@@ -515,12 +507,12 @@ class _TaintProblem(DataflowProblem[State]):
             sink = self.guarded(self.models.sink(reached.symbol), reached.arguments)
             if sink is None:
                 continue
-            for position, deps in enumerate(reached.argument_dependencies):
-                taint, witness = mapped(deps)
+            for position, (deps, cleared) in enumerate(reached.positional()):
+                taint, witness = mapped(deps, cleared)
                 if witness is not None:
                     self.report(flows, sink, taint, witness, call, through, reached.location, position, reached.arguments)
-            for name, deps in reached.keyword_dependencies:
-                taint, witness = mapped(deps)
+            for name, deps, cleared in reached.named():
+                taint, witness = mapped(deps, cleared)
                 if witness is not None:
                     self.report(
                         flows, sink, taint, witness, call, through, reached.location, None, reached.arguments, keyword=name
@@ -533,7 +525,7 @@ class _TaintProblem(DataflowProblem[State]):
                     if stored_source is not None:
                         stored = stored.join(Taint(stored_source.kinds, frozenset({stored_source})))
                 self.store(state, values[mutation.parameter], mutation.field, stored)
-        result = mapped(summary.return_dependencies)[0]
+        result = mapped(summary.return_dependencies, summary.return_cleared)[0]
         for symbol in sorted(summary.return_externals, key=str):
             returned_source = self.models.source(symbol)
             if returned_source is not None:
@@ -816,9 +808,9 @@ def propagate_taint(
     heap: HeapFacts | None = None,
     seeds: Mapping[HeapLocation, Taint] | None = None,
     module: str = "",
-    escaped: frozenset[str] = frozenset(),
+    clearing: Clearing | None = None,
 ) -> TaintFacts:
-    problem = _TaintProblem(name, function, models, graph, summaries, parameters, project, heap, seeds, module, escaped)
+    problem = _TaintProblem(name, function, models, graph, summaries, parameters, project, heap, seeds, module, clearing)
     solution = solve(problem, cfg)
     taints: dict[Key, Taint] = {}
     flows: list[TaintFlow] = []
@@ -883,7 +875,7 @@ class TaintAnalysis(FunctionAnalysis[TaintFacts]):
             heap,
             seeds,
             ctx.module.name,
-            ctx.get(EscapedTemplates),
+            models.clearing(ctx.get(EscapedTemplates)),
         )
 
 

coretrace_python/taint/models.py

@@ -16,6 +16,7 @@ from types import MappingProxyType
 from typing import ClassVar
 
 from coretrace_python.analysis import Analysis, AnalysisContext, MissingInputError
+from coretrace_python.interprocedural import Clearing
 from coretrace_python.semantic.symbols import SymbolId
 
 
@@ -334,6 +335,17 @@ class ModelTable:
         found = self._by_symbol[TemplateRender].get(symbol)
         return found if isinstance(found, TemplateRender) else None
 
+    def clearing(self, escaped: frozenset[str] = frozenset()) -> Clearing:
+        """What calls clear from the data they return: every sanitizer its kinds, every
+        template render ``HTML`` when it names one of the ``escaped`` templates."""
+
+        return Clearing(
+            MappingProxyType({s.symbol: s.kinds.value for s in self.sanitizers}),
+            MappingProxyType({r.symbol: (r.position, r.keyword, TaintKind.HTML.value) for r in self.template_renders}),
+            # As call sites record a constant argument: the way Python writes it.
+            frozenset(repr(name) for name in escaped),
+        )
+
 
 class SecurityModelRegistry:
     """Mutable collection point for the models plugins register."""

{coretrace_python_analyzer-0.7.0.dist-info → coretrace_python_analyzer-0.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: coretrace-python-analyzer
-Version: 0.7.0
+Version: 0.8.0
 Summary: Static security analysis infrastructure for Python code
 Project-URL: Homepage, https://github.com/CoreTrace/coretrace-python-analyzer
 Project-URL: Repository, https://github.com/CoreTrace/coretrace-python-analyzer

{coretrace_python_analyzer-0.7.0.dist-info → coretrace_python_analyzer-0.8.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
-coretrace_python/__init__.py,sha256=kEx6lE06VX-mfQ6o7DrayGZNlR2uTGSdGItAWsSIeHI,75
+coretrace_python/__init__.py,sha256=u9j9n2ZvaBeVUS33TRJ3MIoLUWwaBdpCAainRMXI4ok,75
 coretrace_python/__main__.py,sha256=V8DnYtRboISK2VCKm7DGHLREKvqLbqApyWlKBOs26wU,65
-coretrace_python/cache.py,sha256=tt-l1jOibCx0lNKGd7XF_xTlcCeBLewPWOPpS-e3Ynk,12929
-coretrace_python/cli.py,sha256=SwXYUR8DIgppkqom6kKNM701pb4p2omEy-hRXIM5yZE,10946
-coretrace_python/engine.py,sha256=3NxNKzkNTWxa7a4NVkZ0fyh5KitNpDHuTsCrdBa4VxU,28696
+coretrace_python/cache.py,sha256=-pZv-ELm4jMCTXQWSZIobHaT9UHIjFu7IlMkOM0i4LI,13607
+coretrace_python/cli.py,sha256=86rTb3Kcezwk9DBTRg-G3G38ypx8zH4lZbCR467xvpE,11008
+coretrace_python/engine.py,sha256=VWTUTgm9k0DrPYhYrgJQHzXi_L00pxnH8fGfo8insMc,29340
 coretrace_python/abstract/__init__.py,sha256=n5FNkV8Z4LmwG0r06KWdLj4N2CPSgAKwY3896tdmRgM,999
 coretrace_python/abstract/constants.py,sha256=z4-e7IzysxzGi58wFFff1MlEoV8l7AZsB9EL-q6b7vY,9071
 coretrace_python/abstract/heap.py,sha256=OaESXFGC0qzyhAr76xrCu4skeDVzZM4wlenQqfMTkdk,9642
@@ -99,7 +99,7 @@ coretrace_python/findings/__init__.py,sha256=gnJI-uB2CkgZ8I5aFYP_N5yZb7FGNIy7Lir
 coretrace_python/findings/baseline.py,sha256=fbGQuULRsCjhxSNdwuLfwsvql9ZUfF3jqCXrRmIgOoQ,3468
 coretrace_python/findings/coverage.py,sha256=U8URLEz6EN4B_TTQlLFCOtEmUbINt7a0Bexb-6OUFlI,1203
 coretrace_python/findings/model.py,sha256=zRK1FDuLgiVY4ch6H_AN9rDxC1orNW4wc8YN3cfVFfc,1424
-coretrace_python/findings/refutation.py,sha256=MmFANBozz0sodIhGUhrgr7BCEwhw5s80kRPRMDRDIyU,20707
+coretrace_python/findings/refutation.py,sha256=UeYCz70LwyL8AMimhs3ys9cH0j1_YbdHVu7y3VvmmUg,22026
 coretrace_python/findings/suppressions.py,sha256=kN4q0CA1tzeAubeYe4rUv_aRXda4PAsMZY0r8M5cuQI,2633
 coretrace_python/frontend/__init__.py,sha256=ULI2tf_IRTMboaKM6uzTkGGh4Agy3cFjtw2H1IPrz4g,627
 coretrace_python/frontend/ast_adapter.py,sha256=VL2CgbQqDGSr2Wh-z0EpR0gBB7T29s0xHvfT-BPx08c,21833
@@ -107,10 +107,10 @@ coretrace_python/frontend/parser.py,sha256=6-CG7rWxUYeOGeDP206SJWy4BMQUP9Mr6Jf-t
 coretrace_python/hir/__init__.py,sha256=xMpvj-RSp6bd78a_hhkqbe99-snjERkH-_Lv5o9ASbw,180
 coretrace_python/hir/nodes.py,sha256=4BI0f3UhKbnz9ZIxMoKXM7ujav0BmRpD0OXaoblRxIk,12364
 coretrace_python/hir/visitors.py,sha256=JFnX09hK6trLyRcWgZ5ehOG2ywX3VvkqQ1h82nahYNg,885
-coretrace_python/interprocedural/__init__.py,sha256=SXtIGE3DD4fADIXLDuEvyT_3oIsh7Vio3BL4U6oUm28,1162
+coretrace_python/interprocedural/__init__.py,sha256=rBASF3aAvl5CFAmIISu2gID9mqY_iyi-4Hx0GzDVZsg,1300
 coretrace_python/interprocedural/callgraph.py,sha256=E6enTawqcl4-TObdjvoFe6l1zEedvsWbnjEBD6omzy8,19827
 coretrace_python/interprocedural/modulegraph.py,sha256=4sB7OKjBYfxtm7aMtHm2jRlqFVUNUsdnLhnv9UZ1BWA,8846
-coretrace_python/interprocedural/summaries.py,sha256=9-WGS-X37KJY1ynoXXA5HNfYFLeCyiKVMAIjeWgaVLI,21333
+coretrace_python/interprocedural/summaries.py,sha256=Z_KT3ggH7fgPz1mJL6TBz6V5STka1koIuUBAuYMti7o,26113
 coretrace_python/ir/__init__.py,sha256=ShBPYYqoQFePP-mgBjGezQ2acel0-ReOHF9vCC-G_4Q,232
 coretrace_python/ir/defuse.py,sha256=CZEPnlgntRp4MqwR7kLldd2ZlIlBnr-V9VJAxXxwm0c,2616
 coretrace_python/ir/lowering.py,sha256=V7r_4tEOi801NBOsPdG6pwrRHZIeVTahygYENneLQ4U,31428
@@ -118,7 +118,7 @@ coretrace_python/ir/model.py,sha256=i7LvHEnl5pmWYUcSui17ZGS1rdPVavMpnjt7Tog7P4c,
 coretrace_python/ir/printer.py,sha256=otBIhZUvB_UfB68gid_FbPcSjBUSmxH3v7vBMZZQVkg,9125
 coretrace_python/ir/ssa.py,sha256=cIYbcv90PI6AOeMrw2ZWp88fy7E2ZO4P3tqjtPHra7g,11116
 coretrace_python/plugins/__init__.py,sha256=xRKqkvkvQMmfQACHHoBioYhIyyK4HoMAVogy6EZG8tg,1398
-coretrace_python/plugins/api.py,sha256=NjeDRRwy2K6esSEkg1Ofmv09mqa0El6MlTmRDCWiOmQ,8182
+coretrace_python/plugins/api.py,sha256=FcIe2c4sFB-PAPtYcJkidvdZJkr_xFKT8SvlBP5l38o,8603
 coretrace_python/plugins/detectors.py,sha256=ej5Z_MS0CBqrR4b9xycjTVUIo5arpArpdbr7nL732iI,5700
 coretrace_python/plugins/loader.py,sha256=Use-lHTL4-IJYQBZmODC89zhJbYKGvDSLZHWjZa5NaI,3885
 coretrace_python/plugins/manifest.py,sha256=F24pgQSwApRUL6ZihRSLv9wL_M03A_pZjrj9Zi4Ku3Q,3438
@@ -138,13 +138,13 @@ coretrace_python/source/__init__.py,sha256=0Y8fvtbkETnNCJR9Zq7RS5TSgF9c22pz3SSk1
 coretrace_python/source/manager.py,sha256=JRmMOcWfS1nocbVrdcvfQTqEfIkmnb5JP857TpGywN4,2736
 coretrace_python/source/model.py,sha256=FZnN5mlir3tzbIpWoxnn6YtRi9UcJ_y0sZM60tKr8U4,1727
 coretrace_python/taint/__init__.py,sha256=kfqCEXE5wKFLwz2XrT42_yEv9pW4vPfgg5yYBVVSr2I,1371
-coretrace_python/taint/engine.py,sha256=uB0Mga1xeKF6LVHx601pvojb0NLAPG0DqF8fy3dM8oU,40222
-coretrace_python/taint/models.py,sha256=i3_Sfwdqf_BGXV47jylfF3vuX6nYJcyAPiwC3xerh6g,14592
+coretrace_python/taint/engine.py,sha256=lB6ysdzYNuYhIiRUaxfI4F3cAnaUK55d4lF9AzPWfrU,39742
+coretrace_python/taint/models.py,sha256=B84caPxmQvTdB9H_80nZOqcFlpNd9ofpKI2YL5Uc0UA,15269
 coretrace_python/taint/routes.py,sha256=UAJKXcw_sM9shf58FfDroxVO9bNSm4qrQdZknwUB0lo,6110
 coretrace_python/taint/templates.py,sha256=8Ojb43VXuWxOVifeXRpi3_2mjZfnvIC7GQ-AW80GUYk,11148
-coretrace_python_analyzer-0.7.0.dist-info/METADATA,sha256=lak3Dd-Dpbv_TEo-PAhnBg5r6iwrJmsDM5aa6wogg1w,3216
-coretrace_python_analyzer-0.7.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-coretrace_python_analyzer-0.7.0.dist-info/entry_points.txt,sha256=wdozemiraOkbt65g_Y_zgspuZpoLxJdaCVMJt1Ft3c0,72
-coretrace_python_analyzer-0.7.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-coretrace_python_analyzer-0.7.0.dist-info/licenses/NOTICE,sha256=s3VHj5ngV0ozjW_kZJpg_35ne6JwoBK0ysOjbqVtA0w,113
-coretrace_python_analyzer-0.7.0.dist-info/RECORD,,
+coretrace_python_analyzer-0.8.0.dist-info/METADATA,sha256=6iUDBdRu9mMtAf4EaeD4sVOr4iEdxzT7tkfOBlaNNT8,3216
+coretrace_python_analyzer-0.8.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+coretrace_python_analyzer-0.8.0.dist-info/entry_points.txt,sha256=wdozemiraOkbt65g_Y_zgspuZpoLxJdaCVMJt1Ft3c0,72
+coretrace_python_analyzer-0.8.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+coretrace_python_analyzer-0.8.0.dist-info/licenses/NOTICE,sha256=s3VHj5ngV0ozjW_kZJpg_35ne6JwoBK0ysOjbqVtA0w,113
+coretrace_python_analyzer-0.8.0.dist-info/RECORD,,