coretrace-python-analyzer 0.24.0__py3-none-any.whl → 0.25.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coretrace_python/__init__.py +1 -1
- coretrace_python/source/__init__.py +8 -1
- coretrace_python/source/manager.py +21 -0
- coretrace_python/source/positions.py +3 -20
- coretrace_python/taint/templates.py +264 -104
- {coretrace_python_analyzer-0.24.0.dist-info → coretrace_python_analyzer-0.25.0.dist-info}/METADATA +1 -1
- {coretrace_python_analyzer-0.24.0.dist-info → coretrace_python_analyzer-0.25.0.dist-info}/RECORD +11 -11
- {coretrace_python_analyzer-0.24.0.dist-info → coretrace_python_analyzer-0.25.0.dist-info}/WHEEL +1 -1
- {coretrace_python_analyzer-0.24.0.dist-info → coretrace_python_analyzer-0.25.0.dist-info}/entry_points.txt +0 -0
- {coretrace_python_analyzer-0.24.0.dist-info → coretrace_python_analyzer-0.25.0.dist-info}/licenses/LICENSE +0 -0
- {coretrace_python_analyzer-0.24.0.dist-info → coretrace_python_analyzer-0.25.0.dist-info}/licenses/NOTICE +0 -0
coretrace_python/__init__.py
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
"""Source storage and location primitives."""
|
|
2
2
|
|
|
3
|
-
from coretrace_python.source.manager import
|
|
3
|
+
from coretrace_python.source.manager import (
|
|
4
|
+
LINE_BREAK,
|
|
5
|
+
LineIndex,
|
|
6
|
+
SourceManager,
|
|
7
|
+
decode_text,
|
|
8
|
+
lines_of,
|
|
9
|
+
)
|
|
4
10
|
from coretrace_python.source.model import (
|
|
5
11
|
FileLocation,
|
|
6
12
|
ImportName,
|
|
@@ -14,6 +20,7 @@ __all__ = [
|
|
|
14
20
|
"LINE_BREAK",
|
|
15
21
|
"FileLocation",
|
|
16
22
|
"ImportName",
|
|
23
|
+
"LineIndex",
|
|
17
24
|
"Location",
|
|
18
25
|
"SourceFile",
|
|
19
26
|
"SourceId",
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import re
|
|
6
|
+
from bisect import bisect_right
|
|
6
7
|
from pathlib import Path
|
|
7
8
|
|
|
8
9
|
from coretrace_python.source.model import ImportName, SourceFile, SourceId
|
|
@@ -86,6 +87,26 @@ def lines_of(text: str) -> list[str]:
|
|
|
86
87
|
return lines
|
|
87
88
|
|
|
88
89
|
|
|
90
|
+
class LineIndex:
|
|
91
|
+
"""The line and column of each offset of a text, lines counted as ``lines_of``
|
|
92
|
+
counts them. The start of each line is found once; the line of an offset is then
|
|
93
|
+
found by bisection, in time logarithmic in the lines."""
|
|
94
|
+
|
|
95
|
+
def __init__(self, text: str) -> None:
|
|
96
|
+
self._starts = [0, *(match.end() for match in LINE_BREAK.finditer(text))]
|
|
97
|
+
|
|
98
|
+
def position(self, offset: int) -> tuple[int, int]:
|
|
99
|
+
"""The one-based line and column of ``offset``."""
|
|
100
|
+
|
|
101
|
+
line = bisect_right(self._starts, offset)
|
|
102
|
+
return line, offset - self._starts[line - 1] + 1
|
|
103
|
+
|
|
104
|
+
def offset(self, line: int, column: int) -> int:
|
|
105
|
+
"""The offset of the one-based ``line`` and ``column``."""
|
|
106
|
+
|
|
107
|
+
return self._starts[line - 1] + column - 1
|
|
108
|
+
|
|
109
|
+
|
|
89
110
|
class SourceManager:
|
|
90
111
|
"""Own source text and return one canonical object for each source ID."""
|
|
91
112
|
|
|
@@ -16,9 +16,8 @@ import hashlib
|
|
|
16
16
|
import json
|
|
17
17
|
import re
|
|
18
18
|
import tomllib
|
|
19
|
-
from bisect import bisect_right
|
|
20
19
|
|
|
21
|
-
from coretrace_python.source.manager import
|
|
20
|
+
from coretrace_python.source.manager import LineIndex, lines_of
|
|
22
21
|
from coretrace_python.source.model import FileLocation, SourceId
|
|
23
22
|
|
|
24
23
|
Pointer = tuple[str | int, ...]
|
|
@@ -54,22 +53,6 @@ def pointer_text(path: Pointer) -> str:
|
|
|
54
53
|
return "".join("/" + str(part).replace("~", "~0").replace("/", "~1") for part in path)
|
|
55
54
|
|
|
56
55
|
|
|
57
|
-
class _Lines:
|
|
58
|
-
def __init__(self, text: str) -> None:
|
|
59
|
-
self._starts = [0, *(match.end() for match in LINE_BREAK.finditer(text))]
|
|
60
|
-
|
|
61
|
-
def position(self, offset: int) -> tuple[int, int]:
|
|
62
|
-
"""The one-based line and column of ``offset``."""
|
|
63
|
-
|
|
64
|
-
line = bisect_right(self._starts, offset)
|
|
65
|
-
return line, offset - self._starts[line - 1] + 1
|
|
66
|
-
|
|
67
|
-
def offset(self, line: int, column: int) -> int:
|
|
68
|
-
"""The offset of the one-based ``line`` and ``column``."""
|
|
69
|
-
|
|
70
|
-
return self._starts[line - 1] + column - 1
|
|
71
|
-
|
|
72
|
-
|
|
73
56
|
def _same(found: object, value: object) -> bool:
|
|
74
57
|
return type(found) is type(value) and found == value
|
|
75
58
|
|
|
@@ -87,7 +70,7 @@ class JsonPositions:
|
|
|
87
70
|
raise ValueError("trailing text")
|
|
88
71
|
except (ValueError, IndexError, RecursionError):
|
|
89
72
|
self._values = {}
|
|
90
|
-
self._lines =
|
|
73
|
+
self._lines = LineIndex(text)
|
|
91
74
|
|
|
92
75
|
def locate(self, path: Pointer, value: object) -> tuple[int, int] | None:
|
|
93
76
|
"""The line and column ``value`` starts at under ``path``, or None when the text
|
|
@@ -183,7 +166,7 @@ class TomlPositions:
|
|
|
183
166
|
self._lines = lines_of(text)
|
|
184
167
|
# The text with one line feed per line break, where arrays are read, once each.
|
|
185
168
|
self._text = "\n".join(self._lines)
|
|
186
|
-
self._offsets =
|
|
169
|
+
self._offsets = LineIndex(self._text)
|
|
187
170
|
self._arrays: dict[int, list[tuple[int, int]]] = {}
|
|
188
171
|
state = _LexState()
|
|
189
172
|
for number, line in enumerate(self._lines, start=1):
|
|
@@ -53,7 +53,7 @@ from __future__ import annotations
|
|
|
53
53
|
import re
|
|
54
54
|
from bisect import bisect_left
|
|
55
55
|
from collections import deque
|
|
56
|
-
from collections.abc import Container, Iterable, Iterator, Mapping, Sequence
|
|
56
|
+
from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence
|
|
57
57
|
from dataclasses import dataclass, field
|
|
58
58
|
from pathlib import Path
|
|
59
59
|
from types import MappingProxyType
|
|
@@ -79,7 +79,7 @@ from coretrace_python.semantic.scopes import (
|
|
|
79
79
|
analyze_scopes,
|
|
80
80
|
)
|
|
81
81
|
from coretrace_python.semantic.symbols import SymbolId, analyze_symbols
|
|
82
|
-
from coretrace_python.source import SourceId, SourceManager, SourceSpan
|
|
82
|
+
from coretrace_python.source import LineIndex, SourceId, SourceManager, SourceSpan
|
|
83
83
|
from coretrace_python.taint.models import ModelTable
|
|
84
84
|
|
|
85
85
|
TEMPLATES_DIRECTORY = "templates"
|
|
@@ -935,13 +935,13 @@ class _Application:
|
|
|
935
935
|
|
|
936
936
|
@dataclass(frozen=True)
|
|
937
937
|
class _Reference:
|
|
938
|
-
"""A template another one includes, or ``extends``, at ``
|
|
938
|
+
"""A template another one includes, or ``extends``, at ``span`` in its ``blocks``:
|
|
939
939
|
by ``name``, None when an expression names it. The referenced template's free names
|
|
940
940
|
are ``bindings``, then, unless ``only``, what the ``scope`` binds there, then the
|
|
941
941
|
referencing template's."""
|
|
942
942
|
|
|
943
943
|
name: str | None
|
|
944
|
-
|
|
944
|
+
span: SourceSpan
|
|
945
945
|
bindings: Mapping[str, frozenset[str]] = field(default_factory=dict)
|
|
946
946
|
only: bool = False
|
|
947
947
|
scope: Mapping[str, frozenset[str]] = field(default_factory=dict)
|
|
@@ -961,12 +961,12 @@ class _Reference:
|
|
|
961
961
|
|
|
962
962
|
@dataclass(frozen=True)
|
|
963
963
|
class _Load:
|
|
964
|
-
"""A ``{% load %}`` of ``library`` at ``
|
|
964
|
+
"""A ``{% load %}`` of ``library`` at ``span``, of the ``names`` it selects
|
|
965
965
|
(``{% load a b from lib %}``), None for the whole library."""
|
|
966
966
|
|
|
967
967
|
library: str
|
|
968
968
|
names: frozenset[str] | None
|
|
969
|
-
|
|
969
|
+
span: SourceSpan
|
|
970
970
|
|
|
971
971
|
|
|
972
972
|
@dataclass(frozen=True)
|
|
@@ -989,29 +989,16 @@ class _Library:
|
|
|
989
989
|
@dataclass(frozen=True)
|
|
990
990
|
class _Template:
|
|
991
991
|
"""What one template file applies and references, the ``blocks`` it defines, and
|
|
992
|
-
among them the ``extended`` ones, which render ``{{ block.super }}``,
|
|
993
|
-
|
|
992
|
+
among them the ``extended`` ones, which render ``{{ block.super }}``, the loads under
|
|
993
|
+
which it applies a filter the engine could not resolve, and the block each block
|
|
994
|
+
stands in, None for one standing in none (#285)."""
|
|
994
995
|
|
|
995
996
|
applications: tuple[_Application, ...]
|
|
996
997
|
references: tuple[_Reference, ...]
|
|
997
998
|
blocks: frozenset[str]
|
|
998
999
|
extended: frozenset[str] = frozenset()
|
|
999
1000
|
unresolved: tuple[_Load, ...] = ()
|
|
1000
|
-
|
|
1001
|
-
def rendered(self, blocks: tuple[str, ...], overridden: frozenset[str]) -> bool:
|
|
1002
|
-
"""Whether what stands in ``blocks`` renders when a template extending this one
|
|
1003
|
-
defines the ``overridden`` blocks: a template extending another renders only
|
|
1004
|
-
its blocks, where they replace the blocks of that name in what it extends."""
|
|
1005
|
-
|
|
1006
|
-
extends = any(reference.extends for reference in self.references)
|
|
1007
|
-
return not overridden.intersection(blocks) and (bool(blocks) or not extends)
|
|
1008
|
-
|
|
1009
|
-
@property
|
|
1010
|
-
def replaced(self) -> frozenset[str]:
|
|
1011
|
-
"""The blocks this template replaces in what it extends: those it defines
|
|
1012
|
-
without rendering ``{{ block.super }}``, which keeps the parent's content."""
|
|
1013
|
-
|
|
1014
|
-
return self.blocks - self.extended
|
|
1001
|
+
enclosing: Mapping[str, str | None] = field(default_factory=dict)
|
|
1015
1002
|
|
|
1016
1003
|
|
|
1017
1004
|
def project_templates(root: Path) -> ProjectTemplates:
|
|
@@ -1040,7 +1027,7 @@ def project_templates(root: Path) -> ProjectTemplates:
|
|
|
1040
1027
|
continue
|
|
1041
1028
|
read[path] = _read_scopes(text, SourceId(str(path)), libraries)
|
|
1042
1029
|
for reference in read[path].references:
|
|
1043
|
-
where = f"{relative}:{
|
|
1030
|
+
where = f"{relative}:{reference.span.start_line}"
|
|
1044
1031
|
if reference.name is None:
|
|
1045
1032
|
unread.append(f"{where} names a template by an expression")
|
|
1046
1033
|
elif reference.name not in files:
|
|
@@ -1048,16 +1035,15 @@ def project_templates(root: Path) -> ProjectTemplates:
|
|
|
1048
1035
|
for load in read[path].unresolved:
|
|
1049
1036
|
library = libraries.get(load.library)
|
|
1050
1037
|
reason = "not found" if library is None else "not read" if library.uncertain is None else "found in several files"
|
|
1051
|
-
unread.append(f"{relative}:{
|
|
1038
|
+
unread.append(f"{relative}:{load.span.start_line} loads {load.library!r}, {reason}")
|
|
1052
1039
|
calls = tuple(FilterCall(a.symbol, a.span) for template in read.values() for a in template.applications)
|
|
1053
|
-
#
|
|
1054
|
-
|
|
1055
|
-
reached: dict[tuple[str, frozenset[str]], _Walk] = {}
|
|
1040
|
+
# What each template reaches rendered on its own, for this analysis only.
|
|
1041
|
+
settled = _settled(files, read)
|
|
1056
1042
|
filters = {
|
|
1057
1043
|
name: tuple(
|
|
1058
1044
|
dict.fromkeys(
|
|
1059
1045
|
TemplateFilter(symbol, span, where, tuple(sorted(value)), tuple(sorted(argument)))
|
|
1060
|
-
for symbol, span, where, value, argument in
|
|
1046
|
+
for symbol, span, where, value, argument in settled.get(name, ())
|
|
1061
1047
|
)
|
|
1062
1048
|
)
|
|
1063
1049
|
for name in files
|
|
@@ -1070,70 +1056,245 @@ def project_templates(root: Path) -> ProjectTemplates:
|
|
|
1070
1056
|
_Reached = tuple[SymbolId, SourceSpan, str, frozenset[str], frozenset[str]]
|
|
1071
1057
|
|
|
1072
1058
|
|
|
1073
|
-
class
|
|
1074
|
-
"""What
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
for
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1059
|
+
class _Render(NamedTuple):
|
|
1060
|
+
"""What rendering a template on its own applies, and the templates it includes, each
|
|
1061
|
+
with its reference."""
|
|
1062
|
+
|
|
1063
|
+
applied: tuple[_Reached, ...]
|
|
1064
|
+
references: tuple[tuple[str, _Reference], ...]
|
|
1065
|
+
|
|
1066
|
+
|
|
1067
|
+
def _page(name: str, readable: Callable[[str | None], _Template | None]) -> _Render:
|
|
1068
|
+
"""What rendering ``name`` on its own applies and includes, its extends chain put
|
|
1069
|
+
together as Django puts it (#285).
|
|
1070
|
+
|
|
1071
|
+
The chain runs from ``name`` up to its root, the template extending none. The root's
|
|
1072
|
+
blocks standing in no other block are placed; a placed block renders its most derived
|
|
1073
|
+
definition, then, while a definition renders ``{{ block.super }}``, the next one up;
|
|
1074
|
+
the blocks a rendered definition holds are placed in turn. A filter or an include
|
|
1075
|
+
renders when the definition of the innermost block it stands in renders; outside
|
|
1076
|
+
every block, only in the root, Django ignoring a child's text outside its blocks.
|
|
1077
|
+
|
|
1078
|
+
When the chain reaches no known root, as through a parent named by an expression, a
|
|
1079
|
+
name several files have, a file the engine cannot read, or an extends cycle Django
|
|
1080
|
+
refuses, every block of the chain is placed: no filter is dropped on a placement the
|
|
1081
|
+
engine cannot establish.
|
|
1082
|
+
|
|
1083
|
+
What the chain applies is listed from ``name`` up to the root, and what it includes
|
|
1084
|
+
from the root down to ``name``: the order of a walk of each path through extends."""
|
|
1085
|
+
|
|
1086
|
+
chain: list[tuple[str, _Template]] = []
|
|
1087
|
+
seen: set[str] = set()
|
|
1088
|
+
known = True
|
|
1089
|
+
current: str | None = name
|
|
1090
|
+
while True:
|
|
1091
|
+
template = readable(current)
|
|
1092
|
+
if current is None or template is None or current in seen:
|
|
1093
|
+
known = False
|
|
1094
|
+
break
|
|
1095
|
+
chain.append((current, template))
|
|
1096
|
+
seen.add(current)
|
|
1097
|
+
parent = next((reference for reference in template.references if reference.extends), None)
|
|
1098
|
+
if parent is None:
|
|
1099
|
+
break
|
|
1100
|
+
current = parent.name
|
|
1101
|
+
defining: dict[str, list[int]] = {}
|
|
1102
|
+
held: dict[tuple[int, str | None], list[str]] = {}
|
|
1103
|
+
for level, (_, template) in enumerate(chain):
|
|
1104
|
+
for block in template.blocks:
|
|
1105
|
+
defining.setdefault(block, []).append(level)
|
|
1106
|
+
held.setdefault((level, template.enclosing.get(block)), []).append(block)
|
|
1107
|
+
pending = list(held.get((len(chain) - 1, None), ())) if known else list(defining)
|
|
1108
|
+
placed: set[str] = set()
|
|
1109
|
+
shown: set[tuple[int, str]] = set()
|
|
1110
|
+
while pending:
|
|
1111
|
+
block = pending.pop()
|
|
1112
|
+
if block in placed:
|
|
1122
1113
|
continue
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1114
|
+
placed.add(block)
|
|
1115
|
+
for level in defining.get(block, []):
|
|
1116
|
+
shown.add((level, block))
|
|
1117
|
+
pending.extend(held.get((level, block), ()))
|
|
1118
|
+
if block not in chain[level][1].extended:
|
|
1119
|
+
break
|
|
1120
|
+
|
|
1121
|
+
def renders(level: int, blocks: tuple[str, ...]) -> bool:
|
|
1122
|
+
return (level, blocks[-1]) in shown if blocks else known and level == len(chain) - 1
|
|
1123
|
+
|
|
1124
|
+
applied = tuple(
|
|
1125
|
+
(application.symbol, application.span, template_name, application.value, application.argument)
|
|
1126
|
+
for level, (template_name, template) in enumerate(chain)
|
|
1127
|
+
for application in template.applications
|
|
1128
|
+
if renders(level, application.blocks)
|
|
1129
|
+
)
|
|
1130
|
+
references = tuple(
|
|
1131
|
+
(reference.name, reference)
|
|
1132
|
+
for level in reversed(range(len(chain)))
|
|
1133
|
+
for reference in chain[level][1].references
|
|
1134
|
+
if not reference.extends
|
|
1135
|
+
and reference.name is not None
|
|
1136
|
+
and readable(reference.name) is not None
|
|
1137
|
+
and renders(level, reference.blocks)
|
|
1138
|
+
)
|
|
1139
|
+
return _Render(applied, references)
|
|
1140
|
+
|
|
1141
|
+
|
|
1142
|
+
def _renders(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> dict[str, _Render]:
|
|
1143
|
+
"""What rendering each template on its own applies and includes. A name several files
|
|
1144
|
+
have, or a file the engine could not read, renders nothing."""
|
|
1145
|
+
|
|
1146
|
+
def readable(name: str | None) -> _Template | None:
|
|
1147
|
+
paths = files.get(name or "", [])
|
|
1148
|
+
return read[paths[0]] if len(paths) == 1 and paths[0] in read else None
|
|
1149
|
+
|
|
1150
|
+
return {name: _page(name, readable) for name in files if readable(name) is not None}
|
|
1151
|
+
|
|
1152
|
+
|
|
1153
|
+
def _components(renders: Mapping[str, _Render]) -> list[list[str]]:
|
|
1154
|
+
"""The strongly connected components of the renders, each after every component its
|
|
1155
|
+
references reach: Tarjan's algorithm, without recursion."""
|
|
1156
|
+
|
|
1157
|
+
index: dict[str, int] = {}
|
|
1158
|
+
low: dict[str, int] = {}
|
|
1159
|
+
stack: list[str] = []
|
|
1160
|
+
on_stack: set[str] = set()
|
|
1161
|
+
components: list[list[str]] = []
|
|
1162
|
+
for root in renders:
|
|
1163
|
+
if root in index:
|
|
1128
1164
|
continue
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1165
|
+
# Each entry is a template and the position of the next reference to follow.
|
|
1166
|
+
work = [(root, 0)]
|
|
1167
|
+
while work:
|
|
1168
|
+
current, position = work.pop()
|
|
1169
|
+
if position == 0:
|
|
1170
|
+
index[current] = low[current] = len(index)
|
|
1171
|
+
stack.append(current)
|
|
1172
|
+
on_stack.add(current)
|
|
1173
|
+
references = renders[current].references
|
|
1174
|
+
for following in range(position, len(references)):
|
|
1175
|
+
below = references[following][0]
|
|
1176
|
+
if below not in index:
|
|
1177
|
+
work.append((current, following + 1))
|
|
1178
|
+
work.append((below, 0))
|
|
1179
|
+
break
|
|
1180
|
+
if below in on_stack:
|
|
1181
|
+
low[current] = min(low[current], index[below])
|
|
1182
|
+
else:
|
|
1183
|
+
if low[current] == index[current]:
|
|
1184
|
+
component: list[str] = []
|
|
1185
|
+
while not component or component[-1] != current:
|
|
1186
|
+
component.append(stack.pop())
|
|
1187
|
+
on_stack.discard(component[-1])
|
|
1188
|
+
components.append(component)
|
|
1189
|
+
if work:
|
|
1190
|
+
parent = work[-1][0]
|
|
1191
|
+
low[parent] = min(low[parent], low[current])
|
|
1192
|
+
return components
|
|
1193
|
+
|
|
1194
|
+
|
|
1195
|
+
def _mapped(found: Iterable[_Reached], reference: _Reference) -> Iterator[_Reached]:
|
|
1196
|
+
"""The filters a referenced template reaches, their names mapped through the reference."""
|
|
1197
|
+
|
|
1198
|
+
for symbol, span, where, value, argument in found:
|
|
1199
|
+
yield symbol, span, where, reference.maps(value), reference.maps(argument)
|
|
1200
|
+
|
|
1201
|
+
|
|
1202
|
+
def _canonical(found: _Reached) -> tuple[str, int, int, str, list[str], list[str]]:
|
|
1203
|
+
"""The order of the filters a template in a cycle reaches beyond what it applies."""
|
|
1204
|
+
|
|
1205
|
+
symbol, span, where, value, argument = found
|
|
1206
|
+
return where, span.start_line, span.start_column, str(symbol), sorted(value), sorted(argument)
|
|
1207
|
+
|
|
1208
|
+
|
|
1209
|
+
# A filter application: the function, where it is applied, and the template applying it.
|
|
1210
|
+
_Site = tuple[SymbolId, SourceSpan, str]
|
|
1211
|
+
|
|
1212
|
+
|
|
1213
|
+
def _joined(found: Iterable[_Reached], into: dict[_Site, tuple[frozenset[str], frozenset[str]]]) -> None:
|
|
1214
|
+
"""Join into ``into`` the names reaching the value and the argument of each filter
|
|
1215
|
+
application ``found`` holds."""
|
|
1216
|
+
|
|
1217
|
+
for symbol, span, where, value, argument in found:
|
|
1218
|
+
values, arguments = into.get((symbol, span, where), (frozenset(), frozenset()))
|
|
1219
|
+
into[symbol, span, where] = values | value, arguments | argument
|
|
1220
|
+
|
|
1221
|
+
|
|
1222
|
+
def _facts(names: Mapping[_Site, tuple[frozenset[str], frozenset[str]]], sites: Iterable[_Site]) -> list[_Reached]:
|
|
1223
|
+
"""The filters at ``sites``, each with the names joined at its application."""
|
|
1224
|
+
|
|
1225
|
+
return [(symbol, span, where, *names[symbol, span, where]) for symbol, span, where in sites]
|
|
1226
|
+
|
|
1227
|
+
|
|
1228
|
+
def _settled(files: Mapping[str, list[Path]], read: Mapping[Path, _Template]) -> dict[str, list[_Reached]]:
|
|
1229
|
+
"""The filters the render of each template reaches, in order (#284).
|
|
1230
|
+
|
|
1231
|
+
What a template reaches is the least set holding what it applies, its extends chain
|
|
1232
|
+
put together (#285), and what each template it includes reaches, mapped through the
|
|
1233
|
+
reference's bindings. The sets are computed
|
|
1234
|
+
component by component, each after those its references reach, which are then
|
|
1235
|
+
final. No cycle is cut: a template including itself with other bindings reaches what
|
|
1236
|
+
Django renders at its next levels.
|
|
1237
|
+
|
|
1238
|
+
In a cycle, the names reaching the value and the argument of each filter application
|
|
1239
|
+
are joined: one fact per application, whatever the paths that bring each name. The
|
|
1240
|
+
analysis reads those names position by position, joining what each brings, so the
|
|
1241
|
+
join loses nothing, and it keeps the facts few: bindings permuted around a cycle
|
|
1242
|
+
would otherwise make one fact of every combination of names. In a component, every
|
|
1243
|
+
template starts with no names and is recomputed from its references' facts; names
|
|
1244
|
+
only join, and the applications and names of a project are finitely many, so the
|
|
1245
|
+
names of an application grow at most once for each name. A template is recomputed
|
|
1246
|
+
when a fact it reads grew, until none grows.
|
|
1247
|
+
|
|
1248
|
+
Once the facts are settled, a template outside any cycle lists what it applies,
|
|
1249
|
+
then what each reference reaches, in order, first occurrences only, as a walk of
|
|
1250
|
+
each path would. A template in a cycle has no such first occurrence, the cycle
|
|
1251
|
+
having no start: it lists what it applies, then the rest by template, position,
|
|
1252
|
+
function and names. The order thus depends only on the facts and the templates,
|
|
1253
|
+
never on the order of the computation."""
|
|
1254
|
+
|
|
1255
|
+
renders = _renders(files, read)
|
|
1256
|
+
reached: dict[str, list[_Reached]] = {}
|
|
1257
|
+
for component in _components(renders):
|
|
1258
|
+
single = component[0]
|
|
1259
|
+
if len(component) == 1 and all(below != single for below, _ in renders[single].references):
|
|
1260
|
+
# Outside any cycle: what its references reach is final.
|
|
1261
|
+
found = dict.fromkeys(renders[single].applied)
|
|
1262
|
+
for below, reference in renders[single].references:
|
|
1263
|
+
found.update(dict.fromkeys(_mapped(reached[below], reference)))
|
|
1264
|
+
reached[single] = list(found)
|
|
1265
|
+
continue
|
|
1266
|
+
members = set(component)
|
|
1267
|
+
readers: dict[str, list[str]] = {member: [] for member in component}
|
|
1268
|
+
for member in component:
|
|
1269
|
+
for below, _ in renders[member].references:
|
|
1270
|
+
if below in members:
|
|
1271
|
+
readers[below].append(member)
|
|
1272
|
+
names: dict[str, dict[_Site, tuple[frozenset[str], frozenset[str]]]] = {member: {} for member in component}
|
|
1273
|
+
sizes = dict.fromkeys(component, 0)
|
|
1274
|
+
pending = deque(component)
|
|
1275
|
+
queued = set(component)
|
|
1276
|
+
while pending:
|
|
1277
|
+
member = pending.popleft()
|
|
1278
|
+
queued.discard(member)
|
|
1279
|
+
grown: dict[_Site, tuple[frozenset[str], frozenset[str]]] = {}
|
|
1280
|
+
_joined(renders[member].applied, grown)
|
|
1281
|
+
for below, reference in renders[member].references:
|
|
1282
|
+
below_found = _facts(names[below], names[below]) if below in members else reached[below]
|
|
1283
|
+
_joined(_mapped(below_found, reference), grown)
|
|
1284
|
+
# The names only join, so the count of applications and names only grows.
|
|
1285
|
+
size = sum(1 + len(values) + len(arguments) for values, arguments in grown.values())
|
|
1286
|
+
if size > sizes[member]:
|
|
1287
|
+
names[member], sizes[member] = grown, size
|
|
1288
|
+
for reader in readers[member]:
|
|
1289
|
+
if reader not in queued:
|
|
1290
|
+
pending.append(reader)
|
|
1291
|
+
queued.add(reader)
|
|
1292
|
+
for member in component:
|
|
1293
|
+
applied = _facts(names[member], dict.fromkeys((s, span, w) for s, span, w, _, _ in renders[member].applied))
|
|
1294
|
+
sites = {(s, span, w) for s, span, w, _, _ in applied}
|
|
1295
|
+
rest = _facts(names[member], (site for site in names[member] if site not in sites))
|
|
1296
|
+
reached[member] = [*applied, *sorted(rest, key=_canonical)]
|
|
1297
|
+
return reached
|
|
1137
1298
|
|
|
1138
1299
|
|
|
1139
1300
|
def _read_scopes(text: str, source: SourceId, libraries: Mapping[str, _Library]) -> _Template:
|
|
@@ -1149,10 +1310,16 @@ def _read_scopes(text: str, source: SourceId, libraries: Mapping[str, _Library])
|
|
|
1149
1310
|
opened: list[str] = []
|
|
1150
1311
|
defined: set[str] = set()
|
|
1151
1312
|
extended: set[str] = set()
|
|
1313
|
+
enclosing: dict[str, str | None] = {}
|
|
1152
1314
|
scopes: list[dict[str, frozenset[str]]] = [{}]
|
|
1153
1315
|
blocks: list[tuple[list[tuple[str, int, frozenset[str]]], set[str]]] = []
|
|
1154
1316
|
loaded: list[_Load] = []
|
|
1155
1317
|
unresolved: dict[_Load, None] = {}
|
|
1318
|
+
lines = LineIndex(text)
|
|
1319
|
+
|
|
1320
|
+
def span(offset: int) -> SourceSpan:
|
|
1321
|
+
line, column = lines.position(offset)
|
|
1322
|
+
return SourceSpan(source, line, column)
|
|
1156
1323
|
|
|
1157
1324
|
def free(names: frozenset[str]) -> frozenset[str]:
|
|
1158
1325
|
found: set[str] = set()
|
|
@@ -1192,9 +1359,7 @@ def _read_scopes(text: str, source: SourceId, libraries: Mapping[str, _Library])
|
|
|
1192
1359
|
if isinstance(symbol, _Load):
|
|
1193
1360
|
unresolved[symbol] = None
|
|
1194
1361
|
elif symbol is not None:
|
|
1195
|
-
|
|
1196
|
-
span = SourceSpan(source, line, column)
|
|
1197
|
-
applications.append(_Application(symbol, span, value, free(argument), tuple(opened)))
|
|
1362
|
+
applications.append(_Application(symbol, span(offset), value, free(argument), tuple(opened)))
|
|
1198
1363
|
value |= free(argument)
|
|
1199
1364
|
return value
|
|
1200
1365
|
|
|
@@ -1234,6 +1399,7 @@ def _read_scopes(text: str, source: SourceId, libraries: Mapping[str, _Library])
|
|
|
1234
1399
|
else:
|
|
1235
1400
|
values.setdefault(word, reached)
|
|
1236
1401
|
if name == "block" and len(words) > 1:
|
|
1402
|
+
enclosing.setdefault(words[1], opened[-1] if opened else None)
|
|
1237
1403
|
opened.append(words[1])
|
|
1238
1404
|
defined.add(words[1])
|
|
1239
1405
|
elif name == "endblock" and opened:
|
|
@@ -1251,18 +1417,18 @@ def _read_scopes(text: str, source: SourceId, libraries: Mapping[str, _Library])
|
|
|
1251
1417
|
elif name in ("include", "extends"):
|
|
1252
1418
|
named = _constant(words[1]) if len(words) > 1 else None
|
|
1253
1419
|
if name == "extends":
|
|
1254
|
-
references.append(_Reference(named, offset, extends=True))
|
|
1420
|
+
references.append(_Reference(named, span(offset), extends=True))
|
|
1255
1421
|
else:
|
|
1256
1422
|
context = {k: v for scope in scopes for k, v in scope.items()}
|
|
1257
|
-
references.append(_Reference(named, offset, bindings, "only" in words, context, tuple(opened)))
|
|
1423
|
+
references.append(_Reference(named, span(offset), bindings, "only" in words, context, tuple(opened)))
|
|
1258
1424
|
elif name == "load":
|
|
1259
1425
|
selected = frozenset(words[1 : words.index("from")]) if "from" in words else None
|
|
1260
|
-
loaded.extend(_Load(library, selected, offset) for library in _loaded(words[1:]))
|
|
1426
|
+
loaded.extend(_Load(library, selected, span(offset)) for library in _loaded(words[1:]))
|
|
1261
1427
|
elif "as" in words[1:-1]:
|
|
1262
1428
|
target = words[words.index("as", 1) + 1]
|
|
1263
1429
|
scopes[-1][target] = values.get(words[1], frozenset()) if name == "regroup" else frozenset()
|
|
1264
1430
|
return _Template(
|
|
1265
|
-
tuple(applications), tuple(references), frozenset(defined), frozenset(extended), tuple(unresolved)
|
|
1431
|
+
tuple(applications), tuple(references), frozenset(defined), frozenset(extended), tuple(unresolved), enclosing
|
|
1266
1432
|
)
|
|
1267
1433
|
|
|
1268
1434
|
|
|
@@ -1541,12 +1707,6 @@ def _expressions(text: str) -> Iterator[tuple[int, str, bool]]:
|
|
|
1541
1707
|
yield match.start(2), tag, True
|
|
1542
1708
|
|
|
1543
1709
|
|
|
1544
|
-
def _position(text: str, offset: int) -> tuple[int, int]:
|
|
1545
|
-
"""The one-based line and column of ``offset`` in ``text``."""
|
|
1546
|
-
|
|
1547
|
-
return text.count("\n", 0, offset) + 1, offset - text.rfind("\n", 0, offset)
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
1710
|
class EscapedTemplates(Analysis[frozenset[str]]):
|
|
1551
1711
|
"""The names of the project templates that escape everything they render, provided
|
|
1552
1712
|
by the engine for a project; none on its own."""
|
{coretrace_python_analyzer-0.24.0.dist-info → coretrace_python_analyzer-0.25.0.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: coretrace-python-analyzer
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.25.0
|
|
4
4
|
Summary: Static security analysis infrastructure for Python code
|
|
5
5
|
Project-URL: Homepage, https://github.com/CoreTrace/coretrace-python-analyzer
|
|
6
6
|
Project-URL: Repository, https://github.com/CoreTrace/coretrace-python-analyzer
|
{coretrace_python_analyzer-0.24.0.dist-info → coretrace_python_analyzer-0.25.0.dist-info}/RECORD
RENAMED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
coretrace_python/__init__.py,sha256=
|
|
1
|
+
coretrace_python/__init__.py,sha256=V3KQfEFIUS0wMCMwUjwFcE6mvYiqBT1my4ZjJcqpzEY,76
|
|
2
2
|
coretrace_python/__main__.py,sha256=V8DnYtRboISK2VCKm7DGHLREKvqLbqApyWlKBOs26wU,65
|
|
3
3
|
coretrace_python/cache.py,sha256=bFeK4apcvL4-XqASyqToiHiNZquFVM4dLjdDxUDXjTo,15556
|
|
4
4
|
coretrace_python/cli.py,sha256=G9l03JpIqyjoRmDtwa2ZQZaDpeS4nX3pTzZtRRyBVfs,13972
|
|
@@ -143,20 +143,20 @@ coretrace_python/semantic/identity.py,sha256=uQBdqEXrOtYmGIiR9hZu0Pimv52iW1rOOOv
|
|
|
143
143
|
coretrace_python/semantic/imports.py,sha256=389MZs8iTUc3CyUHG0VAgWFsypd7VPFKkLHmFz9L-N0,5673
|
|
144
144
|
coretrace_python/semantic/scopes.py,sha256=vrFnEC1oX4UoAJHApY1fPH920ZEkGY0Htl8XMpQvSKk,22412
|
|
145
145
|
coretrace_python/semantic/symbols.py,sha256=z0HQJyX2mlKGzMWzZAKgnUdNfzW2qu0oqv3dIZk4gpg,7454
|
|
146
|
-
coretrace_python/source/__init__.py,sha256=
|
|
147
|
-
coretrace_python/source/manager.py,sha256=
|
|
146
|
+
coretrace_python/source/__init__.py,sha256=mwgLk_1n5K8LDRhj7AvNqcFr7_DGRJTZYXF1CeW8MPE,529
|
|
147
|
+
coretrace_python/source/manager.py,sha256=1w9mGG8EDkX37lGw8l5pQnQICD9kYhLLpPZJTGRLkI8,6336
|
|
148
148
|
coretrace_python/source/model.py,sha256=V9wa7wEzU1IWNf4DlSlFZiAcj_46dbOsBCMQPVU8lv4,2866
|
|
149
|
-
coretrace_python/source/positions.py,sha256=
|
|
149
|
+
coretrace_python/source/positions.py,sha256=jb0ll5Sm9DCF7tk6ovY-KGLfPDV6h2WV7L0Z11Wbfu8,16038
|
|
150
150
|
coretrace_python/taint/__init__.py,sha256=KPFK0SFGgohbGVVQDPSCkNXtdz9XUlfBuYko5QFYNh0,1915
|
|
151
151
|
coretrace_python/taint/commands.py,sha256=HBQbSu0YfPQ1qlF8twWp2HQKKc1iOvBzkOVZTUFvakE,1787
|
|
152
152
|
coretrace_python/taint/engine.py,sha256=43QQ-XrCTx4QiX1D2t8tqErxTlnaz11U2QSw486JCE0,58874
|
|
153
153
|
coretrace_python/taint/models.py,sha256=EmnD6DyMznb2GZ0JYyW2D9PBm0I9Q9-mlkCB2PwZYt0,22432
|
|
154
154
|
coretrace_python/taint/routes.py,sha256=UAJKXcw_sM9shf58FfDroxVO9bNSm4qrQdZknwUB0lo,6110
|
|
155
|
-
coretrace_python/taint/templates.py,sha256=
|
|
155
|
+
coretrace_python/taint/templates.py,sha256=7D8AWWHsCoNg6WkYy3sMMEtHBC69yrWMh28QB-UkjOU,80631
|
|
156
156
|
coretrace_python/taint/urls.py,sha256=kb0-uOT12ww7R3Jzs_lPKg9lFs74JI-bpa9pXX105xM,3166
|
|
157
|
-
coretrace_python_analyzer-0.
|
|
158
|
-
coretrace_python_analyzer-0.
|
|
159
|
-
coretrace_python_analyzer-0.
|
|
160
|
-
coretrace_python_analyzer-0.
|
|
161
|
-
coretrace_python_analyzer-0.
|
|
162
|
-
coretrace_python_analyzer-0.
|
|
157
|
+
coretrace_python_analyzer-0.25.0.dist-info/METADATA,sha256=iVQGQatVaXcVVSRaOtGKBvtgyOqODZpHoKhUnGBfGdY,3431
|
|
158
|
+
coretrace_python_analyzer-0.25.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
159
|
+
coretrace_python_analyzer-0.25.0.dist-info/entry_points.txt,sha256=wdozemiraOkbt65g_Y_zgspuZpoLxJdaCVMJt1Ft3c0,72
|
|
160
|
+
coretrace_python_analyzer-0.25.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
|
|
161
|
+
coretrace_python_analyzer-0.25.0.dist-info/licenses/NOTICE,sha256=s3VHj5ngV0ozjW_kZJpg_35ne6JwoBK0ysOjbqVtA0w,113
|
|
162
|
+
coretrace_python_analyzer-0.25.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|