code-node-graph 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- code_node/__init__.py +40 -0
- code_node/__main__.py +7 -0
- code_node/analysis/__init__.py +3 -0
- code_node/analysis/aws_model.py +197 -0
- code_node/analysis/findings.py +462 -0
- code_node/analysis/graph.py +780 -0
- code_node/analysis/mapping.py +114 -0
- code_node/analysis/model.py +289 -0
- code_node/analysis/reader.py +1926 -0
- code_node/analysis/syntax.py +220 -0
- code_node/analysis/terms.py +505 -0
- code_node/cli.py +316 -0
- code_node/deploys/__init__.py +321 -0
- code_node/deploys/sam.py +371 -0
- code_node/deploys/serverless.py +117 -0
- code_node/kit/__init__.py +4 -0
- code_node/kit/blocks.py +184 -0
- code_node/kit/fmt.py +389 -0
- code_node/kit/highlight.py +131 -0
- code_node/kit/render.py +1066 -0
- code_node/kit/text.py +81 -0
- code_node/libraries/__init__.py +333 -0
- code_node/libraries/llm.py +124 -0
- code_node/libraries/mongo.py +169 -0
- code_node/libraries/redis.py +253 -0
- code_node/libraries/sql.py +232 -0
- code_node/pages.py +92 -0
- code_node/platforms/__init__.py +203 -0
- code_node/platforms/aws_lambda.py +290 -0
- code_node/platforms/web.py +301 -0
- code_node/project.py +402 -0
- code_node/report/__init__.py +3 -0
- code_node/report/brands.py +244 -0
- code_node/report/build.py +119 -0
- code_node/report/editor.py +40 -0
- code_node/report/graph.py +1411 -0
- code_node/report/logos/README.md +8 -0
- code_node/report/logos/anthropic-icon.svg +1 -0
- code_node/report/logos/aws-amplify.svg +1 -0
- code_node/report/logos/aws-api-gateway.svg +1 -0
- code_node/report/logos/aws-appflow.svg +1 -0
- code_node/report/logos/aws-appsync.svg +1 -0
- code_node/report/logos/aws-athena.svg +1 -0
- code_node/report/logos/aws-backup.svg +1 -0
- code_node/report/logos/aws-batch.svg +1 -0
- code_node/report/logos/aws-certificate-manager.svg +1 -0
- code_node/report/logos/aws-cloudformation.svg +1 -0
- code_node/report/logos/aws-cloudfront.svg +1 -0
- code_node/report/logos/aws-cloudtrail.svg +1 -0
- code_node/report/logos/aws-cloudwatch.svg +1 -0
- code_node/report/logos/aws-codebuild.svg +1 -0
- code_node/report/logos/aws-codecommit.svg +1 -0
- code_node/report/logos/aws-codedeploy.svg +1 -0
- code_node/report/logos/aws-codepipeline.svg +1 -0
- code_node/report/logos/aws-cognito.svg +1 -0
- code_node/report/logos/aws-config.svg +1 -0
- code_node/report/logos/aws-documentdb.svg +1 -0
- code_node/report/logos/aws-dynamodb.svg +1 -0
- code_node/report/logos/aws-ec2.svg +1 -0
- code_node/report/logos/aws-ecs.svg +1 -0
- code_node/report/logos/aws-eks.svg +1 -0
- code_node/report/logos/aws-elasticache.svg +1 -0
- code_node/report/logos/aws-elb.svg +1 -0
- code_node/report/logos/aws-eventbridge.svg +1 -0
- code_node/report/logos/aws-glacier.svg +1 -0
- code_node/report/logos/aws-glue.svg +1 -0
- code_node/report/logos/aws-iam.svg +1 -0
- code_node/report/logos/aws-keyspaces.svg +1 -0
- code_node/report/logos/aws-kinesis.svg +1 -0
- code_node/report/logos/aws-kms.svg +1 -0
- code_node/report/logos/aws-lake-formation.svg +1 -0
- code_node/report/logos/aws-lambda.svg +1 -0
- code_node/report/logos/aws-lightsail.svg +1 -0
- code_node/report/logos/aws-mq.svg +1 -0
- code_node/report/logos/aws-msk.svg +1 -0
- code_node/report/logos/aws-neptune.svg +1 -0
- code_node/report/logos/aws-open-search.svg +1 -0
- code_node/report/logos/aws-quicksight.svg +1 -0
- code_node/report/logos/aws-rds.svg +1 -0
- code_node/report/logos/aws-redshift.svg +1 -0
- code_node/report/logos/aws-route53.svg +1 -0
- code_node/report/logos/aws-s3.svg +1 -0
- code_node/report/logos/aws-secrets-manager.svg +1 -0
- code_node/report/logos/aws-ses.svg +1 -0
- code_node/report/logos/aws-shield.svg +1 -0
- code_node/report/logos/aws-sns.svg +1 -0
- code_node/report/logos/aws-sqs.svg +1 -0
- code_node/report/logos/aws-step-functions.svg +1 -0
- code_node/report/logos/aws-systems-manager.svg +1 -0
- code_node/report/logos/aws-timestream.svg +1 -0
- code_node/report/logos/aws-waf.svg +1 -0
- code_node/report/logos/aws-xray.svg +1 -0
- code_node/report/logos/aws.svg +1 -0
- code_node/report/logos/azure-icon.svg +1 -0
- code_node/report/logos/django-icon.svg +1 -0
- code_node/report/logos/elasticsearch.svg +1 -0
- code_node/report/logos/fastapi-icon.svg +1 -0
- code_node/report/logos/firebase-icon.svg +1 -0
- code_node/report/logos/flask.svg +1 -0
- code_node/report/logos/google-cloud.svg +1 -0
- code_node/report/logos/kafka-icon.svg +1 -0
- code_node/report/logos/mariadb-icon.svg +1 -0
- code_node/report/logos/mongodb-icon.svg +1 -0
- code_node/report/logos/mysql-icon.svg +1 -0
- code_node/report/logos/openai-icon.svg +1 -0
- code_node/report/logos/postgresql.svg +1 -0
- code_node/report/logos/python.svg +1 -0
- code_node/report/logos/rabbitmq-icon.svg +1 -0
- code_node/report/logos/redis.svg +1 -0
- code_node/report/logos/serverless.svg +1 -0
- code_node/report/logos/slack-icon.svg +1 -0
- code_node/report/logos/small/aws-api-gateway.svg +1 -0
- code_node/report/logos/small/aws-appflow.svg +1 -0
- code_node/report/logos/small/aws-appsync.svg +1 -0
- code_node/report/logos/small/aws-athena.svg +1 -0
- code_node/report/logos/small/aws-backup.svg +1 -0
- code_node/report/logos/small/aws-batch.svg +1 -0
- code_node/report/logos/small/aws-certificate-manager.svg +1 -0
- code_node/report/logos/small/aws-cloudformation.svg +1 -0
- code_node/report/logos/small/aws-cloudfront.svg +1 -0
- code_node/report/logos/small/aws-cloudtrail.svg +1 -0
- code_node/report/logos/small/aws-cloudwatch.svg +1 -0
- code_node/report/logos/small/aws-codebuild.svg +1 -0
- code_node/report/logos/small/aws-codecommit.svg +1 -0
- code_node/report/logos/small/aws-codedeploy.svg +1 -0
- code_node/report/logos/small/aws-codepipeline.svg +1 -0
- code_node/report/logos/small/aws-cognito.svg +1 -0
- code_node/report/logos/small/aws-config.svg +1 -0
- code_node/report/logos/small/aws-documentdb.svg +1 -0
- code_node/report/logos/small/aws-dynamodb.svg +1 -0
- code_node/report/logos/small/aws-ec2.svg +1 -0
- code_node/report/logos/small/aws-ecs.svg +1 -0
- code_node/report/logos/small/aws-eks.svg +1 -0
- code_node/report/logos/small/aws-elasticache.svg +1 -0
- code_node/report/logos/small/aws-elb.svg +1 -0
- code_node/report/logos/small/aws-eventbridge.svg +1 -0
- code_node/report/logos/small/aws-glacier.svg +1 -0
- code_node/report/logos/small/aws-glue.svg +1 -0
- code_node/report/logos/small/aws-iam.svg +1 -0
- code_node/report/logos/small/aws-keyspaces.svg +1 -0
- code_node/report/logos/small/aws-kinesis.svg +1 -0
- code_node/report/logos/small/aws-kms.svg +1 -0
- code_node/report/logos/small/aws-lake-formation.svg +1 -0
- code_node/report/logos/small/aws-lambda.svg +1 -0
- code_node/report/logos/small/aws-mq.svg +1 -0
- code_node/report/logos/small/aws-msk.svg +1 -0
- code_node/report/logos/small/aws-neptune.svg +1 -0
- code_node/report/logos/small/aws-open-search.svg +1 -0
- code_node/report/logos/small/aws-quicksight.svg +1 -0
- code_node/report/logos/small/aws-rds.svg +1 -0
- code_node/report/logos/small/aws-redshift.svg +1 -0
- code_node/report/logos/small/aws-route53.svg +1 -0
- code_node/report/logos/small/aws-s3.svg +1 -0
- code_node/report/logos/small/aws-secrets-manager.svg +1 -0
- code_node/report/logos/small/aws-ses.svg +1 -0
- code_node/report/logos/small/aws-shield.svg +1 -0
- code_node/report/logos/small/aws-sns.svg +1 -0
- code_node/report/logos/small/aws-sqs.svg +1 -0
- code_node/report/logos/small/aws-step-functions.svg +1 -0
- code_node/report/logos/small/aws-systems-manager.svg +1 -0
- code_node/report/logos/small/aws-timestream.svg +1 -0
- code_node/report/logos/small/aws-waf.svg +1 -0
- code_node/report/logos/small/aws-xray.svg +1 -0
- code_node/report/logos/snowflake-icon.svg +1 -0
- code_node/report/logos/sqlite.svg +1 -0
- code_node/report/logos/stripe.svg +1 -0
- code_node/report/logos/supabase-icon.svg +1 -0
- code_node/report/logos/twilio-icon.svg +1 -0
- code_node/report/outline.py +219 -0
- code_node/report/style.py +358 -0
- code_node/report/tabs.py +73 -0
- code_node/report/walk.py +624 -0
- code_node/serve.py +277 -0
- code_node_graph-0.1.0.dist-info/METADATA +165 -0
- code_node_graph-0.1.0.dist-info/RECORD +178 -0
- code_node_graph-0.1.0.dist-info/WHEEL +4 -0
- code_node_graph-0.1.0.dist-info/entry_points.txt +2 -0
- code_node_graph-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,1926 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The code reader: loads the entry point's module the way Python imports it and follows the
|
|
3
|
+
entry point, keeping what each name holds, into a CodeMap.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import ast
|
|
9
|
+
import sys
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from typing import Any, Generator, Iterable
|
|
13
|
+
|
|
14
|
+
from ..kit.text import _clip
|
|
15
|
+
|
|
16
|
+
from .terms import _JUNK_RE
|
|
17
|
+
from .terms import (
|
|
18
|
+
_ADDED_MODULES,
|
|
19
|
+
_AWS_KEY_RE,
|
|
20
|
+
_B64_DECODES,
|
|
21
|
+
_CACHES,
|
|
22
|
+
_COLLECTIONS,
|
|
23
|
+
_COMMANDS,
|
|
24
|
+
_DEFAULT_TIMEOUTS,
|
|
25
|
+
_DICT_METHODS,
|
|
26
|
+
_FILE_WRITERS,
|
|
27
|
+
_GUESSES,
|
|
28
|
+
_HTTP_METHODS,
|
|
29
|
+
_HTTP_SESSIONS,
|
|
30
|
+
_JSON_LOADS,
|
|
31
|
+
_LOG_LEVELS,
|
|
32
|
+
_MAP_DEPTH,
|
|
33
|
+
_MAP_FILES,
|
|
34
|
+
_MAP_RUNS,
|
|
35
|
+
_MAP_STEPS,
|
|
36
|
+
_REMOVED_MODULES,
|
|
37
|
+
_RESOURCE_ACTIONS,
|
|
38
|
+
_ROUTE_METHODS,
|
|
39
|
+
_RUNTIME_MODULES,
|
|
40
|
+
_TRANSFERS,
|
|
41
|
+
_URL_INCLUDES,
|
|
42
|
+
_URL_PATHS,
|
|
43
|
+
_VIEW_METHODS,
|
|
44
|
+
_VIEW_VERBS,
|
|
45
|
+
_VIEWSET_ACTIONS,
|
|
46
|
+
)
|
|
47
|
+
from .syntax import (
|
|
48
|
+
_a,
|
|
49
|
+
_const,
|
|
50
|
+
_def_line,
|
|
51
|
+
_doc_line,
|
|
52
|
+
_ends_early,
|
|
53
|
+
_is_main_guard,
|
|
54
|
+
_names_in,
|
|
55
|
+
_negated,
|
|
56
|
+
_parse,
|
|
57
|
+
_secret_spots,
|
|
58
|
+
_swallows,
|
|
59
|
+
_target,
|
|
60
|
+
_unparse,
|
|
61
|
+
)
|
|
62
|
+
from .aws_model import _access, _aws_model, _event_label, _iam_action, _service_name
|
|
63
|
+
from .model import AwsCall, CodeFunction, CodeMap, CodeSpot, CodeStep, EnvRead, EventField, ImportUse, LibraryCall
|
|
64
|
+
from ..libraries import Call, Handle, Library, Use, _Argument, get_library, library_for
|
|
65
|
+
from ..platforms import get_platform
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
_CLI_COMMANDS = ("command", "callback") # @app.command(): a Typer or click command, run like a route
|
|
69
|
+
|
|
70
|
+
_PROPERTIES = ("property", "cached_property") # a method run when its attribute is read: obj.size
|
|
71
|
+
|
|
72
|
+
_ACCESSORS = ("setter", "deleter", "getter") # @size.setter: another method by the property's name, not its getter
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _is_text(node: Any) -> bool:
|
|
76
|
+
"""
|
|
77
|
+
Whether an expression is text written in the code or put together there: a literal, an f-string, or + / % on
|
|
78
|
+
one.
|
|
79
|
+
"""
|
|
80
|
+
if isinstance(_const(node), str) or isinstance(node, ast.JoinedStr):
|
|
81
|
+
return True
|
|
82
|
+
return (
|
|
83
|
+
isinstance(node, ast.BinOp)
|
|
84
|
+
and isinstance(node.op, (ast.Add, ast.Mod))
|
|
85
|
+
and (_is_text(node.left) or _is_text(node.right))
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _parse_text(code: str) -> Any:
|
|
90
|
+
try:
|
|
91
|
+
return ast.parse(code, mode="eval").body
|
|
92
|
+
except SyntaxError:
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass(frozen=True)
|
|
97
|
+
class _Sym:
|
|
98
|
+
"""
|
|
99
|
+
What a name holds, as far as how_it_works() can tell without running the code.
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
kind: str # 'mod' (an outside module or name, dotted), 'own' (one of its own modules), 'func', 'class', 'obj' (an
|
|
103
|
+
# instance of one of its classes), 'bound' (one's method), 'client', 'resource', 'res' (a resource object: a
|
|
104
|
+
# Table), 'coll' (a resource collection), 'paginator', 'batch', 'method' (one of theirs), 'meta', 'event',
|
|
105
|
+
# 'evmethod', 'context', 'env' (os.environ), 'envget', 'envval', 'logger', 'log', 'http', 'httpcall', 'builtin',
|
|
106
|
+
# 'enumerate', 'session' (a boto3 Session), 'super' (super() in a method of class `name`, its self or cls as inner),
|
|
107
|
+
# 'lib' (a library plugin's handle: name is the plugin), 'libmember' (one's method `sub`, the handle as inner),
|
|
108
|
+
# 'text' (text the code put together: sub is its code, path what from the input went into it), 'urls' (Django's
|
|
109
|
+
# include(...): name is the routes' router, the URL conf's module or 'module.router')
|
|
110
|
+
name: str = ""
|
|
111
|
+
sub: str = ""
|
|
112
|
+
path: tuple[str, ...] = ()
|
|
113
|
+
inner: _Sym | None = None
|
|
114
|
+
handle: Handle | None = None
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@dataclass
|
|
118
|
+
class _Frame:
|
|
119
|
+
"""
|
|
120
|
+
A function (or a module's top level) being read: what its names hold, and where it is.
|
|
121
|
+
"""
|
|
122
|
+
|
|
123
|
+
module: str
|
|
124
|
+
file: str
|
|
125
|
+
function: str # 'etl.handler', or the module ('etl') at its top level
|
|
126
|
+
scope: dict[str, _Sym | None]
|
|
127
|
+
node: Any = None # the function's ast node; None at a module's top level
|
|
128
|
+
outers: list[dict[str, _Sym | None]] = field(default_factory=list) # the functions around it (closures)
|
|
129
|
+
loops: list[str] = field(default_factory=list) # the loops it's in, its callers' first
|
|
130
|
+
guards: list[str] = field(default_factory=list) # the tests of the ifs it's in
|
|
131
|
+
early: set[int] = field(default_factory=set) # which of guards come from an `if x: return` above, not around it
|
|
132
|
+
globals: set[str] = field(default_factory=set)
|
|
133
|
+
optional: int = 0 # inside a try that catches ImportError
|
|
134
|
+
cached: bool = False # @lru_cache and the like: it runs once
|
|
135
|
+
returns: list[_Sym | None] = field(default_factory=list)
|
|
136
|
+
depth: int = 0
|
|
137
|
+
words: set[str] | None = None # every name and text in it, for spotting a next-page token
|
|
138
|
+
|
|
139
|
+
@property
|
|
140
|
+
def top(self) -> bool:
|
|
141
|
+
return self.node is None
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@dataclass
|
|
145
|
+
class _Module:
|
|
146
|
+
name: str
|
|
147
|
+
file: str | None
|
|
148
|
+
tree: Any = None
|
|
149
|
+
scope: dict[str, _Sym | None] = field(default_factory=dict)
|
|
150
|
+
steps: list[CodeStep] = field(default_factory=list)
|
|
151
|
+
loaded: bool = False
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@dataclass
|
|
155
|
+
class _Collect:
|
|
156
|
+
"""
|
|
157
|
+
What one statement does, gathered while its expressions are read.
|
|
158
|
+
"""
|
|
159
|
+
|
|
160
|
+
notes: list[tuple[str, str]] = field(default_factory=list)
|
|
161
|
+
calls: list[CodeStep] = field(default_factory=list)
|
|
162
|
+
holds: list[tuple[str, str]] = field(default_factory=list) # a name it sets -> what that holds, in words
|
|
163
|
+
given: list[tuple[str, str]] = field(default_factory=list) # an AWS call's argument -> where it comes from
|
|
164
|
+
|
|
165
|
+
def note(self, kind: str, text: str) -> None:
|
|
166
|
+
if (kind, text) not in self.notes:
|
|
167
|
+
self.notes.append((kind, text))
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class _CodeReader:
|
|
171
|
+
"""
|
|
172
|
+
Follows a Python function's code from its entry point with the ast module, never running it. It keeps track of what
|
|
173
|
+
each name holds (a boto3 client, part of its input, one of its own functions, an environment variable...) as far
|
|
174
|
+
as it can tell, so it knows what each line calls, and fills a CodeMap. One per map_code().
|
|
175
|
+
"""
|
|
176
|
+
|
|
177
|
+
def __init__(
|
|
178
|
+
self,
|
|
179
|
+
texts: dict[str, str],
|
|
180
|
+
names: list[str],
|
|
181
|
+
root: str,
|
|
182
|
+
cmap: CodeMap,
|
|
183
|
+
version: tuple[int, int] | None,
|
|
184
|
+
libraries: set[str],
|
|
185
|
+
layers: bool,
|
|
186
|
+
):
|
|
187
|
+
self.texts, self.root, self.cmap, self.version, self.layers = texts, root, cmap, version, layers
|
|
188
|
+
self.platform = get_platform(cmap.platform)
|
|
189
|
+
self.tops: set[str] = set() # what's at the package's top: folders and modules
|
|
190
|
+
for name in names:
|
|
191
|
+
if name.startswith(root):
|
|
192
|
+
first, slash, _ = name[len(root) :].partition("/")
|
|
193
|
+
self.tops.add(first if slash else first.split(".")[0])
|
|
194
|
+
self.libraries = libraries
|
|
195
|
+
self.sources: dict[str, str] = {} # its own modules: name -> file
|
|
196
|
+
for path in texts:
|
|
197
|
+
if not path.startswith(root) or not path.endswith(".py"):
|
|
198
|
+
continue
|
|
199
|
+
rel = path[len(root) : -3]
|
|
200
|
+
module = rel.replace("/", ".")
|
|
201
|
+
if module.split(".")[0] in libraries or "-" in module or "site-packages" in module:
|
|
202
|
+
continue
|
|
203
|
+
module = module[: -len(".__init__")] if module.endswith(".__init__") else module
|
|
204
|
+
if module and module != "__init__":
|
|
205
|
+
self.sources[module] = path
|
|
206
|
+
self.modules: dict[str, _Module] = {}
|
|
207
|
+
self.defs: dict[str, tuple[str, Any, str | None]] = {} # key -> (module, node, class key)
|
|
208
|
+
self.classes: dict[str, dict[str, Any]] = {}
|
|
209
|
+
self.closures: dict[str, list[dict[str, _Sym | None]]] = {}
|
|
210
|
+
self.cached: set[str] = set()
|
|
211
|
+
self.cur = _Collect()
|
|
212
|
+
self.stack: list[str] = []
|
|
213
|
+
self.memo: dict[Any, _Sym | None] = {}
|
|
214
|
+
self.shown: set[str] = set()
|
|
215
|
+
self.reached: dict[str, CodeFunction] = {}
|
|
216
|
+
self.count = 0
|
|
217
|
+
self.routes: list[tuple[str, str, str, str]] = [] # (method, path, function key, router)
|
|
218
|
+
self.event_reads: dict[tuple[str, ...], EventField] = {}
|
|
219
|
+
self.seen: set[Any] = set()
|
|
220
|
+
self.target: set[str] = set() # the names the statement being read assigns to
|
|
221
|
+
self.loading = 0 # modules being loaded: what runs now runs when Python imports the code
|
|
222
|
+
self.runs = 0 # function bodies read
|
|
223
|
+
self.max_depth = _MAP_DEPTH # calls followed from the entry point
|
|
224
|
+
self.keywords: tuple[Any, dict[str, _Sym | None]] = (None, {}) # the call being read, and its keywords' values
|
|
225
|
+
self.values: tuple[Any, dict[int, _Sym | None]] = (None, {}) # ...and every argument's, by the node's id
|
|
226
|
+
|
|
227
|
+
# ---------------------------------------------------------------- reading
|
|
228
|
+
|
|
229
|
+
@contextmanager
|
|
230
|
+
def _collecting(self) -> Generator[_Collect, None, None]:
|
|
231
|
+
saved, self.cur = self.cur, _Collect()
|
|
232
|
+
try:
|
|
233
|
+
yield self.cur
|
|
234
|
+
finally:
|
|
235
|
+
self.cur = saved
|
|
236
|
+
|
|
237
|
+
def read(self, handler: str) -> None:
|
|
238
|
+
cmap = self.cmap
|
|
239
|
+
module, _, name = handler.rpartition(".")
|
|
240
|
+
module = module.replace("/", ".")
|
|
241
|
+
cls_name = ""
|
|
242
|
+
if module not in self.sources and module.rpartition(".")[0] in self.sources:
|
|
243
|
+
module, _, cls_name = module.rpartition(".")
|
|
244
|
+
mod = self._load_chain(module)
|
|
245
|
+
cmap.startup = [step for m in self._chain(module) if m.name != module for step in self._module_step(m)]
|
|
246
|
+
cmap.startup += mod.steps
|
|
247
|
+
value = mod.scope.get(name)
|
|
248
|
+
bound = None
|
|
249
|
+
if cls_name:
|
|
250
|
+
owner = mod.scope.get(cls_name)
|
|
251
|
+
member = self._member(owner.name, name) if owner is not None and owner.kind == "class" else None
|
|
252
|
+
value = _Sym("func", member[1]) if member and member[0] == "method" else None
|
|
253
|
+
bound = _Sym("obj", owner.name) if value is not None and owner is not None else None
|
|
254
|
+
file = mod.file or cmap.handler_file or ""
|
|
255
|
+
if value is not None and value.kind in ("func", "bound"):
|
|
256
|
+
key = value.name
|
|
257
|
+
node = self.defs[key][1]
|
|
258
|
+
cmap.entry, cmap.entry_line = key, node.lineno
|
|
259
|
+
positional = node.args.posonlyargs + node.args.args
|
|
260
|
+
plain = len(positional) - len(node.args.defaults)
|
|
261
|
+
cmap.params = [a.arg + ("=…" if i >= plain else "") for i, a in enumerate(positional)]
|
|
262
|
+
cmap.params += ["*" + node.args.vararg.arg] if node.args.vararg else []
|
|
263
|
+
cmap.is_async = isinstance(node, ast.AsyncFunctionDef)
|
|
264
|
+
route = next((path for _, path, routed, _ in self.routes if routed == key), None)
|
|
265
|
+
given = self._route_args(key, route) if route is not None else {} # a route's parameters, by name
|
|
266
|
+
inputs: list[_Sym | None] = [] if given else [_Sym(kind) for kind in self.platform.inputs]
|
|
267
|
+
with self._collecting() as got:
|
|
268
|
+
self._run(key, inputs, given, None, None, bound=bound)
|
|
269
|
+
cmap.steps = got.calls
|
|
270
|
+
elif not cls_name and (name in mod.scope or self.routes):
|
|
271
|
+
assigned = next(
|
|
272
|
+
(
|
|
273
|
+
n
|
|
274
|
+
for n in (mod.tree.body if mod.tree else [])
|
|
275
|
+
if isinstance(n, ast.Assign) and any(isinstance(t, ast.Name) and t.id == name for t in n.targets)
|
|
276
|
+
),
|
|
277
|
+
None,
|
|
278
|
+
)
|
|
279
|
+
cmap.entry, cmap.entry_line = f"{mod.name}.{name}", assigned.lineno if assigned else 0
|
|
280
|
+
step = CodeStep(
|
|
281
|
+
file, cmap.entry_line, _unparse(assigned) if assigned else f"{name} = …", "def", name=cmap.entry
|
|
282
|
+
)
|
|
283
|
+
frame = _Frame(mod.name, file, cmap.entry, {}, node=assigned)
|
|
284
|
+
with self._collecting() as got:
|
|
285
|
+
for method, path, key, _ in list(self.routes):
|
|
286
|
+
cls = self.defs[key][2]
|
|
287
|
+
self._run(
|
|
288
|
+
key,
|
|
289
|
+
[],
|
|
290
|
+
self._route_args(key, path),
|
|
291
|
+
frame,
|
|
292
|
+
None,
|
|
293
|
+
bound=_Sym("obj", cls) if cls else None,
|
|
294
|
+
label=f"{method} {path}",
|
|
295
|
+
)
|
|
296
|
+
step.steps = [
|
|
297
|
+
CodeStep(c.file, c.line, c.code, "def", c.notes, c.steps, c.calls, c.name, c.seen) for c in got.calls
|
|
298
|
+
]
|
|
299
|
+
if not self.routes:
|
|
300
|
+
step.notes.append(("more", "an object this can't follow, not a function"))
|
|
301
|
+
cmap.steps = [step]
|
|
302
|
+
if cmap.entry is None and mod.tree is not None:
|
|
303
|
+
takes = len(self.platform.inputs)
|
|
304
|
+
cmap.handlers = [
|
|
305
|
+
f"{n.name}()"
|
|
306
|
+
for n in mod.tree.body
|
|
307
|
+
if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))
|
|
308
|
+
and (not takes or len(n.args.posonlyargs + n.args.args) == takes)
|
|
309
|
+
]
|
|
310
|
+
cmap.functions = list(self.reached.values())
|
|
311
|
+
cmap.event = list(self.event_reads.values())
|
|
312
|
+
cmap.routes = [(method, path, key) for method, path, key, _ in self.routes]
|
|
313
|
+
loaded = [m for m in self.modules.values() if m.tree is not None]
|
|
314
|
+
cmap.other_files = sum(
|
|
315
|
+
1 for name, path in self.sources.items() if name not in self.modules and not _JUNK_RE.search(path)
|
|
316
|
+
)
|
|
317
|
+
for m in loaded:
|
|
318
|
+
for node in m.tree.body:
|
|
319
|
+
members = node.body if isinstance(node, ast.ClassDef) else [node]
|
|
320
|
+
prefix = f"{m.name}.{node.name}." if isinstance(node, ast.ClassDef) else f"{m.name}."
|
|
321
|
+
cmap.unreached += sum(
|
|
322
|
+
1
|
|
323
|
+
for f in members
|
|
324
|
+
if isinstance(f, (ast.FunctionDef, ast.AsyncFunctionDef)) and prefix + f.name not in self.reached
|
|
325
|
+
)
|
|
326
|
+
self._secrets(m)
|
|
327
|
+
for name, path in self.sources.items():
|
|
328
|
+
for match in _AWS_KEY_RE.finditer(self.texts[path]):
|
|
329
|
+
line = self.texts[path].count("\n", 0, match.start()) + 1
|
|
330
|
+
self._spot("secret", path, line, name, "", "an AWS access key", ok=False)
|
|
331
|
+
|
|
332
|
+
def _module_step(self, mod: _Module) -> list[CodeStep]:
|
|
333
|
+
if not mod.steps or mod.file is None:
|
|
334
|
+
return []
|
|
335
|
+
return [CodeStep(mod.file, 1, f"{mod.file}, run when it's imported", "module", steps=mod.steps, name=mod.name)]
|
|
336
|
+
|
|
337
|
+
def _chain(self, module: str) -> list[_Module]:
|
|
338
|
+
parts = module.split(".")
|
|
339
|
+
return [
|
|
340
|
+
self.modules[".".join(parts[:i])] for i in range(1, len(parts) + 1) if ".".join(parts[:i]) in self.modules
|
|
341
|
+
]
|
|
342
|
+
|
|
343
|
+
def _load_chain(self, module: str) -> _Module:
|
|
344
|
+
"""
|
|
345
|
+
A module and the packages it's in, loaded the way Python imports them: each package's __init__ first.
|
|
346
|
+
"""
|
|
347
|
+
parts = module.split(".")
|
|
348
|
+
for i in range(1, len(parts)):
|
|
349
|
+
if ".".join(parts[:i]) in self.sources:
|
|
350
|
+
self._load(".".join(parts[:i]))
|
|
351
|
+
return self._load(module)
|
|
352
|
+
|
|
353
|
+
def _load(self, name: str) -> _Module:
|
|
354
|
+
if name in self.modules:
|
|
355
|
+
return self.modules[name]
|
|
356
|
+
mod = self.modules[name] = _Module(name, self.sources.get(name))
|
|
357
|
+
if mod.file is None:
|
|
358
|
+
return mod
|
|
359
|
+
text = self.texts[mod.file]
|
|
360
|
+
if len(self.modules) > _MAP_FILES:
|
|
361
|
+
self.cmap.limited = f"read the first {_MAP_FILES} of its modules"
|
|
362
|
+
return mod
|
|
363
|
+
try:
|
|
364
|
+
mod.tree = _parse(text, mod.file)
|
|
365
|
+
except (SyntaxError, ValueError) as exc:
|
|
366
|
+
where = f"line {exc.lineno}: " if isinstance(exc, SyntaxError) and exc.lineno else ""
|
|
367
|
+
why = f"{where}{getattr(exc, 'msg', None) or exc}"
|
|
368
|
+
self.cmap.skipped.append((mod.file, why))
|
|
369
|
+
self._spot("syntax", mod.file, getattr(exc, "lineno", 0) or 0, name, "", why, ok=False)
|
|
370
|
+
return mod
|
|
371
|
+
self.cmap.files += 1
|
|
372
|
+
self.cmap.lines += text.count("\n") + (0 if text.endswith("\n") else 1)
|
|
373
|
+
frame = _Frame(name, mod.file, name, mod.scope)
|
|
374
|
+
mod.scope.setdefault("__name__", None)
|
|
375
|
+
self.loading += 1
|
|
376
|
+
try:
|
|
377
|
+
mod.steps = self._block(mod.tree.body, frame)
|
|
378
|
+
finally:
|
|
379
|
+
self.loading -= 1
|
|
380
|
+
mod.loaded = True
|
|
381
|
+
return mod
|
|
382
|
+
|
|
383
|
+
# ---------------------------------------------------------------- statements
|
|
384
|
+
|
|
385
|
+
def _block(self, stmts: list[Any], frame: _Frame) -> list[CodeStep]:
|
|
386
|
+
"""
|
|
387
|
+
A block's lines. After `if x: ... continue` (or return, raise, break) with no else, the lines below it run only
|
|
388
|
+
when x isn't so: that's a guard on them too.
|
|
389
|
+
"""
|
|
390
|
+
out: list[CodeStep] = []
|
|
391
|
+
guards = len(frame.guards)
|
|
392
|
+
for stmt in stmts:
|
|
393
|
+
out.extend(self._stmt(stmt, frame))
|
|
394
|
+
if (
|
|
395
|
+
isinstance(stmt, ast.If)
|
|
396
|
+
and not stmt.orelse
|
|
397
|
+
and _ends_early(stmt.body)
|
|
398
|
+
and not (frame.top and _is_main_guard(stmt.test))
|
|
399
|
+
):
|
|
400
|
+
frame.early.add(len(frame.guards))
|
|
401
|
+
frame.guards.append(_negated(stmt.test))
|
|
402
|
+
del frame.guards[guards:]
|
|
403
|
+
frame.early = {i for i in frame.early if i < guards}
|
|
404
|
+
return out
|
|
405
|
+
|
|
406
|
+
def _step(
|
|
407
|
+
self,
|
|
408
|
+
node: Any,
|
|
409
|
+
frame: _Frame,
|
|
410
|
+
code: str,
|
|
411
|
+
kind: str,
|
|
412
|
+
got: _Collect | None = None,
|
|
413
|
+
steps: Iterable[CodeStep] = (),
|
|
414
|
+
) -> CodeStep:
|
|
415
|
+
self.count += 1
|
|
416
|
+
notes = got.notes if got else []
|
|
417
|
+
events = [text for k, text in notes if k == "event"]
|
|
418
|
+
notes = [
|
|
419
|
+
(k, text)
|
|
420
|
+
for k, text in notes
|
|
421
|
+
if k != "event" or not any(other.startswith((text + "[", text + ".", text + " (")) for other in events)
|
|
422
|
+
]
|
|
423
|
+
return CodeStep(
|
|
424
|
+
frame.file,
|
|
425
|
+
getattr(node, "lineno", 0),
|
|
426
|
+
code,
|
|
427
|
+
kind,
|
|
428
|
+
notes,
|
|
429
|
+
list(steps),
|
|
430
|
+
got.calls if got else [],
|
|
431
|
+
holds=got.holds if got else [],
|
|
432
|
+
given=got.given if got else [],
|
|
433
|
+
)
|
|
434
|
+
|
|
435
|
+
def _stmt(self, stmt: Any, frame: _Frame) -> list[CodeStep]:
|
|
436
|
+
if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
437
|
+
frame.scope[stmt.name] = _Sym("func", self._define(stmt, frame))
|
|
438
|
+
return []
|
|
439
|
+
if isinstance(stmt, ast.ClassDef):
|
|
440
|
+
return self._define_class(stmt, frame)
|
|
441
|
+
if isinstance(stmt, (ast.Import, ast.ImportFrom)):
|
|
442
|
+
return self._import(stmt, frame)
|
|
443
|
+
if isinstance(stmt, (ast.Global, ast.Nonlocal)):
|
|
444
|
+
frame.globals.update(stmt.names)
|
|
445
|
+
return []
|
|
446
|
+
if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Constant):
|
|
447
|
+
return [] # a docstring
|
|
448
|
+
if isinstance(stmt, (ast.For, ast.AsyncFor, ast.While)):
|
|
449
|
+
return self._loop(stmt, frame)
|
|
450
|
+
if isinstance(stmt, ast.If):
|
|
451
|
+
return self._if(stmt, frame)
|
|
452
|
+
if isinstance(stmt, (ast.With, ast.AsyncWith)):
|
|
453
|
+
return self._with(stmt, frame)
|
|
454
|
+
if isinstance(stmt, ast.Try) or type(stmt).__name__ == "TryStar":
|
|
455
|
+
return self._try(stmt, frame)
|
|
456
|
+
if isinstance(stmt, ast.Match):
|
|
457
|
+
return self._match(stmt, frame)
|
|
458
|
+
kind = "do"
|
|
459
|
+
with self._collecting() as got:
|
|
460
|
+
if isinstance(stmt, ast.Return):
|
|
461
|
+
kind = "return"
|
|
462
|
+
value = self._eval(stmt.value, frame)
|
|
463
|
+
frame.returns.append(value)
|
|
464
|
+
if frame.function == self.cmap.entry and stmt.value is not None:
|
|
465
|
+
self.cmap.returns.append((_unparse(stmt.value, 2000), frame.file, stmt.lineno))
|
|
466
|
+
elif isinstance(stmt, ast.Raise):
|
|
467
|
+
kind = "raise"
|
|
468
|
+
self._eval(stmt.exc, frame)
|
|
469
|
+
self._eval(stmt.cause, frame)
|
|
470
|
+
elif isinstance(stmt, (ast.Assign, ast.AnnAssign)):
|
|
471
|
+
targets = stmt.targets if isinstance(stmt, ast.Assign) else [stmt.target]
|
|
472
|
+
self.target = {t.id for t in targets if isinstance(t, ast.Name)}
|
|
473
|
+
value = self._eval(stmt.value, frame)
|
|
474
|
+
self.target = set()
|
|
475
|
+
if value is None and _is_text(stmt.value): # QUERY = "SELECT ...": kept, for a call it's passed to
|
|
476
|
+
value = _Sym("text", sub=_unparse(stmt.value, 400))
|
|
477
|
+
if stmt.value is not None:
|
|
478
|
+
for target in targets:
|
|
479
|
+
self._bind(target, value, frame)
|
|
480
|
+
self._holds(target, value)
|
|
481
|
+
else:
|
|
482
|
+
for child in ast.iter_child_nodes(stmt):
|
|
483
|
+
if isinstance(child, ast.expr):
|
|
484
|
+
self._eval(child, frame)
|
|
485
|
+
tainted = any(what.startswith("text built from") for _, what in got.holds) # where the input goes next
|
|
486
|
+
if (kind != "do" and not frame.top) or got.notes or got.calls or tainted:
|
|
487
|
+
return [self._step(stmt, frame, _unparse(stmt), kind, got)]
|
|
488
|
+
return []
|
|
489
|
+
|
|
490
|
+
def _loop(self, stmt: Any, frame: _Frame) -> list[CodeStep]:
|
|
491
|
+
with self._collecting() as head:
|
|
492
|
+
if isinstance(stmt, ast.While):
|
|
493
|
+
self._eval(stmt.test, frame)
|
|
494
|
+
header = f"while {_unparse(stmt.test, 110)}:"
|
|
495
|
+
else:
|
|
496
|
+
items = self._eval(stmt.iter, frame)
|
|
497
|
+
self._bind_loop(stmt.target, items, frame)
|
|
498
|
+
if not (items is not None and items.kind == "enumerate"):
|
|
499
|
+
self._holds(stmt.target, self._item(items), each=True)
|
|
500
|
+
prefix = "async for" if isinstance(stmt, ast.AsyncFor) else "for"
|
|
501
|
+
header = f"{prefix} {_target(stmt.target)} in {_unparse(stmt.iter, 100)}:"
|
|
502
|
+
frame.loops.append(header)
|
|
503
|
+
body = self._block(stmt.body, frame)
|
|
504
|
+
frame.loops.pop()
|
|
505
|
+
orelse = self._block(stmt.orelse, frame)
|
|
506
|
+
out = [self._step(stmt, frame, header, "loop", head, body)] if head.notes or head.calls or body else []
|
|
507
|
+
if orelse:
|
|
508
|
+
out.append(CodeStep(frame.file, stmt.orelse[0].lineno - 1, "else:", "else", steps=orelse))
|
|
509
|
+
return out
|
|
510
|
+
|
|
511
|
+
def _if(self, stmt: ast.If, frame: _Frame) -> list[CodeStep]:
|
|
512
|
+
if frame.top and _is_main_guard(stmt.test):
|
|
513
|
+
return [] # runs only when the file is run as a script, never when it's imported
|
|
514
|
+
with self._collecting() as head:
|
|
515
|
+
self._eval(stmt.test, frame)
|
|
516
|
+
test = _unparse(stmt.test, 110)
|
|
517
|
+
frame.guards.append(test)
|
|
518
|
+
body = self._block(stmt.body, frame)
|
|
519
|
+
frame.guards.pop()
|
|
520
|
+
rest: list[CodeStep] = []
|
|
521
|
+
if len(stmt.orelse) == 1 and isinstance(stmt.orelse[0], ast.If):
|
|
522
|
+
rest = self._if(stmt.orelse[0], frame)
|
|
523
|
+
if rest and rest[0].kind == "if" and rest[0].line == stmt.orelse[0].lineno:
|
|
524
|
+
rest[0].code = "el" + rest[0].code
|
|
525
|
+
elif stmt.orelse:
|
|
526
|
+
frame.guards.append(_negated(stmt.test))
|
|
527
|
+
orelse = self._block(stmt.orelse, frame)
|
|
528
|
+
frame.guards.pop()
|
|
529
|
+
if orelse:
|
|
530
|
+
rest = [CodeStep(frame.file, stmt.orelse[0].lineno - 1, "else:", "else", steps=orelse)]
|
|
531
|
+
if head.notes or head.calls or body or rest:
|
|
532
|
+
return [self._step(stmt, frame, f"if {test}:", "if", head, body), *rest]
|
|
533
|
+
return []
|
|
534
|
+
|
|
535
|
+
def _with(self, stmt: Any, frame: _Frame) -> list[CodeStep]:
|
|
536
|
+
"""
|
|
537
|
+
A with block: each of its own classes' __enter__ runs first (`as` takes what it returns), then the lines in it,
|
|
538
|
+
then each __exit__, last in first out, shown as a step of their own after the lines.
|
|
539
|
+
"""
|
|
540
|
+
enter, leave = ("__aenter__", "__aexit__") if isinstance(stmt, ast.AsyncWith) else ("__enter__", "__exit__")
|
|
541
|
+
entered: list[_Sym] = []
|
|
542
|
+
with self._collecting() as head:
|
|
543
|
+
parts = []
|
|
544
|
+
for item in stmt.items:
|
|
545
|
+
value = self._eval(item.context_expr, frame)
|
|
546
|
+
ran, given = self._dunder(value, enter, stmt, frame)
|
|
547
|
+
if value is not None and self._dunder_of(value, leave):
|
|
548
|
+
entered.append(value)
|
|
549
|
+
if item.optional_vars is not None:
|
|
550
|
+
self._bind(item.optional_vars, given if ran else value, frame)
|
|
551
|
+
self._holds(item.optional_vars, given if ran else value)
|
|
552
|
+
parts.append(
|
|
553
|
+
_unparse(item.context_expr, 100)
|
|
554
|
+
+ (f" as {_unparse(item.optional_vars, 30)}" if item.optional_vars is not None else "")
|
|
555
|
+
)
|
|
556
|
+
body = self._block(stmt.body, frame)
|
|
557
|
+
with self._collecting() as leaving:
|
|
558
|
+
for value in reversed(entered):
|
|
559
|
+
self._dunder(value, leave, stmt, frame)
|
|
560
|
+
if leaving.calls:
|
|
561
|
+
body.append(self._step(stmt, frame, f"# leaving the with block: {leave} runs", "do", leaving))
|
|
562
|
+
header = f"{'async with' if isinstance(stmt, ast.AsyncWith) else 'with'} {', '.join(parts)}:"
|
|
563
|
+
return [self._step(stmt, frame, header, "with", head, body)] if head.notes or head.calls or body else []
|
|
564
|
+
|
|
565
|
+
def _dunder_of(self, value: _Sym, name: str) -> str | None:
|
|
566
|
+
"""
|
|
567
|
+
The method one of its own objects has by a name Python calls for it ('__enter__'), or None.
|
|
568
|
+
"""
|
|
569
|
+
found = self._member(value.name, name) if value.kind == "obj" else None
|
|
570
|
+
return found[1] if found and found[0] == "method" else None
|
|
571
|
+
|
|
572
|
+
def _dunder(self, value: _Sym | None, name: str, node: Any, frame: _Frame) -> tuple[bool, _Sym | None]:
|
|
573
|
+
"""
|
|
574
|
+
Runs a method Python calls by itself (__enter__, __exit__) on one of its own objects: whether there was one,
|
|
575
|
+
and what it returned.
|
|
576
|
+
"""
|
|
577
|
+
key = self._dunder_of(value, name) if value is not None else None
|
|
578
|
+
if key is None:
|
|
579
|
+
return False, None
|
|
580
|
+
return True, self._run(key, [], {}, frame, node, bound=value)
|
|
581
|
+
|
|
582
|
+
def _try(self, stmt: Any, frame: _Frame) -> list[CodeStep]:
|
|
583
|
+
optional = any(
|
|
584
|
+
h.type is None or _names_in(h.type, ("ImportError", "ModuleNotFoundError", "Exception"))
|
|
585
|
+
for h in stmt.handlers
|
|
586
|
+
)
|
|
587
|
+
frame.optional += optional
|
|
588
|
+
body = self._block(stmt.body, frame)
|
|
589
|
+
frame.optional -= optional
|
|
590
|
+
handlers = []
|
|
591
|
+
for h in stmt.handlers:
|
|
592
|
+
with self._collecting() as head:
|
|
593
|
+
self._eval(h.type, frame)
|
|
594
|
+
if h.name:
|
|
595
|
+
frame.scope[h.name] = None
|
|
596
|
+
silent = _swallows(h) and not frame.top
|
|
597
|
+
if silent:
|
|
598
|
+
head.note("warn", "drops the error")
|
|
599
|
+
hbody = self._block(h.body, frame)
|
|
600
|
+
header = (
|
|
601
|
+
"except"
|
|
602
|
+
+ (f" {_unparse(h.type, 60)}" if h.type is not None else "")
|
|
603
|
+
+ (f" as {h.name}" if h.name else "")
|
|
604
|
+
+ ":"
|
|
605
|
+
)
|
|
606
|
+
if silent:
|
|
607
|
+
code = " ".join([header] + [_unparse(s, 40) for s in h.body])
|
|
608
|
+
self._spot("swallow", frame.file, h.lineno, frame.function, code)
|
|
609
|
+
if hbody or head.notes:
|
|
610
|
+
handlers.append(self._step(h, frame, header, "except", head, hbody))
|
|
611
|
+
orelse = self._block(stmt.orelse, frame)
|
|
612
|
+
final = self._block(stmt.finalbody, frame)
|
|
613
|
+
if not (body or handlers or orelse or final):
|
|
614
|
+
return []
|
|
615
|
+
out = [CodeStep(frame.file, stmt.lineno, "try:", "try", steps=body), *handlers]
|
|
616
|
+
if orelse:
|
|
617
|
+
out.append(CodeStep(frame.file, stmt.orelse[0].lineno - 1, "else:", "else", steps=orelse))
|
|
618
|
+
if final:
|
|
619
|
+
out.append(CodeStep(frame.file, stmt.finalbody[0].lineno - 1, "finally:", "else", steps=final))
|
|
620
|
+
return out
|
|
621
|
+
|
|
622
|
+
def _match(self, stmt: Any, frame: _Frame) -> list[CodeStep]:
|
|
623
|
+
with self._collecting() as head:
|
|
624
|
+
self._eval(stmt.subject, frame)
|
|
625
|
+
cases = []
|
|
626
|
+
for case in stmt.cases:
|
|
627
|
+
with self._collecting() as got:
|
|
628
|
+
self._eval(case.guard, frame)
|
|
629
|
+
body = self._block(case.body, frame)
|
|
630
|
+
if body or got.notes:
|
|
631
|
+
guard = f" if {_unparse(case.guard, 60)}" if case.guard is not None else ""
|
|
632
|
+
cases.append(
|
|
633
|
+
self._step(case.pattern, frame, f"case {_unparse(case.pattern, 80)}{guard}:", "if", got, body)
|
|
634
|
+
)
|
|
635
|
+
if head.notes or cases:
|
|
636
|
+
return [self._step(stmt, frame, f"match {_unparse(stmt.subject, 100)}:", "if", head, cases)]
|
|
637
|
+
return []
|
|
638
|
+
|
|
639
|
+
# ---------------------------------------------------------------- definitions and imports
|
|
640
|
+
|
|
641
|
+
def _define(self, node: Any, frame: _Frame, cls: str | None = None) -> str:
|
|
642
|
+
key = f"{cls or (frame.module if frame.top else frame.function)}.{node.name}"
|
|
643
|
+
self.defs[key] = (frame.module, node, cls)
|
|
644
|
+
if not frame.top:
|
|
645
|
+
self.closures[key] = [frame.scope, *frame.outers]
|
|
646
|
+
for d in node.decorator_list:
|
|
647
|
+
target = d.func if isinstance(d, ast.Call) else d
|
|
648
|
+
name = target.id if isinstance(target, ast.Name) else getattr(target, "attr", "")
|
|
649
|
+
if name in _CACHES:
|
|
650
|
+
self.cached.add(key)
|
|
651
|
+
if (
|
|
652
|
+
isinstance(d, ast.Call)
|
|
653
|
+
and isinstance(d.func, ast.Attribute)
|
|
654
|
+
and d.func.attr in _ROUTE_METHODS
|
|
655
|
+
and d.args
|
|
656
|
+
and isinstance(_const(d.args[0]), str)
|
|
657
|
+
and str(_const(d.args[0])).startswith("/")
|
|
658
|
+
):
|
|
659
|
+
verb = d.func.attr.upper()
|
|
660
|
+
if verb in ("ROUTE", "API_ROUTE"):
|
|
661
|
+
methods = next((kw.value for kw in d.keywords if kw.arg == "methods"), None)
|
|
662
|
+
found = [str(_const(e)) for e in getattr(methods, "elts", []) if isinstance(_const(e), str)]
|
|
663
|
+
verb = ", ".join(found) or "GET"
|
|
664
|
+
self.routes.append((verb, str(_const(d.args[0])), key, _unparse(d.func.value, 60)))
|
|
665
|
+
elif isinstance(d, ast.Call) and isinstance(d.func, ast.Attribute) and d.func.attr in _CLI_COMMANDS:
|
|
666
|
+
given = next((kw.value for kw in d.keywords if kw.arg == "name"), d.args[0] if d.args else None)
|
|
667
|
+
command = _const(given) if given is not None else None
|
|
668
|
+
command = command if isinstance(command, str) else node.name.replace("_", "-")
|
|
669
|
+
if d.func.attr == "callback":
|
|
670
|
+
command = "(before every command)"
|
|
671
|
+
self.routes.append(("CLI", command, key, _unparse(d.func.value, 60)))
|
|
672
|
+
return key
|
|
673
|
+
|
|
674
|
+
def _route_args(self, key: str, path: str) -> dict[str, _Sym | None]:
|
|
675
|
+
"""
|
|
676
|
+
What a function a route runs is called with, by parameter, as its platform says (FastAPI's order_id from
|
|
677
|
+
/orders/{order_id}, Django's request): each is part of the input, and listed as read from it.
|
|
678
|
+
"""
|
|
679
|
+
module, node, cls = self.defs[key]
|
|
680
|
+
imports = {
|
|
681
|
+
v.name.split(".")[0] for v in self.modules[module].scope.values() if v is not None and v.kind == "mod"
|
|
682
|
+
}
|
|
683
|
+
found = self.platform.route_inputs(node, path, imports, bound=cls is not None)
|
|
684
|
+
a = node.args
|
|
685
|
+
positional = a.posonlyargs + a.args
|
|
686
|
+
optional = {p.arg for p in positional[len(positional) - len(a.defaults) :]} | {
|
|
687
|
+
p.arg for p, default in zip(a.kwonlyargs, a.kw_defaults) if default is not None
|
|
688
|
+
}
|
|
689
|
+
for name, part in found.items():
|
|
690
|
+
if part and part not in self.event_reads:
|
|
691
|
+
self.event_reads[part] = EventField(
|
|
692
|
+
part, self.modules[module].file or "", node.lineno, name not in optional
|
|
693
|
+
)
|
|
694
|
+
return {name: _Sym("event", path=part) for name, part in found.items()}
|
|
695
|
+
|
|
696
|
+
def _url(self, name: str, node: ast.Call, args: list[_Sym | None], frame: _Frame) -> None:
|
|
697
|
+
"""
|
|
698
|
+
A Django URL conf's path('orders/<int:pk>/', view): a route for a view function, for each method of a
|
|
699
|
+
class-based view (View.as_view()), or, for include(...), the included routes under this path.
|
|
700
|
+
"""
|
|
701
|
+
route = _const(node.args[0]) if node.args else None
|
|
702
|
+
view = node.args[1] if len(node.args) > 1 else None
|
|
703
|
+
if not isinstance(route, str) or view is None:
|
|
704
|
+
return
|
|
705
|
+
if name != "django.urls.path":
|
|
706
|
+
route = route.removeprefix("^").removesuffix("$")
|
|
707
|
+
held = args[1] if len(args) > 1 else None
|
|
708
|
+
if held is not None and held.kind == "urls":
|
|
709
|
+
self.routes = [
|
|
710
|
+
(method, f"/{route}{path[1:]}", key, frame.module)
|
|
711
|
+
if router == held.name
|
|
712
|
+
else (method, path, key, router)
|
|
713
|
+
for method, path, key, router in self.routes
|
|
714
|
+
]
|
|
715
|
+
elif held is not None and held.kind == "func":
|
|
716
|
+
verbs = ""
|
|
717
|
+
for d in self.defs[held.name][1].decorator_list:
|
|
718
|
+
called = d.func if isinstance(d, ast.Call) else d
|
|
719
|
+
said = called.id if isinstance(called, ast.Name) else getattr(called, "attr", "")
|
|
720
|
+
listed = d.args[0] if isinstance(d, ast.Call) and d.args else None
|
|
721
|
+
if said in _VIEW_VERBS:
|
|
722
|
+
verbs = _VIEW_VERBS[said]
|
|
723
|
+
elif said in ("require_http_methods", "api_view"):
|
|
724
|
+
verbs = ", ".join(str(_const(e)) for e in getattr(listed, "elts", []) if _const(e)) or "GET"
|
|
725
|
+
self.routes.append((verbs or "ANY", f"/{route}", held.name, frame.module))
|
|
726
|
+
elif isinstance(view, ast.Call) and isinstance(view.func, ast.Attribute) and view.func.attr == "as_view":
|
|
727
|
+
cls = self._eval(view.func.value, frame)
|
|
728
|
+
if cls is None or cls.kind != "class":
|
|
729
|
+
return
|
|
730
|
+
for method in _VIEW_METHODS:
|
|
731
|
+
found = self._member(cls.name, method)
|
|
732
|
+
if found and found[0] == "method":
|
|
733
|
+
self.routes.append((method.upper(), f"/{route}", found[1], frame.module))
|
|
734
|
+
|
|
735
|
+
def _include(self, node: ast.Call, frame: _Frame) -> _Sym | None:
|
|
736
|
+
"""
|
|
737
|
+
Django's include('shop.urls') (or include(router.urls)): the URL conf it reads, loaded so its routes are
|
|
738
|
+
found, for the path() it's given to put them under.
|
|
739
|
+
"""
|
|
740
|
+
target = node.args[0] if node.args else None
|
|
741
|
+
if isinstance(target, ast.Tuple) and target.elts:
|
|
742
|
+
target = target.elts[0]
|
|
743
|
+
module = _const(target)
|
|
744
|
+
if isinstance(module, str) and module in self.sources:
|
|
745
|
+
self._load_chain(module)
|
|
746
|
+
return _Sym("urls", module)
|
|
747
|
+
if isinstance(target, ast.Attribute) and target.attr == "urls":
|
|
748
|
+
return _Sym("urls", f"{frame.module}.{_unparse(target.value, 60)}")
|
|
749
|
+
return None
|
|
750
|
+
|
|
751
|
+
def _viewset(self, prefix: str, node: ast.Call, router: str, frame: _Frame) -> None:
|
|
752
|
+
"""
|
|
753
|
+
A Django REST framework router's register('orders', OrderViewSet): a route for each of the ViewSet's own
|
|
754
|
+
actions (list, retrieve...) and each method of an APIView.
|
|
755
|
+
"""
|
|
756
|
+
cls = self._eval(node.args[1], frame)
|
|
757
|
+
if cls is None or cls.kind != "class":
|
|
758
|
+
return
|
|
759
|
+
actions = {**_VIEWSET_ACTIONS, **{method: (method.upper(), False) for method in _VIEW_METHODS}}
|
|
760
|
+
for action, (verb, detail) in actions.items():
|
|
761
|
+
found = self._member(cls.name, action)
|
|
762
|
+
if found and found[0] == "method":
|
|
763
|
+
self.routes.append((verb, f"/{prefix}/" + ("<pk>/" if detail else ""), found[1], router))
|
|
764
|
+
|
|
765
|
+
def _define_class(self, node: ast.ClassDef, frame: _Frame) -> list[CodeStep]:
|
|
766
|
+
key = f"{frame.module if frame.top else frame.function}.{node.name}"
|
|
767
|
+
info: dict[str, Any] = {"methods": {}, "attrs": {}, "bases": []}
|
|
768
|
+
self.classes[key] = info
|
|
769
|
+
frame.scope[node.name] = _Sym("class", key)
|
|
770
|
+
for base in node.bases:
|
|
771
|
+
found = self._eval(base, frame)
|
|
772
|
+
if found is not None and found.kind == "class":
|
|
773
|
+
info["bases"].append(found.name)
|
|
774
|
+
steps = []
|
|
775
|
+
for stmt in node.body:
|
|
776
|
+
if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
777
|
+
if any(isinstance(d, ast.Attribute) and d.attr in _ACCESSORS for d in stmt.decorator_list):
|
|
778
|
+
continue # @size.setter: reading obj.size still runs the getter
|
|
779
|
+
info["methods"][stmt.name] = self._define(stmt, frame, cls=key)
|
|
780
|
+
elif isinstance(stmt, (ast.Assign, ast.AnnAssign)) and stmt.value is not None:
|
|
781
|
+
with self._collecting() as got:
|
|
782
|
+
value = self._eval(stmt.value, frame)
|
|
783
|
+
targets = stmt.targets if isinstance(stmt, ast.Assign) else [stmt.target]
|
|
784
|
+
for target in targets:
|
|
785
|
+
if isinstance(target, ast.Name):
|
|
786
|
+
info["attrs"][target.id] = value
|
|
787
|
+
if got.notes or got.calls:
|
|
788
|
+
steps.append(self._step(stmt, frame, _unparse(stmt), "do", got))
|
|
789
|
+
return [CodeStep(frame.file, node.lineno, f"class {node.name}:", "do", steps=steps)] if steps else []
|
|
790
|
+
|
|
791
|
+
def _member(self, cls: str, name: str, seen: set[str] | None = None) -> tuple[str, Any] | None:
|
|
792
|
+
"""
|
|
793
|
+
A class's attribute or method, looked for in its own classes it's made from too.
|
|
794
|
+
"""
|
|
795
|
+
seen = seen if seen is not None else set()
|
|
796
|
+
info = self.classes.get(cls)
|
|
797
|
+
if info is None or cls in seen:
|
|
798
|
+
return None
|
|
799
|
+
seen.add(cls)
|
|
800
|
+
if name in info["attrs"]:
|
|
801
|
+
return "attr", info["attrs"][name]
|
|
802
|
+
if name in info["methods"]:
|
|
803
|
+
return "method", info["methods"][name]
|
|
804
|
+
for base in info["bases"]:
|
|
805
|
+
found = self._member(base, name, seen)
|
|
806
|
+
if found:
|
|
807
|
+
return found
|
|
808
|
+
return None
|
|
809
|
+
|
|
810
|
+
def _own(self, dotted: str) -> str | None:
|
|
811
|
+
return dotted if dotted in self.sources else None
|
|
812
|
+
|
|
813
|
+
def _import(self, stmt: Any, frame: _Frame) -> list[CodeStep]:
|
|
814
|
+
before = set(self.modules)
|
|
815
|
+
if isinstance(stmt, ast.Import):
|
|
816
|
+
for alias in stmt.names:
|
|
817
|
+
own = self._own(alias.name)
|
|
818
|
+
self._record_import(alias.name, bool(own), stmt, frame)
|
|
819
|
+
bound = alias.asname or alias.name.split(".")[0]
|
|
820
|
+
if own:
|
|
821
|
+
self._load_chain(own)
|
|
822
|
+
frame.scope[bound] = _Sym("own", own if alias.asname else own.split(".")[0])
|
|
823
|
+
elif alias.name.split(".")[0] in self.sources:
|
|
824
|
+
frame.scope[bound] = _Sym("own", alias.name.split(".")[0])
|
|
825
|
+
else:
|
|
826
|
+
frame.scope[bound] = _Sym("mod", alias.name if alias.asname else bound)
|
|
827
|
+
else:
|
|
828
|
+
base = stmt.module or ""
|
|
829
|
+
if stmt.level:
|
|
830
|
+
package = frame.module.split(".")
|
|
831
|
+
is_package = (self.sources.get(frame.module) or "").endswith("__init__.py")
|
|
832
|
+
keep = len(package) - (stmt.level - (1 if is_package else 0))
|
|
833
|
+
base = ".".join(package[: max(keep, 0)] + ([stmt.module] if stmt.module else []))
|
|
834
|
+
own = bool(stmt.level) or base in self.sources or base.split(".")[0] in self.sources
|
|
835
|
+
self._record_import(base or ".", own, stmt, frame)
|
|
836
|
+
if own and base in self.sources:
|
|
837
|
+
self._load_chain(base)
|
|
838
|
+
for alias in stmt.names:
|
|
839
|
+
if alias.name == "*":
|
|
840
|
+
if own and base in self.modules:
|
|
841
|
+
frame.scope.update({k: v for k, v in self.modules[base].scope.items() if not k.startswith("_")})
|
|
842
|
+
continue
|
|
843
|
+
sub = f"{base}.{alias.name}" if base else alias.name
|
|
844
|
+
if own and sub in self.sources:
|
|
845
|
+
self._load_chain(sub)
|
|
846
|
+
value: _Sym | None = _Sym("own", sub)
|
|
847
|
+
elif own:
|
|
848
|
+
value = self.modules[base].scope.get(alias.name) if base in self.modules else None
|
|
849
|
+
else:
|
|
850
|
+
value = _Sym("event") if sub in self.platform.input_objects else _Sym("mod", sub)
|
|
851
|
+
frame.scope[alias.asname or alias.name] = value
|
|
852
|
+
loaded = [self.modules[name] for name in self.modules if name not in before]
|
|
853
|
+
calls = [step for mod in loaded for step in self._module_step(mod)]
|
|
854
|
+
if calls:
|
|
855
|
+
step = self._step(stmt, frame, _unparse(stmt), "do")
|
|
856
|
+
step.calls = calls
|
|
857
|
+
return [step]
|
|
858
|
+
return []
|
|
859
|
+
|
|
860
|
+
def _record_import(self, module: str, own: bool, stmt: Any, frame: _Frame) -> None:
|
|
861
|
+
top = module.split(".")[0]
|
|
862
|
+
version = self.version or (3, 99)
|
|
863
|
+
if own or module == ".":
|
|
864
|
+
source = "own"
|
|
865
|
+
elif top in _REMOVED_MODULES:
|
|
866
|
+
source = "removed" if version >= _REMOVED_MODULES[top] else "stdlib"
|
|
867
|
+
elif top in _ADDED_MODULES:
|
|
868
|
+
source = "stdlib" if version >= _ADDED_MODULES[top] else "missing"
|
|
869
|
+
elif top in sys.stdlib_module_names:
|
|
870
|
+
source = "stdlib"
|
|
871
|
+
elif top in _RUNTIME_MODULES:
|
|
872
|
+
source = "runtime"
|
|
873
|
+
elif top in self.tops or top in self.libraries:
|
|
874
|
+
source = "package"
|
|
875
|
+
else:
|
|
876
|
+
source = "layer" if self.layers else "missing"
|
|
877
|
+
key = (module, frame.file, stmt.lineno)
|
|
878
|
+
if key not in self.seen:
|
|
879
|
+
self.seen.add(key)
|
|
880
|
+
self.cmap.imports.append(
|
|
881
|
+
ImportUse(module, source, frame.file, stmt.lineno, bool(frame.optional), frame.top)
|
|
882
|
+
)
|
|
883
|
+
|
|
884
|
+
# ---------------------------------------------------------------- names
|
|
885
|
+
|
|
886
|
+
def _lookup(self, name: str, frame: _Frame) -> _Sym | None:
|
|
887
|
+
for scope in (frame.scope, *frame.outers):
|
|
888
|
+
if name in scope:
|
|
889
|
+
return scope[name]
|
|
890
|
+
module = self.modules.get(frame.module)
|
|
891
|
+
if module is not None and name in module.scope:
|
|
892
|
+
return module.scope[name]
|
|
893
|
+
if name in ("print", "open", "enumerate", "sorted", "reversed", "list", "tuple", "iter"):
|
|
894
|
+
return _Sym("builtin", name)
|
|
895
|
+
return None
|
|
896
|
+
|
|
897
|
+
def _bind(self, target: Any, value: _Sym | None, frame: _Frame) -> None:
|
|
898
|
+
if isinstance(target, ast.Name):
|
|
899
|
+
frame.scope[target.id] = value
|
|
900
|
+
if target.id in frame.globals and not frame.top and frame.module in self.modules:
|
|
901
|
+
self.modules[frame.module].scope[target.id] = value
|
|
902
|
+
elif isinstance(target, ast.Attribute):
|
|
903
|
+
owner = self._eval(target.value, frame)
|
|
904
|
+
if owner is not None and owner.kind in ("obj", "class") and owner.name in self.classes:
|
|
905
|
+
self.classes[owner.name]["attrs"][target.attr] = value
|
|
906
|
+
elif owner is not None and owner.kind == "own" and owner.name in self.modules:
|
|
907
|
+
self.modules[owner.name].scope[target.attr] = value
|
|
908
|
+
elif isinstance(target, (ast.Tuple, ast.List)):
|
|
909
|
+
for elt in target.elts:
|
|
910
|
+
self._bind(elt.value if isinstance(elt, ast.Starred) else elt, None, frame)
|
|
911
|
+
|
|
912
|
+
def _holds(self, target: Any, value: _Sym | None, each: bool = False) -> None:
|
|
913
|
+
"""
|
|
914
|
+
Notes what a name (or self.name) set on this line holds, when that can be said: shown beside the line.
|
|
915
|
+
"""
|
|
916
|
+
said = self._describe(value)
|
|
917
|
+
if said and isinstance(target, (ast.Name, ast.Attribute)):
|
|
918
|
+
self.cur.holds.append((_unparse(target, 40), f"each {said}" if each else said))
|
|
919
|
+
|
|
920
|
+
def _describe(self, value: _Sym | None) -> str:
|
|
921
|
+
"""
|
|
922
|
+
What a value is, in a few words: 'event Records[].s3.bucket.name', 'env ARCHIVE_BUCKET', 'an S3 client',
|
|
923
|
+
'a DynamoDB Table', 'an OrderStore'. '' for what it can't say.
|
|
924
|
+
"""
|
|
925
|
+
if value is None:
|
|
926
|
+
return ""
|
|
927
|
+
kind, name = value.kind, value.name
|
|
928
|
+
service = _service_name(name) if name and name != "?" else "boto3"
|
|
929
|
+
last = name.rsplit(".", 1)[-1]
|
|
930
|
+
said = {
|
|
931
|
+
"event": self._input_label(value.path) if value.path else f"the {self.platform.input_word}",
|
|
932
|
+
"envval": f"env {name}",
|
|
933
|
+
"env": "os.environ",
|
|
934
|
+
"context": "its context",
|
|
935
|
+
"client": f"{_a(service)} {service} client",
|
|
936
|
+
"resource": f"{_a(service)} {service} resource",
|
|
937
|
+
"res": f"{_a(service)} {service} {value.sub or 'resource object'}",
|
|
938
|
+
"coll": f"{_a(service)} {service} collection",
|
|
939
|
+
"batch": f"{_a(service)} {service} batch writer",
|
|
940
|
+
"paginator": f"{_a(service)} {service} {value.sub} paginator",
|
|
941
|
+
"obj": f"{_a(last)} {last}",
|
|
942
|
+
"class": f"the class {last}",
|
|
943
|
+
"func": f"{last}()",
|
|
944
|
+
"bound": f"{last}()",
|
|
945
|
+
"logger": "a logger",
|
|
946
|
+
"http": f"{_a(name)} {name} session" if name else "an HTTP session",
|
|
947
|
+
"own": f"the module {name}",
|
|
948
|
+
}
|
|
949
|
+
if kind == "lib" and value.handle is not None:
|
|
950
|
+
library = get_library(name)
|
|
951
|
+
return library.describe(value.handle) if library is not None else ""
|
|
952
|
+
if kind == "text":
|
|
953
|
+
return f"text built from {', '.join(value.path)}" if value.path else ""
|
|
954
|
+
return said.get(kind, "")
|
|
955
|
+
|
|
956
|
+
def _item(self, items: _Sym | None) -> _Sym | None:
|
|
957
|
+
"""
|
|
958
|
+
What a loop over `items` gets each time round.
|
|
959
|
+
"""
|
|
960
|
+
if items is None:
|
|
961
|
+
return None
|
|
962
|
+
if items.kind == "event":
|
|
963
|
+
return _Sym("event", path=items.path + ("[]",))
|
|
964
|
+
if items.kind == "coll":
|
|
965
|
+
return _Sym("res", items.name, "ObjectSummary" if items.name == "s3" else "")
|
|
966
|
+
return None
|
|
967
|
+
|
|
968
|
+
def _bind_loop(self, target: Any, items: _Sym | None, frame: _Frame) -> None:
|
|
969
|
+
if items is not None and items.kind == "enumerate" and isinstance(target, ast.Tuple) and len(target.elts) == 2:
|
|
970
|
+
self._bind(target.elts[0], None, frame)
|
|
971
|
+
self._bind(target.elts[1], self._item(items.inner), frame)
|
|
972
|
+
else:
|
|
973
|
+
self._bind(target, self._item(items), frame)
|
|
974
|
+
|
|
975
|
+
# ---------------------------------------------------------------- expressions
|
|
976
|
+
|
|
977
|
+
def _eval(self, node: Any, frame: _Frame) -> _Sym | None:
|
|
978
|
+
if node is None:
|
|
979
|
+
return None
|
|
980
|
+
if isinstance(node, ast.Name):
|
|
981
|
+
return self._lookup(node.id, frame)
|
|
982
|
+
if isinstance(node, ast.Attribute):
|
|
983
|
+
return self._attribute(node, frame)
|
|
984
|
+
if isinstance(node, ast.Call):
|
|
985
|
+
return self._call(node, frame)
|
|
986
|
+
if isinstance(node, ast.Subscript):
|
|
987
|
+
return self._subscript(node, frame)
|
|
988
|
+
if isinstance(node, (ast.Await, ast.Starred)):
|
|
989
|
+
return self._eval(node.value, frame)
|
|
990
|
+
if isinstance(node, ast.BoolOp):
|
|
991
|
+
values = [self._eval(v, frame) for v in node.values]
|
|
992
|
+
return next((v for v in values if v is not None), None)
|
|
993
|
+
if isinstance(node, ast.IfExp):
|
|
994
|
+
self._eval(node.test, frame)
|
|
995
|
+
first, second = self._eval(node.body, frame), self._eval(node.orelse, frame)
|
|
996
|
+
return first or second
|
|
997
|
+
if isinstance(node, ast.NamedExpr):
|
|
998
|
+
value = self._eval(node.value, frame)
|
|
999
|
+
self._bind(node.target, value, frame)
|
|
1000
|
+
return value
|
|
1001
|
+
if isinstance(node, ast.Lambda):
|
|
1002
|
+
return None
|
|
1003
|
+
if isinstance(node, (ast.JoinedStr, ast.BinOp)):
|
|
1004
|
+
for child in ast.iter_child_nodes(node):
|
|
1005
|
+
if isinstance(child, ast.expr):
|
|
1006
|
+
self._eval(child, frame)
|
|
1007
|
+
return self._text(node, frame)
|
|
1008
|
+
if isinstance(node, (ast.ListComp, ast.SetComp, ast.GeneratorExp, ast.DictComp)):
|
|
1009
|
+
saved = dict(frame.scope)
|
|
1010
|
+
depth = len(frame.loops)
|
|
1011
|
+
for gen in node.generators:
|
|
1012
|
+
self._bind_loop(gen.target, self._eval(gen.iter, frame), frame)
|
|
1013
|
+
frame.loops.append(f"for {_target(gen.target)} in {_unparse(gen.iter, 80)}")
|
|
1014
|
+
for cond in gen.ifs:
|
|
1015
|
+
self._eval(cond, frame)
|
|
1016
|
+
if isinstance(node, ast.DictComp):
|
|
1017
|
+
self._eval(node.key, frame)
|
|
1018
|
+
self._eval(node.value, frame)
|
|
1019
|
+
else:
|
|
1020
|
+
self._eval(node.elt, frame)
|
|
1021
|
+
del frame.loops[depth:]
|
|
1022
|
+
frame.scope.clear()
|
|
1023
|
+
frame.scope.update(saved)
|
|
1024
|
+
return None
|
|
1025
|
+
for child in ast.iter_child_nodes(node):
|
|
1026
|
+
if isinstance(child, ast.expr):
|
|
1027
|
+
self._eval(child, frame)
|
|
1028
|
+
return None
|
|
1029
|
+
|
|
1030
|
+
def _attribute(self, node: ast.Attribute, frame: _Frame) -> _Sym | None:
|
|
1031
|
+
base = self._eval(node.value, frame)
|
|
1032
|
+
attr = node.attr
|
|
1033
|
+
if base is None:
|
|
1034
|
+
return None
|
|
1035
|
+
kind = base.kind
|
|
1036
|
+
if kind == "mod":
|
|
1037
|
+
dotted = f"{base.name}.{attr}"
|
|
1038
|
+
if dotted in self.platform.input_objects:
|
|
1039
|
+
return _Sym("event")
|
|
1040
|
+
return _Sym("env") if dotted == "os.environ" else _Sym("mod", dotted)
|
|
1041
|
+
if kind == "own":
|
|
1042
|
+
sub = f"{base.name}.{attr}"
|
|
1043
|
+
if sub in self.sources:
|
|
1044
|
+
self._load_chain(sub)
|
|
1045
|
+
return _Sym("own", sub)
|
|
1046
|
+
return self._load(base.name).scope.get(attr)
|
|
1047
|
+
if kind in ("obj", "class", "super"):
|
|
1048
|
+
if kind == "super": # looked up in the classes it's made from, and bound to the same self
|
|
1049
|
+
found = next((m for b in self.classes[base.name]["bases"] for m in [self._member(b, attr)] if m), None)
|
|
1050
|
+
owner = base.inner or _Sym("obj", base.name)
|
|
1051
|
+
else:
|
|
1052
|
+
found, owner = self._member(base.name, attr), base
|
|
1053
|
+
if found is None:
|
|
1054
|
+
return None
|
|
1055
|
+
what, value = found
|
|
1056
|
+
if what == "attr":
|
|
1057
|
+
return value
|
|
1058
|
+
decorators = {getattr(d, "id", "") or getattr(d, "attr", "") for d in self.defs[value][1].decorator_list}
|
|
1059
|
+
if decorators & set(_PROPERTIES) and owner.kind != "class": # obj.size runs its getter
|
|
1060
|
+
return self._run(value, [], {}, frame, node, bound=owner)
|
|
1061
|
+
classmethod_ = "classmethod" in decorators
|
|
1062
|
+
if "staticmethod" in decorators or (owner.kind == "class" and not classmethod_):
|
|
1063
|
+
return _Sym("func", value)
|
|
1064
|
+
return _Sym("bound", value, inner=_Sym("class", owner.name) if classmethod_ else owner)
|
|
1065
|
+
if kind == "event":
|
|
1066
|
+
if attr in _DICT_METHODS:
|
|
1067
|
+
return _Sym("evmethod", attr, path=base.path)
|
|
1068
|
+
path = base.path + (attr if base.path else self.platform.input_aliases.get(attr, attr),)
|
|
1069
|
+
self._event_read(path, node, frame, required=True)
|
|
1070
|
+
return _Sym("event", path=path)
|
|
1071
|
+
if kind == "env":
|
|
1072
|
+
return _Sym("envget", attr) if attr in ("get", "setdefault", "pop") else None
|
|
1073
|
+
if kind == "session":
|
|
1074
|
+
return _Sym("mod", f"boto3.{attr}")
|
|
1075
|
+
if kind in ("client", "resource", "res", "coll", "paginator", "batch"):
|
|
1076
|
+
if attr == "meta":
|
|
1077
|
+
return _Sym("meta", base.name)
|
|
1078
|
+
if kind in ("res", "resource") and (base.name, attr) in _COLLECTIONS:
|
|
1079
|
+
return _Sym("coll", base.name, _COLLECTIONS[(base.name, attr)])
|
|
1080
|
+
return _Sym("method", base.name, attr, inner=base)
|
|
1081
|
+
if kind == "meta":
|
|
1082
|
+
return _Sym("client", base.name) if attr == "client" else None
|
|
1083
|
+
if kind == "logger":
|
|
1084
|
+
return _Sym("log", attr)
|
|
1085
|
+
if kind == "http":
|
|
1086
|
+
return _Sym("httpcall", base.name, attr, inner=base)
|
|
1087
|
+
if kind == "lib" and base.handle is not None:
|
|
1088
|
+
library = get_library(base.name)
|
|
1089
|
+
found = library.attribute(base.handle, attr) if library is not None else None
|
|
1090
|
+
return (
|
|
1091
|
+
_Sym("lib", base.name, handle=found)
|
|
1092
|
+
if found is not None
|
|
1093
|
+
else _Sym("libmember", base.name, attr, inner=base)
|
|
1094
|
+
)
|
|
1095
|
+
return None
|
|
1096
|
+
|
|
1097
|
+
def _subscript(self, node: ast.Subscript, frame: _Frame) -> _Sym | None:
|
|
1098
|
+
base = self._eval(node.value, frame)
|
|
1099
|
+
key = _const(node.slice)
|
|
1100
|
+
if not isinstance(node.slice, ast.Constant):
|
|
1101
|
+
self._eval(node.slice, frame)
|
|
1102
|
+
if base is None:
|
|
1103
|
+
return None
|
|
1104
|
+
if base.kind == "env" and isinstance(key, str):
|
|
1105
|
+
self._env_read(key, node, frame, required=True)
|
|
1106
|
+
return _Sym("envval", key)
|
|
1107
|
+
if base.kind == "event":
|
|
1108
|
+
if isinstance(key, str):
|
|
1109
|
+
path = base.path + (key,)
|
|
1110
|
+
self._event_read(path, node, frame, required=True)
|
|
1111
|
+
return _Sym("event", path=path)
|
|
1112
|
+
if isinstance(key, int):
|
|
1113
|
+
return _Sym("event", path=base.path + ("[]",))
|
|
1114
|
+
if isinstance(node.slice, ast.Slice):
|
|
1115
|
+
return base
|
|
1116
|
+
if base.kind == "lib" and base.handle is not None:
|
|
1117
|
+
library = get_library(base.name)
|
|
1118
|
+
found = library.item(base.handle, key) if library is not None else None
|
|
1119
|
+
return _Sym("lib", base.name, handle=found) if found is not None else None
|
|
1120
|
+
return None
|
|
1121
|
+
|
|
1122
|
+
def _call(self, node: ast.Call, frame: _Frame) -> _Sym | None:
|
|
1123
|
+
if isinstance(node.func, ast.Name) and node.func.id == "super" and self._lookup("super", frame) is None:
|
|
1124
|
+
return self._super(node, frame)
|
|
1125
|
+
func = self._eval(node.func, frame)
|
|
1126
|
+
args = [self._eval(a, frame) for a in node.args]
|
|
1127
|
+
kwargs = {}
|
|
1128
|
+
for kw in node.keywords:
|
|
1129
|
+
value = self._eval(kw.value, frame)
|
|
1130
|
+
if kw.arg:
|
|
1131
|
+
kwargs[kw.arg] = value
|
|
1132
|
+
self.keywords = (node, kwargs)
|
|
1133
|
+
self.values = (
|
|
1134
|
+
node,
|
|
1135
|
+
{id(a): v for a, v in zip(node.args, args)}
|
|
1136
|
+
| {id(kw.value): kwargs.get(kw.arg) for kw in node.keywords if kw.arg},
|
|
1137
|
+
)
|
|
1138
|
+
if (
|
|
1139
|
+
isinstance(node.func, ast.Attribute)
|
|
1140
|
+
and node.func.attr == "format"
|
|
1141
|
+
and (isinstance(_const(node.func.value), str) or (func is None and _is_text(node.func.value)))
|
|
1142
|
+
):
|
|
1143
|
+
return self._text(node, frame) # "... {}".format(x)
|
|
1144
|
+
if func is None:
|
|
1145
|
+
return self._unknown_call(node, frame)
|
|
1146
|
+
kind = func.kind
|
|
1147
|
+
if kind == "func":
|
|
1148
|
+
return self._run(func.name, args, kwargs, frame, node)
|
|
1149
|
+
if kind == "bound":
|
|
1150
|
+
return self._run(func.name, args, kwargs, frame, node, bound=func.inner)
|
|
1151
|
+
if kind == "class":
|
|
1152
|
+
init = self._member(func.name, "__init__")
|
|
1153
|
+
obj = _Sym("obj", func.name)
|
|
1154
|
+
if init and init[0] == "method":
|
|
1155
|
+
self._run(init[1], args, kwargs, frame, node, bound=obj)
|
|
1156
|
+
return obj
|
|
1157
|
+
if kind == "mod":
|
|
1158
|
+
return self._outside(func.name, node, args, frame)
|
|
1159
|
+
if kind == "builtin":
|
|
1160
|
+
return self._builtin(func.name, node, args, frame)
|
|
1161
|
+
if kind == "method":
|
|
1162
|
+
return self._aws_method(func, node, frame)
|
|
1163
|
+
if kind == "event": # a method of the input that gives a part of it: request.get_json(), request.json()
|
|
1164
|
+
return func
|
|
1165
|
+
if kind == "envget":
|
|
1166
|
+
name = _const(node.args[0]) if node.args else None
|
|
1167
|
+
if isinstance(name, str):
|
|
1168
|
+
default = (
|
|
1169
|
+
node.args[1]
|
|
1170
|
+
if len(node.args) > 1
|
|
1171
|
+
else next((kw.value for kw in node.keywords if kw.arg == "default"), None)
|
|
1172
|
+
)
|
|
1173
|
+
self._env_read(
|
|
1174
|
+
name, node, frame, required=False, default=_unparse(default, 60) if default is not None else ""
|
|
1175
|
+
)
|
|
1176
|
+
return _Sym("envval", name)
|
|
1177
|
+
return None
|
|
1178
|
+
if kind == "evmethod":
|
|
1179
|
+
key = _const(node.args[0]) if node.args else None
|
|
1180
|
+
if func.name in ("get", "pop", "setdefault") and isinstance(key, str):
|
|
1181
|
+
path = func.path + (key,)
|
|
1182
|
+
self._event_read(path, node, frame, required=False)
|
|
1183
|
+
return _Sym("event", path=path)
|
|
1184
|
+
return None
|
|
1185
|
+
if kind == "log":
|
|
1186
|
+
if func.name in _LOG_LEVELS:
|
|
1187
|
+
self.cur.note("log", _LOG_LEVELS[func.name])
|
|
1188
|
+
return None
|
|
1189
|
+
if kind == "httpcall":
|
|
1190
|
+
if func.sub in _HTTP_METHODS:
|
|
1191
|
+
self._http(func.name, func.sub, node, frame, given="timeout" in (func.inner.path if func.inner else ()))
|
|
1192
|
+
return None
|
|
1193
|
+
if kind == "libmember" and func.inner is not None and func.inner.handle is not None:
|
|
1194
|
+
library = get_library(func.name)
|
|
1195
|
+
if library is None:
|
|
1196
|
+
return None
|
|
1197
|
+
view = self._view(func.sub, node, frame)
|
|
1198
|
+
return self._library(library, library.call(func.inner.handle, func.sub, view), view, frame)
|
|
1199
|
+
return None
|
|
1200
|
+
|
|
1201
|
+
def _super(self, node: ast.Call, frame: _Frame) -> _Sym | None:
|
|
1202
|
+
"""
|
|
1203
|
+
super() in a method (or super(Class, self)): the class it's in and its self (or cls), so what's read from it
|
|
1204
|
+
is looked for in the classes that class is made from.
|
|
1205
|
+
"""
|
|
1206
|
+
if len(node.args) == 2:
|
|
1207
|
+
cls, owner = self._eval(node.args[0], frame), self._eval(node.args[1], frame)
|
|
1208
|
+
key = cls.name if cls is not None and cls.kind == "class" else None
|
|
1209
|
+
else:
|
|
1210
|
+
_, function, key = self.defs.get(frame.function, (None, None, None))
|
|
1211
|
+
params = function.args.posonlyargs + function.args.args if function is not None else []
|
|
1212
|
+
owner = frame.scope.get(params[0].arg) if params else None
|
|
1213
|
+
return _Sym("super", key, inner=owner) if key in self.classes else None
|
|
1214
|
+
|
|
1215
|
+
def _unknown_call(self, node: ast.Call, frame: _Frame) -> _Sym | None:
|
|
1216
|
+
"""
|
|
1217
|
+
A call to something it couldn't follow: a method name only AWS has, a file written with pandas, a log
|
|
1218
|
+
line, or a web framework's router.
|
|
1219
|
+
"""
|
|
1220
|
+
if not isinstance(node.func, ast.Attribute):
|
|
1221
|
+
return None
|
|
1222
|
+
attr = node.func.attr
|
|
1223
|
+
receiver = _unparse(node.func.value, 60)
|
|
1224
|
+
if attr in _GUESSES:
|
|
1225
|
+
service, op = _GUESSES[attr]
|
|
1226
|
+
self._aws(service, op, node, frame, guessed=True)
|
|
1227
|
+
elif attr in _FILE_WRITERS and node.args:
|
|
1228
|
+
self._write(node.args[0], node, frame)
|
|
1229
|
+
elif attr in _LOG_LEVELS and "log" in receiver.lower():
|
|
1230
|
+
self.cur.note("log", _LOG_LEVELS[attr])
|
|
1231
|
+
elif attr == "register" and len(node.args) >= 2 and isinstance(_const(node.args[0]), str):
|
|
1232
|
+
self._viewset(str(_const(node.args[0])), node, f"{frame.module}.{receiver}", frame)
|
|
1233
|
+
elif attr == "resolve" and any(router == receiver for *_, router in self.routes):
|
|
1234
|
+
for method, path, key, router in self.routes:
|
|
1235
|
+
if router == receiver:
|
|
1236
|
+
self._run(key, [], {}, frame, node, label=f"{method} {path}")
|
|
1237
|
+
return None
|
|
1238
|
+
|
|
1239
|
+
def _outside(self, name: str, node: ast.Call, args: list[_Sym | None], frame: _Frame) -> _Sym | None:
|
|
1240
|
+
"""
|
|
1241
|
+
A call to something from outside the package: boto3, os, json, requests, logging, time...
|
|
1242
|
+
"""
|
|
1243
|
+
first = args[0] if args else None
|
|
1244
|
+
last = name.rsplit(".", 1)[-1]
|
|
1245
|
+
if name in _URL_INCLUDES:
|
|
1246
|
+
return self._include(node, frame)
|
|
1247
|
+
if name in _URL_PATHS:
|
|
1248
|
+
self._url(name, node, args, frame)
|
|
1249
|
+
return None
|
|
1250
|
+
library = library_for(name.split(".")[0])
|
|
1251
|
+
if library is not None:
|
|
1252
|
+
view = self._view(name, node, frame)
|
|
1253
|
+
return self._library(library, library.make(name, view), view, frame)
|
|
1254
|
+
if name.startswith("boto3") and last in ("client", "resource"):
|
|
1255
|
+
return self._make_client(last, node, frame)
|
|
1256
|
+
if name in ("boto3.Session", "boto3.session.Session"):
|
|
1257
|
+
return _Sym("session")
|
|
1258
|
+
if name == "os.getenv":
|
|
1259
|
+
key = _const(node.args[0]) if node.args else None
|
|
1260
|
+
if isinstance(key, str):
|
|
1261
|
+
default = node.args[1] if len(node.args) > 1 else None
|
|
1262
|
+
self._env_read(
|
|
1263
|
+
key, node, frame, required=False, default=_unparse(default, 60) if default is not None else ""
|
|
1264
|
+
)
|
|
1265
|
+
return _Sym("envval", key)
|
|
1266
|
+
return None
|
|
1267
|
+
if name in _JSON_LOADS or name in _B64_DECODES:
|
|
1268
|
+
if first is not None and first.kind == "event":
|
|
1269
|
+
return _Sym("event", path=first.path + ("{json}" if name in _JSON_LOADS else "{base64}",))
|
|
1270
|
+
return None
|
|
1271
|
+
if name in ("urllib.parse.unquote_plus", "urllib.parse.unquote", "str.strip"):
|
|
1272
|
+
return first
|
|
1273
|
+
if name == "logging.getLogger" or last == "Logger":
|
|
1274
|
+
return _Sym("logger")
|
|
1275
|
+
if name.startswith("logging.") and last in _LOG_LEVELS:
|
|
1276
|
+
self.cur.note("log", _LOG_LEVELS[last])
|
|
1277
|
+
return None
|
|
1278
|
+
top = name.split(".")[0]
|
|
1279
|
+
if name in _HTTP_SESSIONS:
|
|
1280
|
+
given = any(kw.arg == "timeout" for kw in node.keywords) or (top == "http" and len(node.args) >= 3)
|
|
1281
|
+
return _Sym("http", _HTTP_SESSIONS[name], path=("timeout",) if given else ())
|
|
1282
|
+
if top in ("requests", "httpx") and last in _HTTP_METHODS and name.count(".") == 1:
|
|
1283
|
+
self._http(top, last, node, frame)
|
|
1284
|
+
return None
|
|
1285
|
+
if name == "urllib.request.urlopen":
|
|
1286
|
+
self._http("urllib.request", "get", node, frame, given=len(node.args) >= 3)
|
|
1287
|
+
return None
|
|
1288
|
+
if name in ("time.sleep", "asyncio.sleep"):
|
|
1289
|
+
wait = _unparse(node.args[0], 30) if node.args else "?"
|
|
1290
|
+
self._spot(
|
|
1291
|
+
"sleep", frame.file, node.lineno, frame.function, _unparse(node, 80), wait, loop=self._loop_of(frame)
|
|
1292
|
+
)
|
|
1293
|
+
self.cur.note(
|
|
1294
|
+
"sleep",
|
|
1295
|
+
f"waits {wait} s"
|
|
1296
|
+
if isinstance(_const(node.args[0] if node.args else None), (int, float))
|
|
1297
|
+
else f"waits ({wait})",
|
|
1298
|
+
)
|
|
1299
|
+
return None
|
|
1300
|
+
if name in _COMMANDS:
|
|
1301
|
+
shell = name in ("os.system", "os.popen") or any(
|
|
1302
|
+
kw.arg == "shell" and _const(kw.value) is True for kw in node.keywords
|
|
1303
|
+
)
|
|
1304
|
+
inputs = self._inputs(node.args[0], frame) if node.args and shell else ()
|
|
1305
|
+
self._spot(
|
|
1306
|
+
"command", frame.file, node.lineno, frame.function, _unparse(node, 80), ", ".join(inputs), ok=not inputs
|
|
1307
|
+
)
|
|
1308
|
+
self.cur.note("command", "runs a command")
|
|
1309
|
+
if inputs:
|
|
1310
|
+
self.cur.note("warn", f"a shell command built from {inputs[0]}")
|
|
1311
|
+
return None
|
|
1312
|
+
if (
|
|
1313
|
+
top == "pandas"
|
|
1314
|
+
and last.startswith("read_")
|
|
1315
|
+
and isinstance(_const(node.args[0] if node.args else None), str)
|
|
1316
|
+
):
|
|
1317
|
+
if str(_const(node.args[0])).startswith("s3://"):
|
|
1318
|
+
self._aws("s3", "GetObject", node, frame)
|
|
1319
|
+
return None
|
|
1320
|
+
|
|
1321
|
+
def _builtin(self, name: str, node: ast.Call, args: list[_Sym | None], frame: _Frame) -> _Sym | None:
|
|
1322
|
+
if name == "print":
|
|
1323
|
+
self.cur.note("log", "prints")
|
|
1324
|
+
elif name == "open" and node.args:
|
|
1325
|
+
mode = (
|
|
1326
|
+
_const(node.args[1])
|
|
1327
|
+
if len(node.args) > 1
|
|
1328
|
+
else next((_const(kw.value) for kw in node.keywords if kw.arg == "mode"), "r")
|
|
1329
|
+
)
|
|
1330
|
+
if isinstance(mode, str) and any(c in mode for c in "wax+"):
|
|
1331
|
+
self._write(node.args[0], node, frame)
|
|
1332
|
+
else:
|
|
1333
|
+
path = _const(node.args[0])
|
|
1334
|
+
self.cur.note("file", f"reads {_clip(path, 40)}" if isinstance(path, str) else "reads a file")
|
|
1335
|
+
elif name == "enumerate" and args and args[0] is not None:
|
|
1336
|
+
return _Sym("enumerate", inner=args[0])
|
|
1337
|
+
elif name in ("sorted", "reversed", "list", "tuple", "iter") and args:
|
|
1338
|
+
return args[0]
|
|
1339
|
+
return None
|
|
1340
|
+
|
|
1341
|
+
def _write(self, path_node: Any, node: ast.Call, frame: _Frame) -> None:
|
|
1342
|
+
"""
|
|
1343
|
+
A file the code writes, and where: its platform says whether it can be written there.
|
|
1344
|
+
"""
|
|
1345
|
+
prefix = None
|
|
1346
|
+
if isinstance(_const(path_node), str):
|
|
1347
|
+
prefix = str(_const(path_node))
|
|
1348
|
+
elif isinstance(path_node, ast.JoinedStr) and path_node.values and isinstance(_const(path_node.values[0]), str):
|
|
1349
|
+
prefix = str(_const(path_node.values[0]))
|
|
1350
|
+
if prefix is None:
|
|
1351
|
+
self.cur.note("file", "writes a file")
|
|
1352
|
+
return
|
|
1353
|
+
if prefix.startswith("s3://"):
|
|
1354
|
+
self._aws("s3", "PutObject", node, frame)
|
|
1355
|
+
return
|
|
1356
|
+
self._spot("write", frame.file, node.lineno, frame.function, _unparse(node, 80), prefix)
|
|
1357
|
+
self.cur.note("file", f"writes {_clip(prefix, 40)}")
|
|
1358
|
+
|
|
1359
|
+
def _make_client(self, kind: str, node: ast.Call, frame: _Frame) -> _Sym:
|
|
1360
|
+
service = (
|
|
1361
|
+
_const(node.args[0])
|
|
1362
|
+
if node.args
|
|
1363
|
+
else next((_const(kw.value) for kw in node.keywords if kw.arg == "service_name"), None)
|
|
1364
|
+
)
|
|
1365
|
+
service = service if isinstance(service, str) else "?"
|
|
1366
|
+
once = self._once(frame)
|
|
1367
|
+
name = _service_name(service) if service != "?" else "an AWS"
|
|
1368
|
+
self._spot(
|
|
1369
|
+
"client",
|
|
1370
|
+
frame.file,
|
|
1371
|
+
node.lineno,
|
|
1372
|
+
frame.function,
|
|
1373
|
+
_unparse(node, 80),
|
|
1374
|
+
service,
|
|
1375
|
+
ok=once,
|
|
1376
|
+
loop=self._loop_of(frame),
|
|
1377
|
+
)
|
|
1378
|
+
self.cur.note("client", f"makes {_a(name)} {name} {'client' if kind == 'client' else 'resource'}")
|
|
1379
|
+
return _Sym(kind if kind == "client" else "resource", service)
|
|
1380
|
+
|
|
1381
|
+
def _aws_method(self, func: _Sym, node: ast.Call, frame: _Frame) -> _Sym | None:
|
|
1382
|
+
owner, service, method = func.inner, func.name, func.sub
|
|
1383
|
+
if owner is None:
|
|
1384
|
+
return None
|
|
1385
|
+
if owner.kind == "client":
|
|
1386
|
+
if method == "get_paginator":
|
|
1387
|
+
op = self._op(service, _const(node.args[0]) if node.args else None)
|
|
1388
|
+
return _Sym("paginator", service, op) if op else None
|
|
1389
|
+
if method == "get_waiter":
|
|
1390
|
+
self.cur.note(
|
|
1391
|
+
"aws-reads", f"{_service_name(service)} waiter {_const(node.args[0]) if node.args else ''}".strip()
|
|
1392
|
+
)
|
|
1393
|
+
return None
|
|
1394
|
+
if method in ("generate_presigned_url", "generate_presigned_post"):
|
|
1395
|
+
what = (
|
|
1396
|
+
_const(node.args[0])
|
|
1397
|
+
if node.args and method.endswith("url")
|
|
1398
|
+
else next((_const(kw.value) for kw in node.keywords if kw.arg == "ClientMethod"), None)
|
|
1399
|
+
)
|
|
1400
|
+
op = (
|
|
1401
|
+
self._op(service, what)
|
|
1402
|
+
if isinstance(what, str)
|
|
1403
|
+
else ("PutObject" if method.endswith("post") else None)
|
|
1404
|
+
)
|
|
1405
|
+
if op:
|
|
1406
|
+
self._aws(service, op, node, frame, access="signs")
|
|
1407
|
+
return None
|
|
1408
|
+
if service == "s3" and method in _TRANSFERS:
|
|
1409
|
+
self._aws(service, _TRANSFERS[method], node, frame)
|
|
1410
|
+
return None
|
|
1411
|
+
if service == "?" and method in _GUESSES:
|
|
1412
|
+
self._aws(*_GUESSES[method], node, frame, guessed=True)
|
|
1413
|
+
return None
|
|
1414
|
+
op = self._op(service, method)
|
|
1415
|
+
if op:
|
|
1416
|
+
self._aws(service, op, node, frame)
|
|
1417
|
+
return None
|
|
1418
|
+
if owner.kind == "resource":
|
|
1419
|
+
if method[:1].isupper():
|
|
1420
|
+
return _Sym("res", service, method)
|
|
1421
|
+
if method == "get_queue_by_name":
|
|
1422
|
+
self._aws("sqs", "GetQueueUrl", node, frame)
|
|
1423
|
+
return _Sym("res", "sqs", "Queue")
|
|
1424
|
+
op = self._op(service, method)
|
|
1425
|
+
if op:
|
|
1426
|
+
self._aws(service, op, node, frame)
|
|
1427
|
+
return None
|
|
1428
|
+
if owner.kind == "res":
|
|
1429
|
+
if method[:1].isupper():
|
|
1430
|
+
return _Sym("res", service, method)
|
|
1431
|
+
if method == "batch_writer":
|
|
1432
|
+
return _Sym("batch", service, "BatchWriteItem")
|
|
1433
|
+
op = _RESOURCE_ACTIONS.get((service, owner.sub), {}).get(method) or self._op(service, method)
|
|
1434
|
+
if op:
|
|
1435
|
+
self._aws(service, op, node, frame)
|
|
1436
|
+
return None
|
|
1437
|
+
if owner.kind == "coll":
|
|
1438
|
+
if method in ("all", "filter", "limit", "page_size", "pages"):
|
|
1439
|
+
self._aws(service, owner.sub, node, frame, pages="all")
|
|
1440
|
+
return owner
|
|
1441
|
+
if method == "delete":
|
|
1442
|
+
self._aws(service, "DeleteObjects", node, frame)
|
|
1443
|
+
return None
|
|
1444
|
+
if owner.kind == "paginator" and method == "paginate":
|
|
1445
|
+
self._aws(service, owner.sub, node, frame, pages="all")
|
|
1446
|
+
elif owner.kind == "batch" and method in ("put_item", "delete_item"):
|
|
1447
|
+
self._aws(service, owner.sub, node, frame)
|
|
1448
|
+
return None
|
|
1449
|
+
|
|
1450
|
+
# ---------------------------------------------------------------- calls into its own code
|
|
1451
|
+
|
|
1452
|
+
def _run(
|
|
1453
|
+
self,
|
|
1454
|
+
key: str,
|
|
1455
|
+
args: list[_Sym | None],
|
|
1456
|
+
kwargs: dict[str, _Sym | None],
|
|
1457
|
+
caller: _Frame | None,
|
|
1458
|
+
call: Any,
|
|
1459
|
+
bound: _Sym | None = None,
|
|
1460
|
+
label: str = "",
|
|
1461
|
+
) -> _Sym | None:
|
|
1462
|
+
"""
|
|
1463
|
+
One of its own functions, called: its steps (shown the first time), and what it returns.
|
|
1464
|
+
"""
|
|
1465
|
+
module, node, cls = self.defs[key]
|
|
1466
|
+
file = self.modules[module].file or ""
|
|
1467
|
+
depth = caller.depth + 1 if caller is not None and caller.function != caller.module else 0
|
|
1468
|
+
info = self.reached.get(key)
|
|
1469
|
+
if info is None:
|
|
1470
|
+
end = getattr(node, "end_lineno", None) or node.lineno
|
|
1471
|
+
info = self.reached[key] = CodeFunction(
|
|
1472
|
+
key,
|
|
1473
|
+
file,
|
|
1474
|
+
node.lineno,
|
|
1475
|
+
end - node.lineno + 1,
|
|
1476
|
+
_unparse(node.args, 100),
|
|
1477
|
+
_doc_line(node),
|
|
1478
|
+
depth=depth,
|
|
1479
|
+
is_async=isinstance(node, ast.AsyncFunctionDef),
|
|
1480
|
+
)
|
|
1481
|
+
info.depth = min(info.depth, depth)
|
|
1482
|
+
if caller is not None and call is not None:
|
|
1483
|
+
where = f"{caller.function} line {call.lineno}"
|
|
1484
|
+
if where not in info.called_from:
|
|
1485
|
+
info.called_from.append(where)
|
|
1486
|
+
if caller.guards: # the if nearest around the call, else the early exit above it nearest
|
|
1487
|
+
around = [g for i, g in enumerate(caller.guards) if i not in caller.early]
|
|
1488
|
+
info.when[where] = (around or caller.guards)[-1]
|
|
1489
|
+
step = CodeStep(
|
|
1490
|
+
file, node.lineno, _def_line(node), "def", name=key, when=list(caller.guards) if caller is not None else []
|
|
1491
|
+
)
|
|
1492
|
+
if label:
|
|
1493
|
+
step.notes.append(("route", label))
|
|
1494
|
+
self.cur.calls.append(step)
|
|
1495
|
+
if key in self.stack:
|
|
1496
|
+
step.notes.append(("more", "calls itself: not followed again"))
|
|
1497
|
+
step.seen = True
|
|
1498
|
+
return None
|
|
1499
|
+
if depth > self.max_depth or self.count > _MAP_STEPS or self.runs > _MAP_RUNS:
|
|
1500
|
+
step.notes.append(
|
|
1501
|
+
(
|
|
1502
|
+
"more",
|
|
1503
|
+
"not opened: "
|
|
1504
|
+
+ ("too many calls deep" if depth > self.max_depth else "the outline is long enough"),
|
|
1505
|
+
)
|
|
1506
|
+
)
|
|
1507
|
+
step.seen = True
|
|
1508
|
+
self.cmap.limited = (
|
|
1509
|
+
f"followed calls {self.max_depth} deep"
|
|
1510
|
+
if depth > self.max_depth
|
|
1511
|
+
else f"stopped opening functions after {_MAP_STEPS:,} lines"
|
|
1512
|
+
if self.count > _MAP_STEPS
|
|
1513
|
+
else f"stopped after reading {_MAP_RUNS:,} function bodies"
|
|
1514
|
+
)
|
|
1515
|
+
return None
|
|
1516
|
+
a = node.args
|
|
1517
|
+
params = [p.arg for p in a.posonlyargs + a.args]
|
|
1518
|
+
values = ([bound] if bound is not None else []) + list(args)
|
|
1519
|
+
scope: dict[str, _Sym | None] = dict(zip(params, values))
|
|
1520
|
+
for name in params[len(values) :] + [p.arg for p in a.kwonlyargs]:
|
|
1521
|
+
scope.setdefault(name, kwargs.get(name))
|
|
1522
|
+
for extra in (a.vararg, a.kwarg):
|
|
1523
|
+
if extra is not None:
|
|
1524
|
+
scope[extra.arg] = None
|
|
1525
|
+
loops = list(caller.loops) if caller is not None else []
|
|
1526
|
+
memo = (
|
|
1527
|
+
key,
|
|
1528
|
+
bool(loops),
|
|
1529
|
+
tuple(sorted(((k, v) for k, v in scope.items() if v is not None), key=lambda kv: kv[0])),
|
|
1530
|
+
)
|
|
1531
|
+
if memo in self.memo:
|
|
1532
|
+
step.seen = key in self.shown
|
|
1533
|
+
if not step.seen:
|
|
1534
|
+
self.shown.add(key)
|
|
1535
|
+
return self.memo[memo]
|
|
1536
|
+
self.memo[memo] = None
|
|
1537
|
+
self.runs += 1
|
|
1538
|
+
frame = _Frame(
|
|
1539
|
+
module,
|
|
1540
|
+
file,
|
|
1541
|
+
key,
|
|
1542
|
+
scope,
|
|
1543
|
+
node=node,
|
|
1544
|
+
outers=self.closures.get(key, []),
|
|
1545
|
+
loops=loops,
|
|
1546
|
+
cached=key in self.cached,
|
|
1547
|
+
depth=depth,
|
|
1548
|
+
)
|
|
1549
|
+
self.stack.append(key)
|
|
1550
|
+
try:
|
|
1551
|
+
steps = self._block(node.body, frame)
|
|
1552
|
+
finally:
|
|
1553
|
+
self.stack.pop()
|
|
1554
|
+
value = next((v for v in frame.returns if v is not None), None)
|
|
1555
|
+
self.memo[memo] = value
|
|
1556
|
+
if key in self.shown:
|
|
1557
|
+
step.seen = True
|
|
1558
|
+
else:
|
|
1559
|
+
self.shown.add(key)
|
|
1560
|
+
step.steps = steps
|
|
1561
|
+
return value
|
|
1562
|
+
|
|
1563
|
+
# ---------------------------------------------------------------- what it reaches
|
|
1564
|
+
|
|
1565
|
+
def _loop_of(self, frame: _Frame) -> str:
|
|
1566
|
+
return frame.loops[-1] if frame.loops else ""
|
|
1567
|
+
|
|
1568
|
+
def _op(self, service: str, method: Any) -> str | None:
|
|
1569
|
+
"""
|
|
1570
|
+
A client method's operation: ('s3', 'get_object') -> 'GetObject'; None when it isn't one.
|
|
1571
|
+
"""
|
|
1572
|
+
if not isinstance(method, str):
|
|
1573
|
+
return None
|
|
1574
|
+
model = _aws_model(service)
|
|
1575
|
+
if model is None:
|
|
1576
|
+
return _GUESSES[method][1] if method in _GUESSES else None
|
|
1577
|
+
return model["ops"].get(method)
|
|
1578
|
+
|
|
1579
|
+
def _aws(
|
|
1580
|
+
self,
|
|
1581
|
+
service: str,
|
|
1582
|
+
op: str,
|
|
1583
|
+
node: ast.Call,
|
|
1584
|
+
frame: _Frame,
|
|
1585
|
+
*,
|
|
1586
|
+
access: str | None = None,
|
|
1587
|
+
pages: str | None = None,
|
|
1588
|
+
guessed: bool = False,
|
|
1589
|
+
) -> None:
|
|
1590
|
+
access = access or _access(op)
|
|
1591
|
+
if pages is None:
|
|
1592
|
+
pages = self._pages(service, op, node, frame)
|
|
1593
|
+
key = (service, op, frame.file, node.lineno)
|
|
1594
|
+
args = self._sources(node)
|
|
1595
|
+
if key not in self.seen:
|
|
1596
|
+
self.seen.add(key)
|
|
1597
|
+
self.cmap.aws.append(
|
|
1598
|
+
AwsCall(
|
|
1599
|
+
service,
|
|
1600
|
+
op,
|
|
1601
|
+
access,
|
|
1602
|
+
_iam_action(service, op),
|
|
1603
|
+
frame.file,
|
|
1604
|
+
node.lineno,
|
|
1605
|
+
frame.function,
|
|
1606
|
+
_unparse(node, 120),
|
|
1607
|
+
self._loop_of(frame),
|
|
1608
|
+
pages,
|
|
1609
|
+
guessed,
|
|
1610
|
+
self.loading > 0,
|
|
1611
|
+
args,
|
|
1612
|
+
)
|
|
1613
|
+
)
|
|
1614
|
+
self.cur.note(f"aws-{access}", f"{_service_name(service)} {op}")
|
|
1615
|
+
for arg, source in args:
|
|
1616
|
+
if (f"{_service_name(service)} {op} {arg}", source) not in self.cur.given:
|
|
1617
|
+
self.cur.given.append((f"{_service_name(service)} {op} {arg}", source))
|
|
1618
|
+
|
|
1619
|
+
def _sources(self, node: ast.Call) -> list[tuple[str, str]]:
|
|
1620
|
+
"""
|
|
1621
|
+
Where a call's keyword arguments come from, the ones that can be told: [('Bucket', 'env ARCHIVE_BUCKET'),
|
|
1622
|
+
('Key', "f'{prefix}/{name}'")]: what a name holds, or the literal or f-string written there.
|
|
1623
|
+
"""
|
|
1624
|
+
values = self.keywords[1] if self.keywords[0] is node else {}
|
|
1625
|
+
found = []
|
|
1626
|
+
for kw in node.keywords:
|
|
1627
|
+
if not kw.arg:
|
|
1628
|
+
continue
|
|
1629
|
+
said = self._describe(values.get(kw.arg))
|
|
1630
|
+
if not said and isinstance(kw.value, (ast.Constant, ast.JoinedStr)):
|
|
1631
|
+
said = _unparse(kw.value, 50)
|
|
1632
|
+
if said:
|
|
1633
|
+
found.append((kw.arg, said))
|
|
1634
|
+
return found
|
|
1635
|
+
|
|
1636
|
+
def _pages(self, service: str, op: str, node: ast.Call, frame: _Frame) -> str:
|
|
1637
|
+
"""
|
|
1638
|
+
'one' when a list call reads only its first page, 'all' when the code follows the next-page token, ''
|
|
1639
|
+
when the call doesn't come in pages, or asks for a set number.
|
|
1640
|
+
"""
|
|
1641
|
+
model = _aws_model(service)
|
|
1642
|
+
if model is None or op not in model["pages"]:
|
|
1643
|
+
return ""
|
|
1644
|
+
inputs, outputs, limit = model["pages"][op]
|
|
1645
|
+
given = {kw.arg for kw in node.keywords if kw.arg}
|
|
1646
|
+
if limit and limit in given:
|
|
1647
|
+
return ""
|
|
1648
|
+
if given & inputs or any(kw.arg is None for kw in node.keywords):
|
|
1649
|
+
return "all"
|
|
1650
|
+
if frame.words is None:
|
|
1651
|
+
tree = (
|
|
1652
|
+
frame.node
|
|
1653
|
+
if frame.node is not None
|
|
1654
|
+
else (self.modules[frame.module].tree if frame.module in self.modules else None)
|
|
1655
|
+
)
|
|
1656
|
+
frame.words = set()
|
|
1657
|
+
for n in ast.walk(tree) if tree is not None else []:
|
|
1658
|
+
if isinstance(n, ast.Name):
|
|
1659
|
+
frame.words.add(n.id)
|
|
1660
|
+
elif isinstance(n, ast.Attribute):
|
|
1661
|
+
frame.words.add(n.attr)
|
|
1662
|
+
elif isinstance(n, ast.Constant) and isinstance(n.value, str):
|
|
1663
|
+
frame.words.add(n.value)
|
|
1664
|
+
return "all" if any(word in frame.words for word in outputs) else "one"
|
|
1665
|
+
|
|
1666
|
+
def _env_read(self, name: str, node: Any, frame: _Frame, *, required: bool, default: str = "") -> None:
|
|
1667
|
+
key = ("env", name, frame.file, node.lineno)
|
|
1668
|
+
if key not in self.seen:
|
|
1669
|
+
self.seen.add(key)
|
|
1670
|
+
self.cmap.env.append(
|
|
1671
|
+
EnvRead(name, frame.file, node.lineno, frame.function, required, default, self.loading > 0)
|
|
1672
|
+
)
|
|
1673
|
+
self.cur.note("env", name)
|
|
1674
|
+
|
|
1675
|
+
def _event_read(self, path: tuple[str, ...], node: Any, frame: _Frame, *, required: bool) -> None:
|
|
1676
|
+
if "*" in path:
|
|
1677
|
+
return
|
|
1678
|
+
found = self.event_reads.get(path)
|
|
1679
|
+
if found is None:
|
|
1680
|
+
self.event_reads[path] = EventField(path, frame.file, node.lineno, required)
|
|
1681
|
+
elif required and not found.required:
|
|
1682
|
+
found.required = True
|
|
1683
|
+
self.cur.note("event", self._input_label(path))
|
|
1684
|
+
|
|
1685
|
+
def _input_label(self, path: tuple[str, ...]) -> str:
|
|
1686
|
+
"""
|
|
1687
|
+
A field of the input as said beside a step, in its platform's word: 'event Records[].body', 'request
|
|
1688
|
+
query.q'.
|
|
1689
|
+
"""
|
|
1690
|
+
return f"{self.platform.input_word} {_event_label(path)}"
|
|
1691
|
+
|
|
1692
|
+
def _http(self, library: str, method: str, node: ast.Call, frame: _Frame, given: bool = False) -> None:
|
|
1693
|
+
args = list(node.args)
|
|
1694
|
+
if method == "request" and args: # requests.request('POST', url), pool.request('GET', url)
|
|
1695
|
+
verb = _const(args[0])
|
|
1696
|
+
method, args = (str(verb).lower() if isinstance(verb, str) else "request"), args[1:]
|
|
1697
|
+
url = args[0] if args else next((kw.value for kw in node.keywords if kw.arg == "url"), None)
|
|
1698
|
+
has_timeout = given or library in _DEFAULT_TIMEOUTS or any(kw.arg == "timeout" for kw in node.keywords)
|
|
1699
|
+
verb = method.upper() if method in _HTTP_METHODS and method != "request" else "request"
|
|
1700
|
+
target = _unparse(url, 60) if url is not None else ""
|
|
1701
|
+
self._spot(
|
|
1702
|
+
"http",
|
|
1703
|
+
frame.file,
|
|
1704
|
+
node.lineno,
|
|
1705
|
+
frame.function,
|
|
1706
|
+
_unparse(node, 100),
|
|
1707
|
+
f"{verb} {target}".strip(),
|
|
1708
|
+
ok=has_timeout,
|
|
1709
|
+
loop=self._loop_of(frame),
|
|
1710
|
+
)
|
|
1711
|
+
shown = _const(url)
|
|
1712
|
+
self.cur.note("http", f"HTTP {verb}" + (f" {_clip(str(shown), 40)}" if isinstance(shown, str) else ""))
|
|
1713
|
+
|
|
1714
|
+
def _spot(
|
|
1715
|
+
self,
|
|
1716
|
+
kind: str,
|
|
1717
|
+
file: str,
|
|
1718
|
+
line: int,
|
|
1719
|
+
function: str,
|
|
1720
|
+
code: str,
|
|
1721
|
+
detail: str = "",
|
|
1722
|
+
*,
|
|
1723
|
+
ok: bool = True,
|
|
1724
|
+
loop: str = "",
|
|
1725
|
+
waits: str = "",
|
|
1726
|
+
fix: str = "",
|
|
1727
|
+
) -> None:
|
|
1728
|
+
key = (kind, file, line, detail)
|
|
1729
|
+
if key not in self.seen:
|
|
1730
|
+
self.seen.add(key)
|
|
1731
|
+
self.cmap.spots.append(
|
|
1732
|
+
CodeSpot(kind, file, line, function, code, detail, ok, loop, self.loading > 0, waits, fix)
|
|
1733
|
+
)
|
|
1734
|
+
|
|
1735
|
+
# ---------------------------------------------------------------- libraries plugins know, and text from the input
|
|
1736
|
+
|
|
1737
|
+
def _once(self, frame: _Frame) -> bool:
|
|
1738
|
+
"""
|
|
1739
|
+
Whether what this line makes is made once and kept: at import, in a cached function, kept in a global, or
|
|
1740
|
+
made only when there isn't one yet (`if client is None:`).
|
|
1741
|
+
"""
|
|
1742
|
+
return (
|
|
1743
|
+
self.loading > 0
|
|
1744
|
+
or frame.cached
|
|
1745
|
+
or bool(self.target & frame.globals)
|
|
1746
|
+
or any("None" in g or g.startswith("not ") for g in frame.guards)
|
|
1747
|
+
)
|
|
1748
|
+
|
|
1749
|
+
def _view(self, name: str, node: ast.Call, frame: _Frame) -> Call:
|
|
1750
|
+
"""
|
|
1751
|
+
A call as a library plugin is shown it.
|
|
1752
|
+
"""
|
|
1753
|
+
known = self.values[1] if self.values[0] is node else {}
|
|
1754
|
+
return Call(
|
|
1755
|
+
name,
|
|
1756
|
+
node,
|
|
1757
|
+
self._loop_of(frame),
|
|
1758
|
+
self.loading > 0,
|
|
1759
|
+
self._once(frame),
|
|
1760
|
+
lambda arg: self._argument(arg, frame, known),
|
|
1761
|
+
)
|
|
1762
|
+
|
|
1763
|
+
def _argument(self, node: Any, frame: _Frame, known: dict[int, _Sym | None]) -> _Argument:
|
|
1764
|
+
"""
|
|
1765
|
+
What's known about one argument: the handle it holds, the text it's built from, and what from the input goes
|
|
1766
|
+
into it. known is what the call's arguments held when it was read.
|
|
1767
|
+
"""
|
|
1768
|
+
held = known.get(id(node)) or self._peek(node, frame)
|
|
1769
|
+
handle = held.handle if held is not None and held.kind == "lib" else None
|
|
1770
|
+
if held is not None and held.kind == "text":
|
|
1771
|
+
built = not isinstance(_const(_parse_text(held.sub)), str)
|
|
1772
|
+
return _Argument(handle, held.sub, held.path, built)
|
|
1773
|
+
built = isinstance(node, (ast.JoinedStr, ast.BinOp)) or (
|
|
1774
|
+
isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "format"
|
|
1775
|
+
)
|
|
1776
|
+
return _Argument(handle, "", self._inputs(node, frame), built)
|
|
1777
|
+
|
|
1778
|
+
def _library(self, library: Library, found: Handle | Use | None, call: Call, frame: _Frame) -> _Sym | None:
|
|
1779
|
+
"""
|
|
1780
|
+
What a library plugin said a call gives or does: a handle kept as what the name holds, or a call put in the
|
|
1781
|
+
map, its step and its box.
|
|
1782
|
+
"""
|
|
1783
|
+
node = call.node
|
|
1784
|
+
if isinstance(found, Handle):
|
|
1785
|
+
if found.opens:
|
|
1786
|
+
service = found.service or library.title
|
|
1787
|
+
self._spot(
|
|
1788
|
+
"connect",
|
|
1789
|
+
frame.file,
|
|
1790
|
+
node.lineno,
|
|
1791
|
+
frame.function,
|
|
1792
|
+
_unparse(node, 80),
|
|
1793
|
+
service,
|
|
1794
|
+
ok=call.once,
|
|
1795
|
+
loop=call.loop,
|
|
1796
|
+
waits=found.waits,
|
|
1797
|
+
fix=found.fix,
|
|
1798
|
+
)
|
|
1799
|
+
self.cur.note("client", f"makes {library.describe(found)}")
|
|
1800
|
+
if found.waits:
|
|
1801
|
+
self.cur.note("warn", f"no timeout: a call waits {found.waits}")
|
|
1802
|
+
return _Sym("lib", library.name, handle=found)
|
|
1803
|
+
if not isinstance(found, Use):
|
|
1804
|
+
return None
|
|
1805
|
+
service = found.service or library.title
|
|
1806
|
+
record = LibraryCall(
|
|
1807
|
+
library.name,
|
|
1808
|
+
service,
|
|
1809
|
+
found.operation,
|
|
1810
|
+
found.access,
|
|
1811
|
+
frame.file,
|
|
1812
|
+
node.lineno,
|
|
1813
|
+
frame.function,
|
|
1814
|
+
_unparse(node, 120),
|
|
1815
|
+
found.target,
|
|
1816
|
+
found.icon or library.icon,
|
|
1817
|
+
call.loop,
|
|
1818
|
+
self.loading > 0,
|
|
1819
|
+
found.flag,
|
|
1820
|
+
found.why,
|
|
1821
|
+
found.batch,
|
|
1822
|
+
)
|
|
1823
|
+
key = ("library", service, found.operation, frame.file, node.lineno)
|
|
1824
|
+
if key not in self.seen:
|
|
1825
|
+
self.seen.add(key)
|
|
1826
|
+
self.cmap.library_calls.append(record)
|
|
1827
|
+
if library.title not in self.cmap.plugins:
|
|
1828
|
+
self.cmap.plugins.append(library.title)
|
|
1829
|
+
self.cur.note(f"lib-{found.access}", f"{service}: {record.row}")
|
|
1830
|
+
if found.flag:
|
|
1831
|
+
self.cur.note("warn", found.flag)
|
|
1832
|
+
return _Sym("lib", library.name, handle=found.gives) if found.gives is not None else None
|
|
1833
|
+
|
|
1834
|
+
def _text(self, node: Any, frame: _Frame) -> _Sym | None:
|
|
1835
|
+
"""
|
|
1836
|
+
Text the code puts together (an f-string, + or % on text, .format()) with something from the input in it:
|
|
1837
|
+
kept with what went in, so a SQL statement or a command built from it says so wherever it's used.
|
|
1838
|
+
"""
|
|
1839
|
+
if isinstance(node, ast.BinOp) and not (
|
|
1840
|
+
isinstance(node.op, (ast.Add, ast.Mod))
|
|
1841
|
+
and (
|
|
1842
|
+
_is_text(node.left)
|
|
1843
|
+
or _is_text(node.right)
|
|
1844
|
+
or self._held_text(node.left, frame)
|
|
1845
|
+
or self._held_text(node.right, frame)
|
|
1846
|
+
)
|
|
1847
|
+
):
|
|
1848
|
+
return None
|
|
1849
|
+
inputs = self._inputs(node, frame)
|
|
1850
|
+
return _Sym("text", sub=_unparse(node, 400), path=inputs) if inputs else None
|
|
1851
|
+
|
|
1852
|
+
def _held_text(self, node: Any, frame: _Frame) -> bool:
|
|
1853
|
+
held = self._peek(node, frame)
|
|
1854
|
+
return held is not None and held.kind == "text"
|
|
1855
|
+
|
|
1856
|
+
def _inputs(self, node: Any, frame: _Frame) -> tuple[str, ...]:
|
|
1857
|
+
"""
|
|
1858
|
+
What from the entry point's input goes into an expression, as said beside a step: ('event body.id',). Read
|
|
1859
|
+
without running anything again.
|
|
1860
|
+
"""
|
|
1861
|
+
found: list[str] = []
|
|
1862
|
+
|
|
1863
|
+
def visit(n: Any) -> None:
|
|
1864
|
+
held = self._peek(n, frame)
|
|
1865
|
+
if held is not None and held.kind == "event":
|
|
1866
|
+
found.append(self._describe(held))
|
|
1867
|
+
return
|
|
1868
|
+
if held is not None and held.kind == "text":
|
|
1869
|
+
found.extend(held.path)
|
|
1870
|
+
return
|
|
1871
|
+
if isinstance(n, ast.Name):
|
|
1872
|
+
return
|
|
1873
|
+
for child in ast.iter_child_nodes(n):
|
|
1874
|
+
if isinstance(child, ast.expr):
|
|
1875
|
+
visit(child)
|
|
1876
|
+
|
|
1877
|
+
if node is not None:
|
|
1878
|
+
visit(node)
|
|
1879
|
+
return tuple(dict.fromkeys(found))
|
|
1880
|
+
|
|
1881
|
+
def _peek(self, node: Any, frame: _Frame) -> _Sym | None:
|
|
1882
|
+
"""
|
|
1883
|
+
What an expression holds, for the few kinds that can be told without reading a call again: a name, part of
|
|
1884
|
+
the input (event['body'], event.get('id'), json.loads(...) of it), or an attribute of one of its objects.
|
|
1885
|
+
"""
|
|
1886
|
+
if isinstance(node, ast.Name):
|
|
1887
|
+
return self._lookup(node.id, frame)
|
|
1888
|
+
if isinstance(node, ast.Subscript):
|
|
1889
|
+
base = self._peek(node.value, frame)
|
|
1890
|
+
if base is not None and base.kind == "event":
|
|
1891
|
+
key = _const(node.slice)
|
|
1892
|
+
return _Sym("event", path=base.path + ((key,) if isinstance(key, str) else ("[]",)))
|
|
1893
|
+
return None
|
|
1894
|
+
if isinstance(node, ast.Attribute):
|
|
1895
|
+
base = self._peek(node.value, frame)
|
|
1896
|
+
if base is None:
|
|
1897
|
+
return None
|
|
1898
|
+
if base.kind == "event" and node.attr not in _DICT_METHODS:
|
|
1899
|
+
attr = node.attr if base.path else self.platform.input_aliases.get(node.attr, node.attr)
|
|
1900
|
+
return _Sym("event", path=base.path + (attr,))
|
|
1901
|
+
if base.kind == "obj":
|
|
1902
|
+
found = self._member(base.name, node.attr)
|
|
1903
|
+
return found[1] if found and found[0] == "attr" else None
|
|
1904
|
+
return None
|
|
1905
|
+
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
|
|
1906
|
+
base = self._peek(node.func.value, frame)
|
|
1907
|
+
key = _const(node.args[0]) if node.args else None
|
|
1908
|
+
if base is not None and base.kind == "event" and node.func.attr in ("get", "pop") and isinstance(key, str):
|
|
1909
|
+
return _Sym("event", path=base.path + (key,))
|
|
1910
|
+
if base is not None and base.kind == "event" and node.func.attr not in _DICT_METHODS:
|
|
1911
|
+
return self._peek(node.func, frame) # request.get_json()
|
|
1912
|
+
dotted = _unparse(node.func, 60)
|
|
1913
|
+
if (dotted in _JSON_LOADS or dotted in _B64_DECODES) and node.args:
|
|
1914
|
+
inner = self._peek(node.args[0], frame)
|
|
1915
|
+
if inner is not None and inner.kind == "event":
|
|
1916
|
+
return _Sym("event", path=inner.path + ("{json}" if dotted in _JSON_LOADS else "{base64}",))
|
|
1917
|
+
return None
|
|
1918
|
+
|
|
1919
|
+
def _secrets(self, mod: _Module) -> None:
|
|
1920
|
+
"""
|
|
1921
|
+
Secret-looking values written in the code: a password, token or key given as a literal. Only their names
|
|
1922
|
+
are kept, never the values (an AWS access key is found in every file, by read()).
|
|
1923
|
+
"""
|
|
1924
|
+
for name, value in _secret_spots(mod.tree):
|
|
1925
|
+
if name != "an AWS access key":
|
|
1926
|
+
self._spot("secret", mod.file or "", value.lineno, mod.name, "", name, ok=False)
|