codeanalyzer-python 1.2.0__py3-none-any.whl → 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeanalyzer/__main__.py +29 -0
- codeanalyzer/artifacts/__init__.py +20 -0
- codeanalyzer/artifacts/config_keys.py +588 -0
- codeanalyzer/artifacts/config_use.py +597 -0
- codeanalyzer/artifacts/config_use_rules.yml +58 -0
- codeanalyzer/artifacts/dependencies.py +237 -0
- codeanalyzer/artifacts/discovery.py +167 -0
- codeanalyzer/artifacts/parsers.py +248 -0
- codeanalyzer/core.py +91 -0
- codeanalyzer/dataflow/builder.py +15 -1
- codeanalyzer/neo4j/project.py +192 -0
- codeanalyzer/neo4j/schema.py +57 -0
- codeanalyzer/options/options.py +5 -0
- codeanalyzer/schema/ids.py +21 -0
- codeanalyzer/schema/py_schema.py +118 -0
- codeanalyzer/syntactic_analysis/symbol_table_builder.py +11 -0
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/METADATA +109 -5
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/RECORD +22 -15
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/WHEEL +0 -0
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/entry_points.txt +0 -0
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/licenses/LICENSE +0 -0
- {codeanalyzer_python-1.2.0.dist-info → codeanalyzer_python-1.3.0.dist-info}/licenses/NOTICE +0 -0
codeanalyzer/__main__.py
CHANGED
|
@@ -193,6 +193,14 @@ def main(
|
|
|
193
193
|
"imports against the ambient Python environment instead.",
|
|
194
194
|
),
|
|
195
195
|
] = False,
|
|
196
|
+
resolve_installed: Annotated[
|
|
197
|
+
bool,
|
|
198
|
+
typer.Option(
|
|
199
|
+
"--resolve-installed",
|
|
200
|
+
help="Additionally bind imports via the project venv's installed metadata "
|
|
201
|
+
"(*.dist-info); output becomes machine-dependent (prov: installed-metadata).",
|
|
202
|
+
),
|
|
203
|
+
] = False,
|
|
196
204
|
file_name: Annotated[
|
|
197
205
|
Optional[Path],
|
|
198
206
|
typer.Option(
|
|
@@ -226,6 +234,24 @@ def main(
|
|
|
226
234
|
"the shipped rules. A malformed file is an error.",
|
|
227
235
|
),
|
|
228
236
|
] = None,
|
|
237
|
+
artifact_text: Annotated[
|
|
238
|
+
bool,
|
|
239
|
+
typer.Option(
|
|
240
|
+
"--artifact-text/--no-artifact-text",
|
|
241
|
+
help="Capture verbatim `source` text on discovered artifacts. "
|
|
242
|
+
"--no-artifact-text empties `source` everywhere (inventory unchanged).",
|
|
243
|
+
),
|
|
244
|
+
] = True,
|
|
245
|
+
artifact_text_max_bytes: Annotated[
|
|
246
|
+
int,
|
|
247
|
+
typer.Option(
|
|
248
|
+
"--artifact-text-max-bytes",
|
|
249
|
+
help="Per-file byte cap on captured artifact `source`; a decodable "
|
|
250
|
+
"file over the cap is truncated (text_truncated=True). "
|
|
251
|
+
"sha256/size_bytes always reflect the full file.",
|
|
252
|
+
min=1,
|
|
253
|
+
),
|
|
254
|
+
] = 262144,
|
|
229
255
|
):
|
|
230
256
|
# Determinism: pin the interpreter hash seed before any analysis (no-op
|
|
231
257
|
# when PYTHONHASHSEED is already set; --version exits before this).
|
|
@@ -303,11 +329,14 @@ def main(
|
|
|
303
329
|
rebuild_analysis=rebuild_analysis,
|
|
304
330
|
skip_tests=skip_tests,
|
|
305
331
|
no_venv=no_venv,
|
|
332
|
+
resolve_installed=resolve_installed,
|
|
306
333
|
file_name=file_name,
|
|
307
334
|
cache_dir=cache_dir,
|
|
308
335
|
clear_cache=clear_cache,
|
|
309
336
|
verbosity=verbosity,
|
|
310
337
|
entrypoint_rules=tuple(entrypoint_rules or ()),
|
|
338
|
+
artifact_text=artifact_text,
|
|
339
|
+
artifact_text_max_bytes=artifact_text_max_bytes,
|
|
311
340
|
)
|
|
312
341
|
|
|
313
342
|
_set_log_level(options.verbosity)
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Non-code artifact capture and dependency extraction (spec 2026-08-27).
|
|
2
|
+
|
|
3
|
+
Capture never drops a file (every non-`.py` file becomes a
|
|
4
|
+
:class:`~codeanalyzer.schema.py_schema.PyArtifact`, rule-matched or not,
|
|
5
|
+
text or binary -- issue #157 follow-up); extraction is narrow (only
|
|
6
|
+
dependency manifests are parsed for meaning in this unit)."""
|
|
7
|
+
|
|
8
|
+
from codeanalyzer.artifacts.config_keys import extract_config_keys, is_config_eligible
|
|
9
|
+
from codeanalyzer.artifacts.config_use import (
|
|
10
|
+
dataflow_intra_tier, dataflow_interproc_tier, detect_config_reads, resolve_uses,
|
|
11
|
+
)
|
|
12
|
+
from codeanalyzer.artifacts.dependencies import build_dependency_view
|
|
13
|
+
from codeanalyzer.artifacts.discovery import discover_artifacts
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"discover_artifacts", "build_dependency_view",
|
|
17
|
+
"extract_config_keys", "is_config_eligible",
|
|
18
|
+
"detect_config_reads", "resolve_uses",
|
|
19
|
+
"dataflow_intra_tier", "dataflow_interproc_tier",
|
|
20
|
+
]
|
|
@@ -0,0 +1,588 @@
|
|
|
1
|
+
"""Config-key flatteners (#152, #165). Pure text-in/records-out, mirroring
|
|
2
|
+
`artifacts/parsers.py`'s idiom: one dispatcher over per-format internals,
|
|
3
|
+
never raising.
|
|
4
|
+
|
|
5
|
+
Namespace dispatch: an env-family basename (`.env`, `.env.*`, `.flaskenv`)
|
|
6
|
+
always wins, regardless of the artifact's declared `format`; otherwise the
|
|
7
|
+
`format` field selects yaml/json/toml/ini/properties/dockerfile. Any other
|
|
8
|
+
format extracts nothing (not a failure -- there's just nothing to flatten).
|
|
9
|
+
|
|
10
|
+
Deployment-env namespaces (#165): a `dockerfile`-format artifact mints TWO
|
|
11
|
+
namespaces from one file -- `ENV K=v` directives mint namespace `env` (the
|
|
12
|
+
whole point is that `os.environ`/`os.getenv` reads, whose detector rule
|
|
13
|
+
prefers namespace `env`, bind to them), `ARG K[=default]` directives mint
|
|
14
|
+
namespace `dockerfile` (build-time only, deliberately not bindable by the
|
|
15
|
+
env detectors). Both share the bare var name as `key`, so an `ARG X` later
|
|
16
|
+
promoted via `ENV X=$X` -- a common idiom -- would collide on `id` (`key`
|
|
17
|
+
alone determines it) if both used the plain id shape; the `dockerfile`
|
|
18
|
+
mint's id is disambiguated with an internal `arg.` prefix (the `key` FIELD
|
|
19
|
+
stays the bare name either way, since nothing about resolution or the
|
|
20
|
+
issue's contract cares how the id looks).
|
|
21
|
+
|
|
22
|
+
A `yaml`-format artifact ALSO gets a supplementary recognition pass after
|
|
23
|
+
the normal dotted-path flattening: well-known compose (`services.<name>.
|
|
24
|
+
environment` map/list) and k8s (`...env.<idx>.name`/`.value`, matched as a
|
|
25
|
+
dotted-path shape at any nesting depth, not schema-anchored) shapes mint
|
|
26
|
+
ADDITIONAL namespace-`env` keys keyed on the bare var name, alongside the
|
|
27
|
+
normal namespace-`yaml` dotted-path ones -- dual-minting is intentional
|
|
28
|
+
(so `os.environ`/`os.getenv` reads bind to compose/k8s-declared vars too),
|
|
29
|
+
never deduped away. The `key` FIELD is always the bare var name (matching
|
|
30
|
+
`env` namespace's exact-match resolution semantics), but the `id` cannot
|
|
31
|
+
reuse that bare name unqualified: a TOP-LEVEL yaml key sharing the same
|
|
32
|
+
name as a recognized env var (e.g. a document with both a bare
|
|
33
|
+
`COMPOSE_ONLY_KEY:` entry and a `services.web.environment.COMPOSE_ONLY_KEY`
|
|
34
|
+
one) would otherwise collide with the plain yaml mint's own bare-key id --
|
|
35
|
+
the yaml mint's key is USUALLY a longer dotted path that can't collide, but
|
|
36
|
+
not always (a top-level leaf's dotted path IS just its bare name). Same fix
|
|
37
|
+
as the dockerfile ARG case: the env-dual-mint's id is disambiguated with an
|
|
38
|
+
internal `env.` prefix (`_build_keys`'s `id_key`), the `key` field itself
|
|
39
|
+
unaffected. This pass is shape-based, not filename/role gated -- any yaml
|
|
40
|
+
artifact whose content happens to match mints the extra keys, matching this
|
|
41
|
+
module's general overlay posture (permissive, never a schema validator).
|
|
42
|
+
|
|
43
|
+
Span precision differs by shape: env/properties/ini/dockerfile are
|
|
44
|
+
line-oriented, so the parse itself knows the exact defining line. yaml/
|
|
45
|
+
json/toml are tree-shaped -- span recovery falls back to a best-effort
|
|
46
|
+
search for the final dotted-key segment on its own line (see
|
|
47
|
+
`_find_key_span`), which can bind the wrong line when the same leaf name
|
|
48
|
+
recurs at another nesting level, or find nothing at all (`span=None`) for
|
|
49
|
+
a minified/single-line file. The compose/k8s env-recognition mint reuses
|
|
50
|
+
this same best-effort search keyed on the bare var name: it finds the
|
|
51
|
+
defining line for compose's map/list forms (`KEY:`/`KEY=` is the var's own
|
|
52
|
+
line) but not k8s's name/value list shape (the var name is a VALUE on a
|
|
53
|
+
`name:` line, not a key label there) -- k8s env-mint spans are `None`,
|
|
54
|
+
an accepted extension of the existing best-effort gap.
|
|
55
|
+
"""
|
|
56
|
+
from __future__ import annotations
|
|
57
|
+
|
|
58
|
+
import configparser
|
|
59
|
+
import json
|
|
60
|
+
import re
|
|
61
|
+
import sys
|
|
62
|
+
from typing import Callable, Dict, List, Optional, Tuple
|
|
63
|
+
|
|
64
|
+
if sys.version_info >= (3, 11):
|
|
65
|
+
import tomllib
|
|
66
|
+
else: # pragma: no cover - exercised on the 3.10 CI leg
|
|
67
|
+
import tomli as tomllib
|
|
68
|
+
|
|
69
|
+
import yaml
|
|
70
|
+
|
|
71
|
+
from codeanalyzer.schema.ids import config_key_id
|
|
72
|
+
from codeanalyzer.schema.py_schema import PyArtifact, PyConfigKey, Span, byte_offsets
|
|
73
|
+
|
|
74
|
+
# A parsed leaf before it becomes a PyConfigKey: (dotted_key, raw_value, span).
|
|
75
|
+
_Entry = Tuple[str, object, Optional[Span]]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# --- dotted-path flattening (yaml/json/toml share this over their parsed
|
|
79
|
+
# dict/list trees) --------------------------------------------------------
|
|
80
|
+
|
|
81
|
+
def _flatten(obj, prefix: str = ""):
|
|
82
|
+
"""Yield (dotted_key, leaf_value) pairs; numeric segments for arrays
|
|
83
|
+
(e.g. "services.web.ports.0")."""
|
|
84
|
+
if isinstance(obj, dict):
|
|
85
|
+
for k, v in obj.items():
|
|
86
|
+
yield from _flatten(v, f"{prefix}.{k}" if prefix else str(k))
|
|
87
|
+
elif isinstance(obj, list):
|
|
88
|
+
for i, v in enumerate(obj):
|
|
89
|
+
yield from _flatten(v, f"{prefix}.{i}" if prefix else str(i))
|
|
90
|
+
else:
|
|
91
|
+
yield prefix, obj
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _line_span(text: str, lines: List[str], lineno: int) -> Span:
|
|
95
|
+
"""Exact span covering `lineno`'s full text (1-based)."""
|
|
96
|
+
line = lines[lineno - 1]
|
|
97
|
+
lo, hi = byte_offsets(text, lineno, 0, lineno, len(line))
|
|
98
|
+
return Span(start=(lineno, 0), end=(lineno, len(line)), bytes=(lo, hi))
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
_BEST_EFFORT_KEY_TAIL = r'["\']?\s*[:=]'
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _find_key_span(text: str, lines: List[str], last_segment: str) -> Optional[Span]:
|
|
105
|
+
"""Best-effort: the FIRST line (in file order) whose stripped-of-leading
|
|
106
|
+
indentation/list-dash content starts with `last_segment` (optionally
|
|
107
|
+
quoted) followed by `:` or `=` -- covers yaml (`key:`), json (`"key":`),
|
|
108
|
+
and toml (`key =`) without per-format branching. Anchored at column 0
|
|
109
|
+
(not a substring search) so it can't latch onto a leaf name that merely
|
|
110
|
+
appears inside a longer token; the cost is that it also can't see keys
|
|
111
|
+
packed onto a single minified line, which is an accepted v1 gap given
|
|
112
|
+
`span` is `Optional`."""
|
|
113
|
+
pattern = re.compile(r'^[\s\-]*["\']?' + re.escape(last_segment) + _BEST_EFFORT_KEY_TAIL)
|
|
114
|
+
for i, line in enumerate(lines, start=1):
|
|
115
|
+
if pattern.match(line):
|
|
116
|
+
return _line_span(text, lines, i)
|
|
117
|
+
return None
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _flatten_structured(data, text: str, lines: List[str]) -> List[_Entry]:
|
|
121
|
+
return [(k, v, _find_key_span(text, lines, k.rsplit(".", 1)[-1])) for k, v in _flatten(data)]
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _parse_yaml(text: str, lines: List[str]) -> List[_Entry]:
|
|
125
|
+
return _flatten_structured(yaml.safe_load(text) or {}, text, lines)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _parse_json(text: str, lines: List[str]) -> List[_Entry]:
|
|
129
|
+
return _flatten_structured(json.loads(text), text, lines)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _parse_toml(text: str, lines: List[str]) -> List[_Entry]:
|
|
133
|
+
return _flatten_structured(tomllib.loads(text), text, lines)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
# --- compose/k8s env recognition (#165): supplemental namespace="env" mint
|
|
137
|
+
# over the SAME flattened (dotted_key, value) pairs the "yaml" namespace
|
|
138
|
+
# uses -- see the module docstring for why dual-minting is safe and
|
|
139
|
+
# intentional. Shape-matched on the dotted path string, not the yaml tree,
|
|
140
|
+
# so both recognizers stay simple regex-over-strings, symmetric with the
|
|
141
|
+
# rest of this module's line/path-oriented parsing. -------------------------
|
|
142
|
+
|
|
143
|
+
_K8S_ENV_NAME = re.compile(r'(?:^|\.)env\.\d+\.name$')
|
|
144
|
+
_COMPOSE_ENV = re.compile(r'^services\.[^.]+\.environment\.(.+)$')
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _recognize_env_shapes(flat: List[Tuple[str, object]]) -> List[Tuple[str, object]]:
|
|
148
|
+
"""`(key, value)` pairs for every compose/k8s env shape found in `flat`
|
|
149
|
+
(the same `_flatten(data)` pairs the "yaml" namespace flattens) --
|
|
150
|
+
NOT dotted paths, the bare var name, matching `env` namespace semantics
|
|
151
|
+
(`config_use.py` resolves that namespace by exact `key ==` match)."""
|
|
152
|
+
by_path = dict(flat)
|
|
153
|
+
out: List[Tuple[str, object]] = []
|
|
154
|
+
for dotted_key, value in flat:
|
|
155
|
+
if _K8S_ENV_NAME.search(dotted_key):
|
|
156
|
+
sibling = dotted_key[: -len("name")] + "value" # ...env.<idx>.value
|
|
157
|
+
out.append((_stringify(value), by_path.get(sibling)))
|
|
158
|
+
continue
|
|
159
|
+
m = _COMPOSE_ENV.match(dotted_key)
|
|
160
|
+
if not m:
|
|
161
|
+
continue
|
|
162
|
+
tail = m.group(1)
|
|
163
|
+
if _ENV_KEY_NAME.match(tail): # map form: tail IS the var name
|
|
164
|
+
out.append((tail, value))
|
|
165
|
+
elif tail.isdigit(): # list form: leaf is "KEY=val" or bare "KEY"
|
|
166
|
+
key, sep, val = _stringify(value).partition("=")
|
|
167
|
+
if _ENV_KEY_NAME.match(key):
|
|
168
|
+
out.append((key, val if sep else None))
|
|
169
|
+
return out
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
# --- env: KEY=value, `#` comments, `export ` prefix, quote stripping -------
|
|
173
|
+
|
|
174
|
+
_ENV_LINE = re.compile(r'^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$')
|
|
175
|
+
_ENV_KEY_NAME = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$') # shared: dockerfile + compose/k8s recognition
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _env_value(raw: str) -> str:
|
|
179
|
+
"""The text after `KEY=` on one line -> the value. A quoted value ends
|
|
180
|
+
at its MATCHING closing quote -- anything after that (including a `#`)
|
|
181
|
+
is trailing comment and is discarded, so a `#` INSIDE the quotes (e.g. a
|
|
182
|
+
URL fragment) is never reached by comment-stripping. An unquoted value
|
|
183
|
+
ends at the first unescaped `" #"` (whitespace then `#`); a bare `#`
|
|
184
|
+
stuck directly to a token (no preceding whitespace) is not a comment
|
|
185
|
+
marker and stays in the value."""
|
|
186
|
+
raw = raw.strip()
|
|
187
|
+
if raw and raw[0] in "'\"":
|
|
188
|
+
quote = raw[0]
|
|
189
|
+
end = raw.find(quote, 1)
|
|
190
|
+
return raw[1:end] if end != -1 else raw[1:]
|
|
191
|
+
return re.split(r"\s#", raw, maxsplit=1)[0].strip()
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _is_env_family(basename: str) -> bool:
|
|
195
|
+
return basename == ".env" or basename.startswith(".env.") or basename == ".flaskenv"
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _parse_env(text: str, lines: List[str]) -> List[_Entry]:
|
|
199
|
+
out: List[_Entry] = []
|
|
200
|
+
for lineno, raw in enumerate(lines, start=1):
|
|
201
|
+
stripped = raw.strip()
|
|
202
|
+
if not stripped or stripped.startswith("#"):
|
|
203
|
+
continue
|
|
204
|
+
m = _ENV_LINE.match(stripped)
|
|
205
|
+
if not m:
|
|
206
|
+
continue
|
|
207
|
+
key, value = m.group(1), _env_value(m.group(2))
|
|
208
|
+
out.append((key, value, _line_span(text, lines, lineno)))
|
|
209
|
+
return out
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
# --- dockerfile: `ENV`/`ARG` directives (#165). Line-based, case-insensitive
|
|
213
|
+
# instruction keywords (Dockerfile convention is uppercase, the spec itself
|
|
214
|
+
# is not case-sensitive); no BuildKit heredoc awareness in v1 -- a heredoc
|
|
215
|
+
# body line is just another line that doesn't match `_DOCKER_ENV`/`_DOCKER_ARG`
|
|
216
|
+
# and is silently skipped, same as any other unparseable line (overlay
|
|
217
|
+
# posture). Multi-stage `FROM ... AS x` scoping is not modeled -- every
|
|
218
|
+
# ENV/ARG in the file is scanned regardless of which stage it's in. --------
|
|
219
|
+
|
|
220
|
+
_DOCKER_ENV = re.compile(r'^ENV\s+(.*)$', re.IGNORECASE)
|
|
221
|
+
_DOCKER_ARG = re.compile(r'^ARG\s+(.*)$', re.IGNORECASE)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _join_continuations(lines: List[str], start_i: int) -> Tuple[str, int]:
|
|
225
|
+
"""From `lines[start_i]`, join any backslash-continued following lines
|
|
226
|
+
into one logical instruction line -- same trailing-backslash-drop +
|
|
227
|
+
leading/trailing-whitespace-strip join `_parse_properties` already uses
|
|
228
|
+
for its own continuations. Returns `(joined_text, index of the LAST
|
|
229
|
+
line consumed)`."""
|
|
230
|
+
i, n = start_i, len(lines)
|
|
231
|
+
parts = [lines[i].strip()]
|
|
232
|
+
while parts[-1].endswith("\\") and i + 1 < n:
|
|
233
|
+
parts[-1] = parts[-1][:-1] # drop just the continuation backslash
|
|
234
|
+
i += 1
|
|
235
|
+
parts.append(lines[i].strip())
|
|
236
|
+
return "".join(parts), i
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _split_ws_respecting_quotes(s: str) -> List[str]:
|
|
240
|
+
r"""Whitespace-split `s`, except inside a matching `'`/`"` span (a quoted
|
|
241
|
+
value may contain spaces) or right after an unquoted `\` -- a backslash
|
|
242
|
+
escapes the next character (`\ ` keeps a literal space in the token
|
|
243
|
+
instead of splitting there, `\\` collapses to one literal backslash),
|
|
244
|
+
mirroring Docker's own shell-style ENV splitting (moby's `Rex\ The\
|
|
245
|
+
Dog` example). A dangling trailing `\` with nothing to escape is kept
|
|
246
|
+
literally rather than raising -- a real trailing continuation backslash
|
|
247
|
+
is already stripped upstream by `_join_continuations`, so this is only
|
|
248
|
+
a defensive fallback. Quote characters stay IN the returned tokens,
|
|
249
|
+
stripped afterward by `_env_value` so there is one quote-stripping
|
|
250
|
+
implementation, not two."""
|
|
251
|
+
tokens: List[str] = []
|
|
252
|
+
buf: List[str] = []
|
|
253
|
+
quote: Optional[str] = None
|
|
254
|
+
i, n = 0, len(s)
|
|
255
|
+
while i < n:
|
|
256
|
+
ch = s[i]
|
|
257
|
+
if quote:
|
|
258
|
+
buf.append(ch)
|
|
259
|
+
if ch == quote:
|
|
260
|
+
quote = None
|
|
261
|
+
elif ch == "\\":
|
|
262
|
+
i += 1
|
|
263
|
+
buf.append(s[i] if i < n else ch)
|
|
264
|
+
elif ch in "'\"":
|
|
265
|
+
quote = ch
|
|
266
|
+
buf.append(ch)
|
|
267
|
+
elif ch.isspace():
|
|
268
|
+
if buf:
|
|
269
|
+
tokens.append("".join(buf))
|
|
270
|
+
buf = []
|
|
271
|
+
else:
|
|
272
|
+
buf.append(ch)
|
|
273
|
+
i += 1
|
|
274
|
+
if buf:
|
|
275
|
+
tokens.append("".join(buf))
|
|
276
|
+
return tokens
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _dockerfile_env_entries(text: str, lines: List[str]) -> List[_Entry]:
|
|
280
|
+
"""`ENV` directives -> `(KEY, value, span)`. Handles `ENV K=v`, multi-key
|
|
281
|
+
`ENV a=1 b=2`, and the legacy single-key `ENV K v` space form (Docker's
|
|
282
|
+
own disambiguation rule: the token right after `ENV` decides the form --
|
|
283
|
+
a `=` in it means one-or-more `key=value` pairs; no `=` means the
|
|
284
|
+
legacy form, where the key is the first word and the REST of the line
|
|
285
|
+
is the value). The legacy form's value is taken VERBATIM -- unlike the
|
|
286
|
+
`key=value` form, real Docker does no quote processing there at all
|
|
287
|
+
(moby's `parseNameVal`), so `ENV NAME "John Doe"` keeps its quotes; the
|
|
288
|
+
key/value separator is general whitespace (a tab is as legal as a
|
|
289
|
+
space), not a literal `" "`."""
|
|
290
|
+
out: List[_Entry] = []
|
|
291
|
+
i, n = 0, len(lines)
|
|
292
|
+
while i < n:
|
|
293
|
+
stripped = lines[i].strip()
|
|
294
|
+
if not stripped or stripped.startswith("#") or not _DOCKER_ENV.match(stripped):
|
|
295
|
+
i += 1
|
|
296
|
+
continue
|
|
297
|
+
start_lineno = i + 1
|
|
298
|
+
joined, i = _join_continuations(lines, i)
|
|
299
|
+
rest = _DOCKER_ENV.match(joined).group(1).strip()
|
|
300
|
+
span = _line_span(text, lines, start_lineno)
|
|
301
|
+
first_token = rest.split(None, 1)[0] if rest else ""
|
|
302
|
+
if "=" in first_token:
|
|
303
|
+
for tok in _split_ws_respecting_quotes(rest):
|
|
304
|
+
key, sep, raw_val = tok.partition("=")
|
|
305
|
+
if sep and _ENV_KEY_NAME.match(key):
|
|
306
|
+
out.append((key, _env_value(raw_val), span))
|
|
307
|
+
else:
|
|
308
|
+
parts = rest.split(None, 1)
|
|
309
|
+
if len(parts) == 2 and _ENV_KEY_NAME.match(parts[0]):
|
|
310
|
+
out.append((parts[0], parts[1], span))
|
|
311
|
+
i += 1
|
|
312
|
+
return out
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _dockerfile_arg_entries(text: str, lines: List[str]) -> List[_Entry]:
|
|
316
|
+
"""`ARG KEY[=default]` -> `(KEY, value, span)`; no `=default` means
|
|
317
|
+
`value=None` (#165's ARG semantics -- distinct from `_stringify`'s "" for
|
|
318
|
+
a modeled null elsewhere in this module: an ARG default that is simply
|
|
319
|
+
ABSENT is not the same fact as a key explicitly set to an empty value)."""
|
|
320
|
+
out: List[_Entry] = []
|
|
321
|
+
i, n = 0, len(lines)
|
|
322
|
+
while i < n:
|
|
323
|
+
stripped = lines[i].strip()
|
|
324
|
+
if not stripped or stripped.startswith("#") or not _DOCKER_ARG.match(stripped):
|
|
325
|
+
i += 1
|
|
326
|
+
continue
|
|
327
|
+
start_lineno = i + 1
|
|
328
|
+
joined, i = _join_continuations(lines, i)
|
|
329
|
+
rest = _DOCKER_ARG.match(joined).group(1).strip()
|
|
330
|
+
span = _line_span(text, lines, start_lineno)
|
|
331
|
+
key, sep, raw_default = rest.partition("=")
|
|
332
|
+
key = key.strip()
|
|
333
|
+
if _ENV_KEY_NAME.match(key):
|
|
334
|
+
out.append((key, _env_value(raw_default) if sep else None, span))
|
|
335
|
+
i += 1
|
|
336
|
+
return out
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
# --- properties: key=value / key: value, `\` continuations, `!`/`#` comments
|
|
340
|
+
|
|
341
|
+
_PROPS_KV = re.compile(r'^(?P<key>[^=:\s]+)\s*[:=]\s*(?P<value>.*)$')
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def _parse_properties(text: str, lines: List[str]) -> List[_Entry]:
|
|
345
|
+
out: List[_Entry] = []
|
|
346
|
+
i, n = 0, len(lines)
|
|
347
|
+
while i < n:
|
|
348
|
+
stripped = lines[i].strip()
|
|
349
|
+
if not stripped or stripped.startswith(("#", "!")):
|
|
350
|
+
i += 1
|
|
351
|
+
continue
|
|
352
|
+
start_lineno = i + 1
|
|
353
|
+
parts = [stripped]
|
|
354
|
+
while parts[-1].endswith("\\") and i + 1 < n:
|
|
355
|
+
parts[-1] = parts[-1][:-1] # drop just the continuation backslash
|
|
356
|
+
i += 1
|
|
357
|
+
parts.append(lines[i].strip()) # continuation: leading whitespace stripped
|
|
358
|
+
m = _PROPS_KV.match("".join(parts))
|
|
359
|
+
if m:
|
|
360
|
+
out.append((m.group("key").strip(), m.group("value").strip(),
|
|
361
|
+
_line_span(text, lines, start_lineno)))
|
|
362
|
+
i += 1
|
|
363
|
+
return out
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
# --- ini: configparser with raw=True (preserve `%(x)s`); exact line via a
|
|
367
|
+
# lightweight parallel section/key scan (configparser gives no line numbers,
|
|
368
|
+
# and `strict=True` already guarantees no duplicate (section, key) pairs) --
|
|
369
|
+
|
|
370
|
+
_INI_SECTION = re.compile(r'^\[(?P<name>[^]]+)\]\s*$')
|
|
371
|
+
_INI_KEY = re.compile(r'^(?P<key>[^\s#;=:][^=:]*?)\s*[:=]')
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _ini_line_map(lines: List[str]) -> Dict[Tuple[str, str], int]:
|
|
375
|
+
out: Dict[Tuple[str, str], int] = {}
|
|
376
|
+
section: Optional[str] = None
|
|
377
|
+
for lineno, line in enumerate(lines, start=1):
|
|
378
|
+
if not line.strip() or line.lstrip().startswith((";", "#")):
|
|
379
|
+
continue
|
|
380
|
+
m = _INI_SECTION.match(line)
|
|
381
|
+
if m:
|
|
382
|
+
section = m.group("name")
|
|
383
|
+
continue
|
|
384
|
+
m = _INI_KEY.match(line)
|
|
385
|
+
if m and section is not None:
|
|
386
|
+
out.setdefault((section, m.group("key")), lineno)
|
|
387
|
+
return out
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _parse_ini(text: str, lines: List[str]) -> List[_Entry]:
|
|
391
|
+
cp = configparser.ConfigParser()
|
|
392
|
+
cp.optionxform = str # preserve on-disk case (also needed for the line-map lookup)
|
|
393
|
+
cp.read_string(text)
|
|
394
|
+
line_map = _ini_line_map(lines)
|
|
395
|
+
out: List[_Entry] = []
|
|
396
|
+
# DEFAULT's own keys, unconditionally: `cp.sections()` never includes
|
|
397
|
+
# "DEFAULT" (configparser convention), so a file with only a [DEFAULT]
|
|
398
|
+
# section would otherwise yield zero keys. `cp.items(section, ...)`
|
|
399
|
+
# below ALSO re-inherits every DEFAULT key into each real section
|
|
400
|
+
# (configparser's fallback-lookup semantics) -- so a key defined only in
|
|
401
|
+
# DEFAULT deliberately appears twice: once as `DEFAULT.<key>` and again
|
|
402
|
+
# as `<section>.<key>` per inheriting section. Both are real, distinct
|
|
403
|
+
# facts (the key is DEFINED in DEFAULT; the section's own resolved
|
|
404
|
+
# value equals it), so both stay -- this is intended duplication, not a
|
|
405
|
+
# bug (see docs/design/specs/2026-08-28-config-key-family-design.md
|
|
406
|
+
# Caveats).
|
|
407
|
+
for key, value in cp.defaults().items():
|
|
408
|
+
lineno = line_map.get(("DEFAULT", key))
|
|
409
|
+
span = _line_span(text, lines, lineno) if lineno else None
|
|
410
|
+
out.append((f"DEFAULT.{key}", value, span))
|
|
411
|
+
for section in cp.sections():
|
|
412
|
+
for key, value in cp.items(section, raw=True):
|
|
413
|
+
lineno = line_map.get((section, key)) or line_map.get(("DEFAULT", key))
|
|
414
|
+
span = _line_span(text, lines, lineno) if lineno else None
|
|
415
|
+
out.append((f"{section}.{key}", value, span))
|
|
416
|
+
return out
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
_NAMESPACE_PARSERS: Dict[str, Callable[[str, List[str]], List[_Entry]]] = {
|
|
420
|
+
"yaml": _parse_yaml, "json": _parse_json, "toml": _parse_toml,
|
|
421
|
+
"ini": _parse_ini, "properties": _parse_properties,
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
# --- reference recognition ---------------------------------------------
|
|
426
|
+
|
|
427
|
+
_REF_TEMPLATE = re.compile(r'\$\{\{[^}]*\}\}')
|
|
428
|
+
_REF_BRACED = re.compile(r'\$\{[A-Za-z_][A-Za-z0-9_]*\}')
|
|
429
|
+
_REF_BARE = re.compile(r'\$[A-Za-z_][A-Za-z0-9_]*')
|
|
430
|
+
_REF_PERCENT = re.compile(r'%\([A-Za-z_][A-Za-z0-9_.]*\)s')
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _find_references(text: str) -> List[str]:
|
|
434
|
+
"""Raw reference tokens in `text`, order of appearance, deduplicated.
|
|
435
|
+
`${{ ... }}` is matched (and masked out of the working copy) FIRST so a
|
|
436
|
+
`${VAR}`/`$VAR` nested inside a template expression is not also counted
|
|
437
|
+
as a standalone reference; `%(name)s` never overlaps a `$`-sigil form so
|
|
438
|
+
it needs no masking."""
|
|
439
|
+
found: List[Tuple[int, str]] = []
|
|
440
|
+
working = text
|
|
441
|
+
for pattern in (_REF_TEMPLATE, _REF_BRACED, _REF_BARE):
|
|
442
|
+
for m in pattern.finditer(working):
|
|
443
|
+
found.append((m.start(), m.group(0)))
|
|
444
|
+
working = pattern.sub(lambda m: " " * len(m.group(0)), working)
|
|
445
|
+
for m in _REF_PERCENT.finditer(text):
|
|
446
|
+
found.append((m.start(), m.group(0)))
|
|
447
|
+
found.sort(key=lambda pair: pair[0])
|
|
448
|
+
seen = set()
|
|
449
|
+
out: List[str] = []
|
|
450
|
+
for _, token in found:
|
|
451
|
+
if token not in seen:
|
|
452
|
+
seen.add(token)
|
|
453
|
+
out.append(token)
|
|
454
|
+
return out
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def _stringify(value: object) -> str:
|
|
458
|
+
if value is None:
|
|
459
|
+
return ""
|
|
460
|
+
if isinstance(value, bool): # yaml/json/toml spell it lowercase on disk
|
|
461
|
+
return "true" if value else "false"
|
|
462
|
+
return str(value)
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def _build_keys(
|
|
466
|
+
artifact_id: str, namespace: str, entries: List[_Entry], capture_value: bool,
|
|
467
|
+
*, raw_value: bool = False, id_key: Optional[Callable[[str], str]] = None,
|
|
468
|
+
) -> List[PyConfigKey]:
|
|
469
|
+
"""Coalesce `entries` (last dotted-key occurrence in file order wins --
|
|
470
|
+
the existing L1 duplicate-key precedent, e.g. a redefined env var) into
|
|
471
|
+
`PyConfigKey` records for one namespace.
|
|
472
|
+
|
|
473
|
+
`raw_value=True` (dockerfile) passes the parsed value straight through
|
|
474
|
+
instead of `_stringify`-ing it, so an ARG's absent default surfaces as
|
|
475
|
+
`value=None` rather than `_stringify`'s "" for a modeled null -- dockerfile
|
|
476
|
+
values are already plain parsed text/`None`, never a yaml/json/toml
|
|
477
|
+
bool/None that needs that coercion.
|
|
478
|
+
|
|
479
|
+
`id_key` remaps `dotted_key` for ID CONSTRUCTION only -- the `.key` FIELD
|
|
480
|
+
always stays the bare `dotted_key`. Two call sites need it, both to keep
|
|
481
|
+
a bare-name mint from colliding with another mint that happens to use
|
|
482
|
+
the same bare name for its OWN id: a Dockerfile ARG's id (`ARG X` then
|
|
483
|
+
`ENV X=$X` is a common promotion idiom -- both would otherwise mint id
|
|
484
|
+
`.../@key/X`), and a yaml artifact's compose/k8s env-dual-mint id (a
|
|
485
|
+
top-level yaml key sharing a name with a recognized env var would
|
|
486
|
+
otherwise collide with the plain yaml mint's own bare-key id). Every
|
|
487
|
+
other namespace omits it, preserving today's id shape."""
|
|
488
|
+
coalesced: Dict[str, Tuple[object, Optional[Span]]] = {}
|
|
489
|
+
for dotted_key, value, span in entries:
|
|
490
|
+
coalesced[dotted_key] = (value, span)
|
|
491
|
+
keys = []
|
|
492
|
+
for dotted_key, (value, span) in coalesced.items():
|
|
493
|
+
text_value = value if raw_value else _stringify(value)
|
|
494
|
+
keys.append(PyConfigKey(
|
|
495
|
+
id=config_key_id(artifact_id, id_key(dotted_key) if id_key else dotted_key),
|
|
496
|
+
key=dotted_key, namespace=namespace,
|
|
497
|
+
value=text_value if capture_value else None,
|
|
498
|
+
span=span, references=_find_references(text_value or ""),
|
|
499
|
+
))
|
|
500
|
+
return keys
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
# --- public API -----------------------------------------------------------
|
|
504
|
+
|
|
505
|
+
def is_config_eligible(artifact: PyArtifact) -> bool:
|
|
506
|
+
"""Whether `artifact` is worth extracting config keys from: an env-family
|
|
507
|
+
basename (`.env`/`.env.*`/`.flaskenv`, regardless of declared format), a
|
|
508
|
+
`dockerfile`-format artifact (#165), or a namespace-bearing format
|
|
509
|
+
(yaml/json/toml/ini/properties). A binary artifact is never eligible --
|
|
510
|
+
there is no decodable text to flatten, and a rule-matched-but-undecodable
|
|
511
|
+
file downgrades to `format="binary"` regardless of its basename (see
|
|
512
|
+
discovery.py), so the binary check wins even over an env-family name.
|
|
513
|
+
|
|
514
|
+
Callers (core.py's wiring) use this to skip the on-disk read + parse
|
|
515
|
+
attempt entirely on artifacts that can never yield config keys, rather
|
|
516
|
+
than relying on `extract_config_keys`'s own not-applicable `([], True)`
|
|
517
|
+
return after already having paid for the read."""
|
|
518
|
+
if artifact.format == "binary":
|
|
519
|
+
return False
|
|
520
|
+
basename = artifact.path.rsplit("/", 1)[-1]
|
|
521
|
+
return (
|
|
522
|
+
_is_env_family(basename)
|
|
523
|
+
or artifact.format == "dockerfile"
|
|
524
|
+
or artifact.format in _NAMESPACE_PARSERS
|
|
525
|
+
)
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
def extract_config_keys(
|
|
529
|
+
artifact: PyArtifact, full_text: str, capture_value: bool,
|
|
530
|
+
) -> Tuple[List[PyConfigKey], bool]:
|
|
531
|
+
"""Flatten `artifact`'s config format into `PyConfigKey` records, reading
|
|
532
|
+
`full_text` (the real on-disk text -- never the possibly-truncated
|
|
533
|
+
`artifact.source`).
|
|
534
|
+
|
|
535
|
+
Returns `(keys, ok)`: the same two-tuple shape as
|
|
536
|
+
`artifacts.parsers.parse_manifest`'s `(records, partial)`, but the
|
|
537
|
+
OPPOSITE polarity -- here `ok` is `True` on success (including the
|
|
538
|
+
not-applicable case: a format with no flattener yields `([], True)`) and
|
|
539
|
+
`False` only when parsing raised. Never raises: every code path below is
|
|
540
|
+
covered by one `try`/`except`, so a malformed file degrades to `([],
|
|
541
|
+
False)` instead of an exception escaping to the caller. `keys` is always
|
|
542
|
+
sorted by `key` (L1 determinism) -- a dockerfile or dual-minted yaml
|
|
543
|
+
artifact concatenates its namespace groups before this one sort, and
|
|
544
|
+
Python's sort is stable, so a same-`key` tie across namespaces still
|
|
545
|
+
resolves deterministically (env before dockerfile; yaml before env).
|
|
546
|
+
|
|
547
|
+
`value` is populated only when `capture_value` is True; `key`,
|
|
548
|
+
`namespace`, `span`, and `references` are extracted unconditionally
|
|
549
|
+
either way (references are recognized in the raw leaf value regardless
|
|
550
|
+
of whether that value is exposed)."""
|
|
551
|
+
basename = artifact.path.rsplit("/", 1)[-1]
|
|
552
|
+
try:
|
|
553
|
+
lines = full_text.splitlines()
|
|
554
|
+
if _is_env_family(basename):
|
|
555
|
+
keys = _build_keys(artifact.id, "env", _parse_env(full_text, lines), capture_value)
|
|
556
|
+
elif artifact.format == "dockerfile":
|
|
557
|
+
keys = _build_keys(
|
|
558
|
+
artifact.id, "env", _dockerfile_env_entries(full_text, lines), capture_value,
|
|
559
|
+
raw_value=True,
|
|
560
|
+
) + _build_keys(
|
|
561
|
+
artifact.id, "dockerfile", _dockerfile_arg_entries(full_text, lines), capture_value,
|
|
562
|
+
raw_value=True, id_key=lambda k: f"arg.{k}",
|
|
563
|
+
)
|
|
564
|
+
elif artifact.format == "yaml":
|
|
565
|
+
# Two independent parses (like every other format here -- each
|
|
566
|
+
# piece parses what IT needs, no shared-state shortcut): `_parse_
|
|
567
|
+
# yaml` for the plain dotted-path entries, a second `safe_load`
|
|
568
|
+
# for the raw tree the env-shape recognizer walks.
|
|
569
|
+
flat = list(_flatten(yaml.safe_load(full_text) or {}))
|
|
570
|
+
env_entries = [
|
|
571
|
+
(k, v, _find_key_span(full_text, lines, k)) for k, v in _recognize_env_shapes(flat)
|
|
572
|
+
]
|
|
573
|
+
keys = (
|
|
574
|
+
_build_keys(artifact.id, "yaml", _parse_yaml(full_text, lines), capture_value)
|
|
575
|
+
+ _build_keys(
|
|
576
|
+
artifact.id, "env", env_entries, capture_value,
|
|
577
|
+
id_key=lambda k: f"env.{k}",
|
|
578
|
+
)
|
|
579
|
+
)
|
|
580
|
+
else:
|
|
581
|
+
parser = _NAMESPACE_PARSERS.get(artifact.format)
|
|
582
|
+
if parser is None:
|
|
583
|
+
return [], True
|
|
584
|
+
keys = _build_keys(artifact.id, artifact.format, parser(full_text, lines), capture_value)
|
|
585
|
+
keys.sort(key=lambda k: k.key)
|
|
586
|
+
return keys, True
|
|
587
|
+
except Exception:
|
|
588
|
+
return [], False
|