reachdiff 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reachdiff/__init__.py +0 -0
- reachdiff/__main__.py +3 -0
- reachdiff/apply.py +52 -0
- reachdiff/cli.py +88 -0
- reachdiff/collect/__init__.py +0 -0
- reachdiff/collect/live.py +463 -0
- reachdiff/collect/offline.py +32 -0
- reachdiff/config.py +135 -0
- reachdiff/diff.py +234 -0
- reachdiff/report.py +215 -0
- reachdiff/resolve.py +104 -0
- reachdiff/rules.py +197 -0
- reachdiff/scope.py +21 -0
- reachdiff/state.py +89 -0
- reachdiff/tfplan.py +352 -0
- reachdiff-0.9.0.dist-info/METADATA +372 -0
- reachdiff-0.9.0.dist-info/RECORD +21 -0
- reachdiff-0.9.0.dist-info/WHEEL +5 -0
- reachdiff-0.9.0.dist-info/entry_points.txt +2 -0
- reachdiff-0.9.0.dist-info/licenses/LICENSE +202 -0
- reachdiff-0.9.0.dist-info/top_level.txt +1 -0
reachdiff/rules.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"""Built-in and configured rules. Every finding names the row or gap that raised it."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections import defaultdict
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from fnmatch import fnmatchcase
|
|
8
|
+
|
|
9
|
+
from reachdiff.config import Config
|
|
10
|
+
from reachdiff.diff import AccessDiff, Change
|
|
11
|
+
from reachdiff.scope import changed_objects, has_access_changes
|
|
12
|
+
from reachdiff.state import BUILTIN_GROUPS, Gap, State, ancestors, kind_of
|
|
13
|
+
from reachdiff.tfplan import Plan
|
|
14
|
+
|
|
15
|
+
BROAD = frozenset({'ALL_PRIVILEGES', 'MANAGE'})
|
|
16
|
+
# A change whose values could not be read may still change access, so it can never PASS.
|
|
17
|
+
UNREADABLE_PLAN_VALUES = frozenset({'unknown_after_apply', 'sensitive_value'})
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True, order=True)
|
|
21
|
+
class Finding:
|
|
22
|
+
severity: str # 'BLOCK' sorts before 'WARN'
|
|
23
|
+
rule: str
|
|
24
|
+
subject: str
|
|
25
|
+
message: str
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def sensitive(state: State, securable: str, globs: tuple[str, ...]) -> bool:
|
|
29
|
+
"""A matching tag on the object, a parent, or a child present in the state."""
|
|
30
|
+
names = set(ancestors(securable)) | {n for n in state.securables if n.startswith(securable + '.')}
|
|
31
|
+
return any(
|
|
32
|
+
fnmatchcase(tag, glob)
|
|
33
|
+
for name in names
|
|
34
|
+
if name in state.securables
|
|
35
|
+
for tag in state.securables[name].tags
|
|
36
|
+
for glob in globs
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _column_tags_unread(state: State, securable: str, changed: set[str]) -> bool:
|
|
41
|
+
"""Column tags are read only for tables the plan changes; any other table here was not checked."""
|
|
42
|
+
names = {securable} | {n for n in state.securables if n.startswith(securable + '.')}
|
|
43
|
+
return any(kind_of(n) == 'table' and n not in changed for n in names)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _subject(row: Change) -> str:
|
|
47
|
+
return f'{row.principal} {row.level} {row.securable}'
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _exempt(row: Change, unless_via: frozenset[str]) -> bool:
|
|
51
|
+
return bool(unless_via) and all(({row.principal} | set(r.main.chain)) & unless_via for r in row.routes)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _glob_tokens(pattern: str) -> list[str]:
|
|
55
|
+
"""Split a glob as fnmatch reads it: `*`, `?`, each `[...]` class and each literal character.
|
|
56
|
+
|
|
57
|
+
An unclosed `[` is a literal; it is re-escaped as `[[]` so that a prefix of the tokens stays a valid glob.
|
|
58
|
+
"""
|
|
59
|
+
tokens, i, n = [], 0, len(pattern)
|
|
60
|
+
while i < n:
|
|
61
|
+
if pattern[i] == '[':
|
|
62
|
+
j = i + 1
|
|
63
|
+
if j < n and pattern[j] == '!':
|
|
64
|
+
j += 1
|
|
65
|
+
if j < n and pattern[j] == ']':
|
|
66
|
+
j += 1
|
|
67
|
+
while j < n and pattern[j] != ']':
|
|
68
|
+
j += 1
|
|
69
|
+
if j < n:
|
|
70
|
+
tokens.append(pattern[i : j + 1])
|
|
71
|
+
i = j + 1
|
|
72
|
+
continue
|
|
73
|
+
tokens.append('[[]')
|
|
74
|
+
else:
|
|
75
|
+
tokens.append(pattern[i])
|
|
76
|
+
i += 1
|
|
77
|
+
return tokens
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _object_matches(securable: str, pattern: str) -> bool:
|
|
81
|
+
"""The pattern matches the row's object, or some object below it, even when `*` spans a dot.
|
|
82
|
+
|
|
83
|
+
A child's name is `securable + '.' + rest`, where rest is not empty and adds at most one more part under a
|
|
84
|
+
catalog and none under a schema. Some child can match when a prefix of the pattern's tokens matches
|
|
85
|
+
`securable + '.'` exactly, and the remaining tokens can match a nonempty rest with no more literal dots
|
|
86
|
+
than rest may contain. (A `*` ending the prefix can also run on into rest.) Matching ignores case, as Unity
|
|
87
|
+
Catalog does.
|
|
88
|
+
"""
|
|
89
|
+
pattern = pattern.lower() # Unity Catalog names are case-insensitive and stored in lowercase
|
|
90
|
+
if fnmatchcase(securable, pattern):
|
|
91
|
+
return True
|
|
92
|
+
budget = 1 - securable.count('.') # catalog 1, schema 0; a table has no children
|
|
93
|
+
if budget < 0:
|
|
94
|
+
return False
|
|
95
|
+
tokens, head = _glob_tokens(pattern), securable + '.'
|
|
96
|
+
return any(
|
|
97
|
+
(k < len(tokens) or tokens[k - 1] == '*')
|
|
98
|
+
and tokens[k:].count('.') <= budget
|
|
99
|
+
and fnmatchcase(head, ''.join(tokens[:k]))
|
|
100
|
+
for k in range(1, len(tokens) + 1)
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _matches(rule, row: Change, before: State, after: State, globs) -> bool:
|
|
105
|
+
"""Matching fails closed: a parent row and a principal of unknown kind can both hide a match."""
|
|
106
|
+
if rule.when != 'any' and row.direction != ('+' if rule.when == 'gain' else '-'):
|
|
107
|
+
return False
|
|
108
|
+
if rule.levels and row.level not in rule.levels:
|
|
109
|
+
return False
|
|
110
|
+
if rule.objects and not any(_object_matches(row.securable, p) for p in rule.objects):
|
|
111
|
+
return False
|
|
112
|
+
if rule.principal_kinds and row.kind not in rule.principal_kinds and row.kind != 'unknown':
|
|
113
|
+
return False
|
|
114
|
+
state = after if row.direction == '+' else before
|
|
115
|
+
if rule.sensitive is not None and sensitive(state, row.securable, globs) != rule.sensitive:
|
|
116
|
+
return False
|
|
117
|
+
return not _exempt(row, rule.unless_via)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def evaluate(
|
|
121
|
+
diff: AccessDiff, plan: Plan, before: State, after: State, config: Config, *, deep: bool
|
|
122
|
+
) -> tuple[Finding, ...]:
|
|
123
|
+
findings: list[Finding] = []
|
|
124
|
+
|
|
125
|
+
def add(rule: str, subject: str, message: str, severity: str | None = None) -> None:
|
|
126
|
+
severity = severity or config.builtin[rule]
|
|
127
|
+
if severity != 'off':
|
|
128
|
+
findings.append(Finding(severity, rule, subject, message))
|
|
129
|
+
|
|
130
|
+
extra: list[Gap] = []
|
|
131
|
+
changed = changed_objects(plan)
|
|
132
|
+
for row in (r for r in diff.rows if r.direction == '+'):
|
|
133
|
+
if row.principal in ('account users', 'users'):
|
|
134
|
+
add(
|
|
135
|
+
'broad_principal', _subject(row), f'{BUILTIN_GROUPS[row.principal]} gain {row.level} on {row.securable}'
|
|
136
|
+
)
|
|
137
|
+
if row.level in ('READ', 'WRITE') and config.builtin['sensitive_access'] != 'off':
|
|
138
|
+
if sensitive(after, row.securable, config.sensitive_tags):
|
|
139
|
+
add(
|
|
140
|
+
'sensitive_access',
|
|
141
|
+
_subject(row),
|
|
142
|
+
f'{row.principal} gains {row.level} on sensitive data in {row.securable}',
|
|
143
|
+
)
|
|
144
|
+
elif kind_of(row.securable) != 'table' and not deep:
|
|
145
|
+
extra.append(
|
|
146
|
+
Gap(
|
|
147
|
+
'sensitivity_not_evaluated',
|
|
148
|
+
f'child table sensitivity not evaluated for {row.securable} (use --deep)',
|
|
149
|
+
)
|
|
150
|
+
)
|
|
151
|
+
elif deep and _column_tags_unread(after, row.securable, changed):
|
|
152
|
+
what = row.securable if kind_of(row.securable) == 'table' else f'child tables under {row.securable}'
|
|
153
|
+
extra.append(Gap('sensitivity_not_evaluated', f'column tags of {what} were not read'))
|
|
154
|
+
for resource in plan.changed_grants():
|
|
155
|
+
if kind_of(resource.securable) == 'table':
|
|
156
|
+
continue
|
|
157
|
+
for principal, privileges in sorted((resource.after or {}).items()):
|
|
158
|
+
broad = sorted((privileges - (resource.before or {}).get(principal, frozenset())) & BROAD)
|
|
159
|
+
if broad:
|
|
160
|
+
add(
|
|
161
|
+
'broad_privilege',
|
|
162
|
+
f'{principal} {"+".join(broad)} {resource.securable}',
|
|
163
|
+
f'{", ".join(broad)} granted to {principal} on {kind_of(resource.securable)} {resource.securable}',
|
|
164
|
+
)
|
|
165
|
+
for resource in plan.changed_owners():
|
|
166
|
+
if resource.after and after.kind(resource.after) == 'user':
|
|
167
|
+
add(
|
|
168
|
+
'individual_owner',
|
|
169
|
+
f'{resource.after} OWNER {resource.securable}',
|
|
170
|
+
f'ownership of {resource.securable} moves to the individual user {resource.after}',
|
|
171
|
+
)
|
|
172
|
+
if has_access_changes(plan):
|
|
173
|
+
for gap in plan.gaps:
|
|
174
|
+
if gap.category == 'conflicting_grant_resources':
|
|
175
|
+
securable, _, detail = gap.detail.partition(': ')
|
|
176
|
+
add('grant_resource_conflict', securable, f'{detail}, and grants can be lost without an error')
|
|
177
|
+
for rule in config.rules:
|
|
178
|
+
for row in diff.all_rows:
|
|
179
|
+
if _matches(rule, row, before, after, config.sensitive_tags):
|
|
180
|
+
add(rule.id, _subject(row), rule.message, rule.severity)
|
|
181
|
+
unreadable = any(g.category in UNREADABLE_PLAN_VALUES for g in plan.gaps)
|
|
182
|
+
if has_access_changes(plan) or plan.not_evaluated or unreadable:
|
|
183
|
+
gaps = set(plan.gaps) | set(before.gaps) | set(after.gaps) | set(diff.gaps) | set(extra)
|
|
184
|
+
if plan.not_evaluated:
|
|
185
|
+
gaps.add(Gap('not_evaluated', f'{len(plan.not_evaluated)} change(s) are outside what v0.1 evaluates'))
|
|
186
|
+
by_category: dict[str, list[str]] = defaultdict(list)
|
|
187
|
+
for gap in sorted(gaps):
|
|
188
|
+
by_category[gap.category].append(gap.detail)
|
|
189
|
+
for category, details in sorted(by_category.items()):
|
|
190
|
+
more = f' (+{len(details) - 1} more)' if len(details) > 1 else ''
|
|
191
|
+
add('unverified', category, details[0] + more)
|
|
192
|
+
return tuple(sorted(set(findings)))
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def status_of(findings: tuple[Finding, ...]) -> str:
|
|
196
|
+
severities = {f.severity for f in findings}
|
|
197
|
+
return 'BLOCK' if 'BLOCK' in severities else 'WARN' if 'WARN' in severities else 'PASS'
|
reachdiff/scope.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Which objects a plan touches."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterable
|
|
6
|
+
|
|
7
|
+
from reachdiff.state import ancestors
|
|
8
|
+
from reachdiff.tfplan import Plan
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def changed_objects(plan: Plan) -> set[str]:
|
|
12
|
+
names = {g.securable for g in plan.changed_grants()}
|
|
13
|
+
return names | {o.securable for o in plan.changed_owners()}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def with_ancestors(names: Iterable[str]) -> set[str]:
|
|
17
|
+
return {a for name in names for a in ancestors(name)}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def has_access_changes(plan: Plan) -> bool:
|
|
21
|
+
return bool(changed_objects(plan)) or bool(plan.changed_memberships())
|
reachdiff/state.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""Snapshot of Unity Catalog access facts. Pure data; no I/O."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
KINDS = ('catalog', 'schema', 'table')
|
|
8
|
+
BUILTIN_GROUPS = {'account users': 'all account users', 'users': 'all workspace users', 'admins': 'workspace admins'}
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def normalize_privilege(value: str) -> str:
|
|
12
|
+
return value.strip().upper().replace(' ', '_')
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def kind_of(name: str) -> str:
|
|
16
|
+
parts = name.split('.')
|
|
17
|
+
if len(parts) > 3 or any(not part for part in parts):
|
|
18
|
+
raise ValueError(f'unsupported securable name: {name!r}')
|
|
19
|
+
return KINDS[len(parts) - 1]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def parent_of(name: str) -> str | None:
|
|
23
|
+
return name.rsplit('.', 1)[0] if '.' in name else None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def ancestors(name: str) -> tuple[str, ...]:
|
|
27
|
+
"""The object itself first, then its schema and catalog."""
|
|
28
|
+
chain = [name]
|
|
29
|
+
while (parent := parent_of(chain[-1])) is not None:
|
|
30
|
+
chain.append(parent)
|
|
31
|
+
return tuple(chain)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True, order=True)
|
|
35
|
+
class Gap:
|
|
36
|
+
category: str
|
|
37
|
+
detail: str
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class Principal:
|
|
42
|
+
name: str
|
|
43
|
+
kind: str # 'user' | 'service_principal' | 'group' | 'unknown'
|
|
44
|
+
scim_id: str | None = None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class Securable:
|
|
49
|
+
name: str
|
|
50
|
+
owner: str | None = None
|
|
51
|
+
tags: frozenset[str] = frozenset() # tag keys on the object and, for tables, its columns
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def kind(self) -> str:
|
|
55
|
+
return kind_of(self.name)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass(frozen=True)
|
|
59
|
+
class Grant:
|
|
60
|
+
principal: str
|
|
61
|
+
securable: str
|
|
62
|
+
privileges: frozenset[str]
|
|
63
|
+
evidence: str = field(compare=False)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class Membership:
|
|
68
|
+
member: str
|
|
69
|
+
group: str
|
|
70
|
+
evidence: str = field(compare=False)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True)
|
|
74
|
+
class State:
|
|
75
|
+
source: str # 'live' | 'offline'
|
|
76
|
+
principals: dict[str, Principal] # keyed by UC grant name
|
|
77
|
+
securables: dict[str, Securable] # keyed by full name
|
|
78
|
+
grants: tuple[Grant, ...]
|
|
79
|
+
memberships: tuple[Membership, ...]
|
|
80
|
+
gaps: tuple[Gap, ...] = ()
|
|
81
|
+
notes: tuple[str, ...] = ()
|
|
82
|
+
unread_groups: frozenset[str] = frozenset() # groups whose members could not be read
|
|
83
|
+
|
|
84
|
+
def kind(self, name: str) -> str:
|
|
85
|
+
if name in self.principals:
|
|
86
|
+
return self.principals[name].kind
|
|
87
|
+
if name in BUILTIN_GROUPS or any(m.group == name for m in self.memberships):
|
|
88
|
+
return 'group'
|
|
89
|
+
return 'user' if '@' in name else 'unknown'
|
reachdiff/tfplan.py
ADDED
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
"""Read `terraform show -json` output. Extract only the resource types reachdiff needs.
|
|
2
|
+
|
|
3
|
+
Plan JSON can contain secrets in plain text. Only the attributes named here are read,
|
|
4
|
+
and error messages name resource addresses, never values.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
|
|
12
|
+
from reachdiff.state import Gap, Principal, kind_of, normalize_privilege
|
|
13
|
+
|
|
14
|
+
SUPPORTED_ATTRS = ('catalog', 'schema', 'table')
|
|
15
|
+
OTHER_ATTRS = (
|
|
16
|
+
'volume',
|
|
17
|
+
'function',
|
|
18
|
+
'model',
|
|
19
|
+
'external_location',
|
|
20
|
+
'storage_credential',
|
|
21
|
+
'metastore',
|
|
22
|
+
'share',
|
|
23
|
+
'foreign_connection',
|
|
24
|
+
'credential',
|
|
25
|
+
)
|
|
26
|
+
MODELED = frozenset({'USE_CATALOG', 'USE_SCHEMA', 'SELECT', 'MODIFY', 'MANAGE', 'ALL_PRIVILEGES', 'BROWSE'})
|
|
27
|
+
GRANT_TYPES = ('databricks_grants', 'databricks_grant')
|
|
28
|
+
OWNER_TYPES = {
|
|
29
|
+
'databricks_catalog': ('name',),
|
|
30
|
+
'databricks_schema': ('catalog_name', 'name'),
|
|
31
|
+
'databricks_sql_table': ('catalog_name', 'schema_name', 'name'),
|
|
32
|
+
}
|
|
33
|
+
IDENTITY_TYPES = {
|
|
34
|
+
'databricks_group': ('group', 'display_name'),
|
|
35
|
+
'databricks_user': ('user', 'user_name'),
|
|
36
|
+
'databricks_service_principal': ('service_principal', 'application_id'),
|
|
37
|
+
}
|
|
38
|
+
RELEVANT = {
|
|
39
|
+
'databricks_grants': SUPPORTED_ATTRS + OTHER_ATTRS + ('grant',),
|
|
40
|
+
'databricks_grant': SUPPORTED_ATTRS + OTHER_ATTRS + ('principal', 'privileges'),
|
|
41
|
+
'databricks_group_member': ('group_id', 'member_id'),
|
|
42
|
+
**{t: keys + ('owner',) for t, keys in OWNER_TYPES.items()},
|
|
43
|
+
}
|
|
44
|
+
ACCOUNT_HOST = re.compile(r'^https://accounts\.')
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class GrantResource:
|
|
49
|
+
address: str
|
|
50
|
+
securable: str
|
|
51
|
+
authoritative: bool
|
|
52
|
+
before: dict[str, frozenset[str]] | None
|
|
53
|
+
after: dict[str, frozenset[str]] | None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass(frozen=True)
|
|
57
|
+
class MembershipResource:
|
|
58
|
+
address: str
|
|
59
|
+
group_id: str
|
|
60
|
+
member_id: str
|
|
61
|
+
before: bool
|
|
62
|
+
after: bool
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class OwnerResource:
|
|
67
|
+
address: str
|
|
68
|
+
securable: str
|
|
69
|
+
before: str | None
|
|
70
|
+
after: str | None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True)
|
|
74
|
+
class Plan:
|
|
75
|
+
hosts: tuple[str, ...]
|
|
76
|
+
grants: tuple[GrantResource, ...]
|
|
77
|
+
memberships: tuple[MembershipResource, ...]
|
|
78
|
+
owners: tuple[OwnerResource, ...]
|
|
79
|
+
identities: dict[str, Principal]
|
|
80
|
+
gaps: tuple[Gap, ...]
|
|
81
|
+
not_evaluated: tuple[str, ...]
|
|
82
|
+
synthetic: bool = False
|
|
83
|
+
|
|
84
|
+
def changed_grants(self) -> tuple[GrantResource, ...]:
|
|
85
|
+
return tuple(g for g in self.grants if g.before != g.after)
|
|
86
|
+
|
|
87
|
+
def changed_memberships(self) -> tuple[MembershipResource, ...]:
|
|
88
|
+
return tuple(m for m in self.memberships if m.before != m.after)
|
|
89
|
+
|
|
90
|
+
def changed_owners(self) -> tuple[OwnerResource, ...]:
|
|
91
|
+
return tuple(o for o in self.owners if o.before != o.after)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _any_true(value: object) -> bool:
|
|
95
|
+
if value is True:
|
|
96
|
+
return True
|
|
97
|
+
if isinstance(value, dict):
|
|
98
|
+
return any(_any_true(v) for v in value.values())
|
|
99
|
+
if isinstance(value, list):
|
|
100
|
+
return any(_any_true(v) for v in value)
|
|
101
|
+
return False
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _flagged(marks: object, keys: tuple[str, ...]) -> bool:
|
|
105
|
+
"""True if a listed attribute is marked in a Terraform unknown/sensitive structure."""
|
|
106
|
+
if marks is True:
|
|
107
|
+
return True
|
|
108
|
+
return isinstance(marks, dict) and any(_any_true(marks.get(k)) for k in keys)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _privileges(value: object, address: str) -> frozenset[str]:
|
|
112
|
+
if not isinstance(value, list) or not all(isinstance(p, str) and p.strip() for p in value):
|
|
113
|
+
raise ValueError(f'{address}: privileges must be a list of names')
|
|
114
|
+
return frozenset(normalize_privilege(p) for p in value)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _securable(values: dict, address: str) -> tuple[str | None, str | None, object]:
|
|
118
|
+
"""(name, None, name) for an evaluated securable, or (None, attribute, value) for another type.
|
|
119
|
+
|
|
120
|
+
The last element only tells the two sides of a change apart. It is never emitted.
|
|
121
|
+
"""
|
|
122
|
+
for attr in SUPPORTED_ATTRS:
|
|
123
|
+
name = values.get(attr)
|
|
124
|
+
if name:
|
|
125
|
+
if not isinstance(name, str):
|
|
126
|
+
raise ValueError(f'{address}: {attr} is not a valid {attr} full name')
|
|
127
|
+
try:
|
|
128
|
+
if kind_of(name) != attr:
|
|
129
|
+
raise ValueError(f'{address}: {attr} is not a valid {attr} full name')
|
|
130
|
+
except ValueError:
|
|
131
|
+
raise ValueError(f'{address}: {attr} is not a valid {attr} full name') from None
|
|
132
|
+
return name.lower(), None, name.lower()
|
|
133
|
+
for attr in OTHER_ATTRS:
|
|
134
|
+
if values.get(attr):
|
|
135
|
+
return None, attr, values[attr]
|
|
136
|
+
raise ValueError(f'{address}: grant resource has no securable attribute')
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _grant_map(values: dict, authoritative: bool, address: str) -> dict[str, frozenset[str]]:
|
|
140
|
+
if not authoritative:
|
|
141
|
+
principal = values.get('principal')
|
|
142
|
+
if not isinstance(principal, str) or not principal:
|
|
143
|
+
raise ValueError(f'{address}: principal must be a nonempty string')
|
|
144
|
+
return {principal: _privileges(values.get('privileges'), address)}
|
|
145
|
+
blocks = values.get('grant') or []
|
|
146
|
+
if not isinstance(blocks, list):
|
|
147
|
+
raise ValueError(f'{address}: grant must be a list')
|
|
148
|
+
result: dict[str, frozenset[str]] = {}
|
|
149
|
+
for block in blocks:
|
|
150
|
+
principal = block.get('principal') if isinstance(block, dict) else None
|
|
151
|
+
if not isinstance(principal, str) or not principal:
|
|
152
|
+
raise ValueError(f'{address}: every grant block needs a principal')
|
|
153
|
+
result[principal] = result.get(principal, frozenset()) | _privileges(block.get('privileges'), address)
|
|
154
|
+
return result
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _edge(values: object, address: str) -> tuple[str, str] | None:
|
|
158
|
+
if values is None:
|
|
159
|
+
return None
|
|
160
|
+
group, member = values.get('group_id'), values.get('member_id')
|
|
161
|
+
if not isinstance(group, str) or not group or not isinstance(member, str) or not member:
|
|
162
|
+
raise ValueError(f'{address}: group_id and member_id must be nonempty strings')
|
|
163
|
+
return group, member
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _owned(values: object, rtype: str, address: str) -> tuple[str | None, str | None]:
|
|
167
|
+
"""(securable name, owner) for an owner-bearing resource, or (None, None)."""
|
|
168
|
+
if values is None:
|
|
169
|
+
return None, None
|
|
170
|
+
parts = [values.get(k) for k in OWNER_TYPES[rtype]]
|
|
171
|
+
if not all(isinstance(p, str) and p for p in parts):
|
|
172
|
+
return None, None
|
|
173
|
+
value = values.get('owner')
|
|
174
|
+
if value is not None and not isinstance(value, str):
|
|
175
|
+
raise ValueError(f'{address}: owner must be a string')
|
|
176
|
+
return '.'.join(parts).lower(), value or None
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _add_identity(values: object, rtype: str, identities: dict[str, Principal], *marks: object) -> None:
|
|
180
|
+
"""marks: the sensitivity structures covering `values`. A name marked sensitive is never read."""
|
|
181
|
+
kind, field = IDENTITY_TYPES[rtype]
|
|
182
|
+
if any(_flagged(m, (field,)) for m in marks):
|
|
183
|
+
return
|
|
184
|
+
if isinstance(values, dict) and isinstance(values.get('id'), str) and isinstance(values.get(field), str):
|
|
185
|
+
identities[values['id']] = Principal(values[field], kind, values['id'])
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _prior_resources(module: object):
|
|
189
|
+
if not isinstance(module, dict):
|
|
190
|
+
return
|
|
191
|
+
yield from (r for r in module.get('resources') or [] if isinstance(r, dict))
|
|
192
|
+
for child in module.get('child_modules') or []:
|
|
193
|
+
yield from _prior_resources(child)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _object(value: object, where: str) -> dict:
|
|
197
|
+
"""An optional JSON object: absent or null reads as empty, anything else must be an object."""
|
|
198
|
+
if value is None:
|
|
199
|
+
return {}
|
|
200
|
+
if not isinstance(value, dict):
|
|
201
|
+
raise ValueError(f'{where} must be an object')
|
|
202
|
+
return value
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _providers(raw: dict) -> tuple[tuple[str, ...], int]:
|
|
206
|
+
configs = _object(
|
|
207
|
+
_object(raw.get('configuration'), 'configuration').get('provider_config'), 'configuration.provider_config'
|
|
208
|
+
)
|
|
209
|
+
hosts, workspace = set(), 0
|
|
210
|
+
for key, config in configs.items():
|
|
211
|
+
if not isinstance(config, dict):
|
|
212
|
+
raise ValueError(f'provider_config {key} must be an object')
|
|
213
|
+
if config.get('name') != 'databricks':
|
|
214
|
+
continue
|
|
215
|
+
expressions = _object(config.get('expressions'), f'provider_config {key}: expressions')
|
|
216
|
+
host = _object(expressions.get('host'), f'provider_config {key}: expressions.host').get('constant_value')
|
|
217
|
+
if 'account_id' in expressions or (isinstance(host, str) and ACCOUNT_HOST.match(host)):
|
|
218
|
+
continue
|
|
219
|
+
workspace += 1
|
|
220
|
+
if isinstance(host, str) and host:
|
|
221
|
+
hosts.add(host.rstrip('/'))
|
|
222
|
+
return tuple(sorted(hosts)), workspace
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _conflicts(grants: list[GrantResource]) -> list[str]:
|
|
226
|
+
"""Grants on one securable set by more than one resource, counted by address. Resources being destroyed count."""
|
|
227
|
+
by_securable: dict[str, list[GrantResource]] = {}
|
|
228
|
+
for resource in grants:
|
|
229
|
+
by_securable.setdefault(resource.securable, []).append(resource)
|
|
230
|
+
details = []
|
|
231
|
+
for securable, resources in sorted(by_securable.items()):
|
|
232
|
+
authoritative = sorted({r.address for r in resources if r.authoritative})
|
|
233
|
+
holders: dict[str, set[str]] = {}
|
|
234
|
+
for r in resources:
|
|
235
|
+
if not r.authoritative:
|
|
236
|
+
for principal in set(r.before or {}) | set(r.after or {}):
|
|
237
|
+
holders.setdefault(principal, set()).add(r.address)
|
|
238
|
+
if authoritative and holders:
|
|
239
|
+
details.append(f'{securable}: databricks_grants and databricks_grant both manage it')
|
|
240
|
+
if len(authoritative) > 1:
|
|
241
|
+
details.append(f'{securable}: {", ".join(authoritative)} each manage all of its grants')
|
|
242
|
+
for principal, addresses in sorted(holders.items()):
|
|
243
|
+
if len(addresses) > 1:
|
|
244
|
+
details.append(f'{securable}: {", ".join(sorted(addresses))} each manage the grants of {principal}')
|
|
245
|
+
return details
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def read_plan(raw: object) -> Plan:
|
|
249
|
+
if not isinstance(raw, dict):
|
|
250
|
+
raise ValueError('plan must be a JSON object produced by terraform show -json')
|
|
251
|
+
version = raw.get('format_version')
|
|
252
|
+
if not isinstance(version, str) or version.split('.')[0] != '1':
|
|
253
|
+
raise ValueError('unsupported plan format_version; expected 1.x from terraform show -json')
|
|
254
|
+
changes = raw.get('resource_changes', [])
|
|
255
|
+
if not isinstance(changes, list):
|
|
256
|
+
raise ValueError('resource_changes must be a list')
|
|
257
|
+
hosts, workspace_providers = _providers(raw)
|
|
258
|
+
gaps: list[Gap] = []
|
|
259
|
+
if workspace_providers > 1:
|
|
260
|
+
gaps.append(
|
|
261
|
+
Gap(
|
|
262
|
+
'multiple_workspace_providers',
|
|
263
|
+
f'the plan has {workspace_providers} workspace '
|
|
264
|
+
'provider configurations; results assume they share one metastore',
|
|
265
|
+
)
|
|
266
|
+
)
|
|
267
|
+
not_evaluated: list[str] = []
|
|
268
|
+
grants, memberships, owners = [], [], []
|
|
269
|
+
identities: dict[str, Principal] = {}
|
|
270
|
+
for resource in _prior_resources(((raw.get('prior_state') or {}).get('values') or {}).get('root_module')):
|
|
271
|
+
if resource.get('type') in IDENTITY_TYPES:
|
|
272
|
+
_add_identity(resource.get('values'), resource['type'], identities, resource.get('sensitive_values'))
|
|
273
|
+
for rc in changes:
|
|
274
|
+
if not isinstance(rc, dict) or not isinstance(rc.get('change'), dict):
|
|
275
|
+
raise ValueError('resource_changes entries must be objects with a change')
|
|
276
|
+
rtype, address, change = rc.get('type'), str(rc.get('address', '?')), rc['change']
|
|
277
|
+
if rtype not in IDENTITY_TYPES and (rc.get('mode') != 'managed' or rtype not in RELEVANT):
|
|
278
|
+
continue
|
|
279
|
+
before, after = change.get('before'), change.get('after')
|
|
280
|
+
if not all(v is None or isinstance(v, dict) for v in (before, after)):
|
|
281
|
+
raise ValueError(f'{address}: change.before and change.after must be objects or null')
|
|
282
|
+
if rtype in IDENTITY_TYPES:
|
|
283
|
+
for values in (before, after):
|
|
284
|
+
_add_identity(values, rtype, identities, change.get('before_sensitive'), change.get('after_sensitive'))
|
|
285
|
+
continue
|
|
286
|
+
if 'forget' in (change.get('actions') or []):
|
|
287
|
+
after = before
|
|
288
|
+
keys = RELEVANT[rtype]
|
|
289
|
+
if _flagged(change.get('before_sensitive'), keys) or _flagged(change.get('after_sensitive'), keys):
|
|
290
|
+
gaps.append(Gap('sensitive_value', f'{address}: a relevant value is marked sensitive and was not read'))
|
|
291
|
+
continue
|
|
292
|
+
if after is not None and _flagged(change.get('after_unknown'), keys):
|
|
293
|
+
gaps.append(Gap('unknown_after_apply', f'{address}: relevant values are known only after apply'))
|
|
294
|
+
after = before
|
|
295
|
+
if rtype in GRANT_TYPES:
|
|
296
|
+
if before is None and after is None:
|
|
297
|
+
continue
|
|
298
|
+
authoritative = rtype == 'databricks_grants'
|
|
299
|
+
old_target = None if before is None else _securable(before, address)
|
|
300
|
+
new_target = None if after is None else _securable(after, address)
|
|
301
|
+
old = None if before is None else _grant_map(before, authoritative, address)
|
|
302
|
+
new = None if after is None else _grant_map(after, authoritative, address)
|
|
303
|
+
if old_target and new_target and old_target[1:] != new_target[1:]:
|
|
304
|
+
# The securable itself changes: the old one loses these grants, the new one gains them.
|
|
305
|
+
sides = [(old_target, old, None), (new_target, None, new)]
|
|
306
|
+
else:
|
|
307
|
+
sides = [(new_target or old_target, old, new)]
|
|
308
|
+
for (securable, other, _), side_old, side_new in sides:
|
|
309
|
+
if other:
|
|
310
|
+
if side_old != side_new:
|
|
311
|
+
not_evaluated.append(f'{address}: grants on a {other.replace("_", " ")}')
|
|
312
|
+
continue
|
|
313
|
+
grants.append(GrantResource(address, securable, authoritative, side_old, side_new))
|
|
314
|
+
for principal in sorted(set(side_old or {}) | set(side_new or {})):
|
|
315
|
+
delta = (side_old or {}).get(principal, frozenset()) ^ (side_new or {}).get(principal, frozenset())
|
|
316
|
+
if delta - MODELED:
|
|
317
|
+
not_evaluated.append(
|
|
318
|
+
f'{address}: {", ".join(sorted(delta - MODELED))} for {principal} on {securable}'
|
|
319
|
+
)
|
|
320
|
+
elif rtype == 'databricks_group_member':
|
|
321
|
+
old, new = _edge(before, address), _edge(after, address)
|
|
322
|
+
if old and new and old != new:
|
|
323
|
+
memberships += [
|
|
324
|
+
MembershipResource(address, *old, True, False),
|
|
325
|
+
MembershipResource(address, *new, False, True),
|
|
326
|
+
]
|
|
327
|
+
elif old or new:
|
|
328
|
+
memberships.append(MembershipResource(address, *(old or new), old is not None, new is not None))
|
|
329
|
+
else:
|
|
330
|
+
(old_name, old_owner), (new_name, new_owner) = _owned(before, rtype, address), _owned(after, rtype, address)
|
|
331
|
+
securable = new_name or old_name
|
|
332
|
+
if securable:
|
|
333
|
+
owners.append(
|
|
334
|
+
OwnerResource(
|
|
335
|
+
address,
|
|
336
|
+
securable,
|
|
337
|
+
old_owner if old_name == securable else None,
|
|
338
|
+
new_owner if new_name == securable else None,
|
|
339
|
+
)
|
|
340
|
+
)
|
|
341
|
+
for detail in _conflicts(grants):
|
|
342
|
+
gaps.append(Gap('conflicting_grant_resources', f'{detail}; the applied result depends on apply order'))
|
|
343
|
+
return Plan(
|
|
344
|
+
hosts,
|
|
345
|
+
tuple(grants),
|
|
346
|
+
tuple(memberships),
|
|
347
|
+
tuple(owners),
|
|
348
|
+
identities,
|
|
349
|
+
tuple(sorted(set(gaps))),
|
|
350
|
+
tuple(sorted(set(not_evaluated))),
|
|
351
|
+
raw.get('reachdiff_fixture') == 'synthetic',
|
|
352
|
+
)
|