reachdiff 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reachdiff/__init__.py +0 -0
- reachdiff/__main__.py +3 -0
- reachdiff/apply.py +52 -0
- reachdiff/cli.py +88 -0
- reachdiff/collect/__init__.py +0 -0
- reachdiff/collect/live.py +463 -0
- reachdiff/collect/offline.py +32 -0
- reachdiff/config.py +135 -0
- reachdiff/diff.py +234 -0
- reachdiff/report.py +215 -0
- reachdiff/resolve.py +104 -0
- reachdiff/rules.py +197 -0
- reachdiff/scope.py +21 -0
- reachdiff/state.py +89 -0
- reachdiff/tfplan.py +352 -0
- reachdiff-0.9.0.dist-info/METADATA +372 -0
- reachdiff-0.9.0.dist-info/RECORD +21 -0
- reachdiff-0.9.0.dist-info/WHEEL +5 -0
- reachdiff-0.9.0.dist-info/entry_points.txt +2 -0
- reachdiff-0.9.0.dist-info/licenses/LICENSE +202 -0
- reachdiff-0.9.0.dist-info/top_level.txt +1 -0
reachdiff/__init__.py
ADDED
|
File without changes
|
reachdiff/__main__.py
ADDED
reachdiff/apply.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Apply a plan's changes to a state using Terraform provider semantics."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import replace
|
|
6
|
+
|
|
7
|
+
from reachdiff.state import Gap, Grant, Membership, Securable, State
|
|
8
|
+
from reachdiff.tfplan import Plan
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def apply_plan(state: State, plan: Plan) -> State:
|
|
12
|
+
grants = list(state.grants)
|
|
13
|
+
# Authoritative resources first, then principal resources, so the result does not depend on list order.
|
|
14
|
+
for resource in sorted(plan.changed_grants(), key=lambda r: not r.authoritative):
|
|
15
|
+
target = resource.after or {}
|
|
16
|
+
if resource.authoritative:
|
|
17
|
+
grants = [g for g in grants if g.securable != resource.securable]
|
|
18
|
+
else:
|
|
19
|
+
names = set(target) | set(resource.before or {})
|
|
20
|
+
grants = [g for g in grants if not (g.securable == resource.securable and g.principal in names)]
|
|
21
|
+
for principal, privileges in sorted(target.items()):
|
|
22
|
+
grants.append(Grant(principal, resource.securable, privileges, f'plan after: {resource.address}'))
|
|
23
|
+
|
|
24
|
+
ids = {p.scim_id: p for p in state.principals.values() if p.scim_id}
|
|
25
|
+
ids.update(plan.identities)
|
|
26
|
+
principals = dict(state.principals)
|
|
27
|
+
gaps = list(state.gaps)
|
|
28
|
+
memberships = list(state.memberships)
|
|
29
|
+
for edge in plan.changed_memberships():
|
|
30
|
+
group, member = ids.get(edge.group_id), ids.get(edge.member_id)
|
|
31
|
+
if group is None or member is None:
|
|
32
|
+
gaps.append(Gap('unresolved_name', f'{edge.address}: SCIM IDs could not be named'))
|
|
33
|
+
continue
|
|
34
|
+
principals.setdefault(group.name, group)
|
|
35
|
+
principals.setdefault(member.name, member)
|
|
36
|
+
link = Membership(member.name, group.name, f'plan after: {edge.address}')
|
|
37
|
+
memberships = [m for m in memberships if m != link]
|
|
38
|
+
if edge.after:
|
|
39
|
+
memberships.append(link)
|
|
40
|
+
|
|
41
|
+
securables = dict(state.securables)
|
|
42
|
+
for resource in plan.changed_owners():
|
|
43
|
+
current = securables.get(resource.securable, Securable(resource.securable))
|
|
44
|
+
securables[resource.securable] = replace(current, owner=resource.after)
|
|
45
|
+
return replace(
|
|
46
|
+
state,
|
|
47
|
+
principals=principals,
|
|
48
|
+
securables=securables,
|
|
49
|
+
grants=tuple(grants),
|
|
50
|
+
memberships=tuple(memberships),
|
|
51
|
+
gaps=tuple(sorted(set(gaps))),
|
|
52
|
+
)
|
reachdiff/cli.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""`reachdiff plan`: who gains or loses Unity Catalog access when a Terraform plan is applied."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from reachdiff.apply import apply_plan
|
|
12
|
+
from reachdiff.collect.offline import build_offline
|
|
13
|
+
from reachdiff.config import load_config
|
|
14
|
+
from reachdiff.diff import compute_diff
|
|
15
|
+
from reachdiff.report import RENDERERS, build_report
|
|
16
|
+
from reachdiff.rules import evaluate, status_of
|
|
17
|
+
from reachdiff.tfplan import read_plan
|
|
18
|
+
|
|
19
|
+
# Without one of these the SDK tries every login it finds, such as the Terraform deployer's ARM_* secrets.
|
|
20
|
+
LOGIN_PINS = ('DATABRICKS_AUTH_TYPE', 'DATABRICKS_CONFIG_PROFILE')
|
|
21
|
+
RANK = {'PASS': 0, 'WARN': 1, 'BLOCK': 2}
|
|
22
|
+
THRESHOLD = {'warn': 1, 'block': 2, 'never': 3}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def exit_code(status: str, fail_on: str) -> int:
|
|
26
|
+
return RANK[status] if RANK[status] >= THRESHOLD[fail_on] else 0
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _parser() -> argparse.ArgumentParser:
|
|
30
|
+
parser = argparse.ArgumentParser(
|
|
31
|
+
prog='reachdiff', description='Who gains or loses Unity Catalog access when this Terraform plan is applied?'
|
|
32
|
+
)
|
|
33
|
+
commands = parser.add_subparsers(dest='command', required=True)
|
|
34
|
+
plan = commands.add_parser('plan', help='analyze terraform show -json output')
|
|
35
|
+
plan.add_argument('--tfplan', required=True, type=Path, help='output of terraform show -json PLANFILE')
|
|
36
|
+
mode = plan.add_mutually_exclusive_group()
|
|
37
|
+
mode.add_argument('--offline', action='store_true', help='use only data in the plan and report blind spots')
|
|
38
|
+
mode.add_argument('--profile', help='Databricks SDK profile for live, read-only reads')
|
|
39
|
+
plan.add_argument(
|
|
40
|
+
'--deep', action='store_true', help='also read child tables of changed schemas and catalogs (live mode only)'
|
|
41
|
+
)
|
|
42
|
+
plan.add_argument('--config', type=Path, help='reachdiff TOML config')
|
|
43
|
+
plan.add_argument('--format', choices=sorted(RENDERERS), default='text')
|
|
44
|
+
plan.add_argument('--output', type=Path, help='write the report to this file (overwrites) instead of stdout')
|
|
45
|
+
plan.add_argument('--fail-on', choices=sorted(THRESHOLD), help='overrides fail_on from the config')
|
|
46
|
+
return parser
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def main(argv: list[str] | None = None) -> int:
|
|
50
|
+
try:
|
|
51
|
+
args = _parser().parse_args(argv)
|
|
52
|
+
except SystemExit as exc: # argparse uses 2 for usage errors; 2 means BLOCK here
|
|
53
|
+
return 0 if exc.code == 0 else 3
|
|
54
|
+
if args.offline and args.deep:
|
|
55
|
+
print('reachdiff: error: --deep needs live mode; it cannot be combined with --offline', file=sys.stderr)
|
|
56
|
+
return 3
|
|
57
|
+
if not (args.offline or args.profile or any(os.environ.get(name) for name in LOGIN_PINS)):
|
|
58
|
+
print(
|
|
59
|
+
'reachdiff: error: live mode needs --profile or DATABRICKS_AUTH_TYPE, so a missing credential cannot '
|
|
60
|
+
'fall back to another login',
|
|
61
|
+
file=sys.stderr,
|
|
62
|
+
)
|
|
63
|
+
return 3
|
|
64
|
+
try:
|
|
65
|
+
config = load_config(args.config)
|
|
66
|
+
plan = read_plan(json.loads(args.tfplan.read_text(encoding='utf-8')))
|
|
67
|
+
if args.offline:
|
|
68
|
+
before = build_offline(plan)
|
|
69
|
+
else:
|
|
70
|
+
from reachdiff.collect.live import collect_live
|
|
71
|
+
|
|
72
|
+
before = collect_live(plan, config, deep=args.deep, profile=args.profile)
|
|
73
|
+
after = apply_plan(before, plan)
|
|
74
|
+
diff = compute_diff(before, after, plan, members_limit=config.limit_members, deep=args.deep)
|
|
75
|
+
findings = evaluate(diff, plan, before, after, config, deep=args.deep)
|
|
76
|
+
status = status_of(findings)
|
|
77
|
+
output = RENDERERS[args.format](build_report(plan, before, after, diff, findings, status))
|
|
78
|
+
if args.output:
|
|
79
|
+
args.output.write_text(output, encoding='utf-8')
|
|
80
|
+
else:
|
|
81
|
+
sys.stdout.write(output)
|
|
82
|
+
except (ValueError, RuntimeError, OSError) as exc:
|
|
83
|
+
print(f'reachdiff: error: {exc}', file=sys.stderr)
|
|
84
|
+
return 3
|
|
85
|
+
except Exception as exc: # the message could quote plan values, so only the type is shown
|
|
86
|
+
print(f'reachdiff: error: unexpected {type(exc).__name__}', file=sys.stderr)
|
|
87
|
+
return 3
|
|
88
|
+
return exit_code(status, args.fail_on or config.fail_on)
|
|
File without changes
|
|
@@ -0,0 +1,463 @@
|
|
|
1
|
+
"""Read-only Databricks reads limited to what a plan touches. The only module that performs I/O.
|
|
2
|
+
|
|
3
|
+
Request shapes follow databricks-sdk 0.140.0 and were checked against an Azure workspace on 1 October 2026; the
|
|
4
|
+
visibility self-check (effective permissions, SCIM Me) is re-checked in milestone 1a.
|
|
5
|
+
Exception text is never kept: SDK errors can include request details.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from urllib.parse import quote
|
|
11
|
+
|
|
12
|
+
from reachdiff.config import Config
|
|
13
|
+
from reachdiff.scope import changed_objects, with_ancestors
|
|
14
|
+
from reachdiff.state import (
|
|
15
|
+
BUILTIN_GROUPS,
|
|
16
|
+
Gap,
|
|
17
|
+
Grant,
|
|
18
|
+
Membership,
|
|
19
|
+
Principal,
|
|
20
|
+
Securable,
|
|
21
|
+
State,
|
|
22
|
+
ancestors,
|
|
23
|
+
kind_of,
|
|
24
|
+
normalize_privilege,
|
|
25
|
+
)
|
|
26
|
+
from reachdiff.tfplan import Plan
|
|
27
|
+
|
|
28
|
+
UC = '/api/2.1/unity-catalog'
|
|
29
|
+
SCIM = '/api/2.0/preview/scim/v2'
|
|
30
|
+
PLURAL = {'catalog': 'catalogs', 'schema': 'schemas', 'table': 'tables'}
|
|
31
|
+
LOOKUPS = {
|
|
32
|
+
'group': ('Groups', 'displayName'),
|
|
33
|
+
'service_principal': ('ServicePrincipals', 'applicationId'),
|
|
34
|
+
'user': ('Users', 'userName'),
|
|
35
|
+
}
|
|
36
|
+
MEMBER_KINDS = {
|
|
37
|
+
'user': 'user',
|
|
38
|
+
'users': 'user',
|
|
39
|
+
'group': 'group',
|
|
40
|
+
'groups': 'group',
|
|
41
|
+
'serviceprincipal': 'service_principal',
|
|
42
|
+
'serviceprincipals': 'service_principal',
|
|
43
|
+
}
|
|
44
|
+
SEES_ALL = frozenset({'READ_METADATA', 'MANAGE'}) # with ownership and metastore admin: see every grant
|
|
45
|
+
NEEDS = 'needs READ METADATA, MANAGE or ownership'
|
|
46
|
+
HIDDEN = f"only the caller's own grants are visible; {NEEDS}"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# The SDK raises ValueError when it cannot get a token, for example when a federation policy refuses the exchange.
|
|
50
|
+
LOGIN_ERRORS = ('Unauthenticated', 'ValueError')
|
|
51
|
+
LOGIN_HINT = (
|
|
52
|
+
"Check DATABRICKS_AUTH_TYPE or the profile, the client ID and, with token federation, the policy's "
|
|
53
|
+
'issuer, subject and audience'
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class _ReadError(Exception):
|
|
58
|
+
"""A failed or malformed read. Carries only an exception type name."""
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class _Reader:
|
|
62
|
+
def __init__(self, client) -> None:
|
|
63
|
+
self.client = client
|
|
64
|
+
self.attempted = self.succeeded = 0
|
|
65
|
+
self.first_failure: str | None = None
|
|
66
|
+
|
|
67
|
+
def get(self, path: str, query: dict | None = None) -> dict:
|
|
68
|
+
self.attempted += 1
|
|
69
|
+
try:
|
|
70
|
+
result = self.client.api_client.do('GET', path, query=query or {})
|
|
71
|
+
except Exception as exc:
|
|
72
|
+
if type(exc).__name__ in LOGIN_ERRORS:
|
|
73
|
+
raise RuntimeError(
|
|
74
|
+
f'Databricks login failed ({type(exc).__name__} on GET {path}). {LOGIN_HINT}'
|
|
75
|
+
) from None
|
|
76
|
+
self.first_failure = self.first_failure or f'{type(exc).__name__} on GET {path}'
|
|
77
|
+
raise _ReadError(type(exc).__name__) from None
|
|
78
|
+
if not isinstance(result, dict):
|
|
79
|
+
self.first_failure = self.first_failure or f'InvalidResponse on GET {path}'
|
|
80
|
+
raise _ReadError('InvalidResponse')
|
|
81
|
+
self.succeeded += 1
|
|
82
|
+
return result
|
|
83
|
+
|
|
84
|
+
def pages(self, path: str, key: str, query: dict | None = None, *, max_results: bool = True) -> list:
|
|
85
|
+
"""Follow next_page_token until it is absent; an empty page does not end the listing."""
|
|
86
|
+
items: list = []
|
|
87
|
+
token, seen = None, set()
|
|
88
|
+
while True:
|
|
89
|
+
params = dict(query or {})
|
|
90
|
+
if max_results:
|
|
91
|
+
params['max_results'] = 0
|
|
92
|
+
if token:
|
|
93
|
+
params['page_token'] = token
|
|
94
|
+
page = self.get(path, params)
|
|
95
|
+
batch = page.get(key, [])
|
|
96
|
+
if not isinstance(batch, list):
|
|
97
|
+
raise _ReadError('InvalidResponse')
|
|
98
|
+
items.extend(batch)
|
|
99
|
+
token = page.get('next_page_token')
|
|
100
|
+
if not token:
|
|
101
|
+
return items
|
|
102
|
+
if not isinstance(token, str) or token in seen:
|
|
103
|
+
raise _ReadError('RepeatedPageToken')
|
|
104
|
+
seen.add(token)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _read_grants(reader: _Reader, name: str, gaps: list[Gap]) -> list[Grant] | None:
|
|
108
|
+
path = f'{UC}/permissions/{kind_of(name)}/{quote(name, safe="")}'
|
|
109
|
+
grants = []
|
|
110
|
+
try:
|
|
111
|
+
for item in reader.pages(path, 'privilege_assignments'):
|
|
112
|
+
principal = item.get('principal') if isinstance(item, dict) else None
|
|
113
|
+
privileges = item.get('privileges') if isinstance(item, dict) else None
|
|
114
|
+
if (
|
|
115
|
+
not isinstance(principal, str)
|
|
116
|
+
or not isinstance(privileges, list)
|
|
117
|
+
or not all(isinstance(p, str) and p.strip() for p in privileges)
|
|
118
|
+
):
|
|
119
|
+
raise _ReadError('InvalidAssignment')
|
|
120
|
+
if privileges:
|
|
121
|
+
grants.append(Grant(principal, name, frozenset(map(normalize_privilege, privileges)), f'GET {path}'))
|
|
122
|
+
except _ReadError as exc:
|
|
123
|
+
gaps.append(Gap('unreadable_object', f'grants on {name} ({exc}); partial data discarded'))
|
|
124
|
+
return None
|
|
125
|
+
return grants
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _tags(reader: _Reader, entity_type: str, name: str, gaps: list[Gap]) -> set[str]:
|
|
129
|
+
path = f'{UC}/entity-tag-assignments/{entity_type}/{quote(name, safe="")}/tags'
|
|
130
|
+
try:
|
|
131
|
+
items = reader.pages(path, 'tag_assignments', max_results=False)
|
|
132
|
+
except _ReadError as exc:
|
|
133
|
+
gaps.append(Gap('tags_unavailable', f'{name} ({exc})'))
|
|
134
|
+
return set()
|
|
135
|
+
return {i['tag_key'] for i in items if isinstance(i, dict) and isinstance(i.get('tag_key'), str)}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _read_securable(reader: _Reader, name: str, column_tags: bool, gaps: list[Gap]) -> Securable:
|
|
139
|
+
kind = kind_of(name)
|
|
140
|
+
owner, columns = None, []
|
|
141
|
+
try:
|
|
142
|
+
detail = reader.get(f'{UC}/{PLURAL[kind]}/{quote(name, safe="")}')
|
|
143
|
+
owner = detail['owner'] if isinstance(detail.get('owner'), str) else None
|
|
144
|
+
columns = [
|
|
145
|
+
c['name'] for c in detail.get('columns') or [] if isinstance(c, dict) and isinstance(c.get('name'), str)
|
|
146
|
+
]
|
|
147
|
+
except _ReadError as exc:
|
|
148
|
+
gaps.append(Gap('unreadable_object', f'owner of {name} ({exc})'))
|
|
149
|
+
tags = _tags(reader, PLURAL[kind], name, gaps)
|
|
150
|
+
if column_tags:
|
|
151
|
+
for column in columns:
|
|
152
|
+
tags |= _tags(reader, 'columns', f'{name}.{column}', gaps)
|
|
153
|
+
return Securable(name, owner, frozenset(tags))
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _children(reader: _Reader, changed: set[str], limit: int, gaps: list[Gap]) -> set[str]:
|
|
157
|
+
schemas: set[str] = set()
|
|
158
|
+
for name in sorted(changed):
|
|
159
|
+
if kind_of(name) == 'schema':
|
|
160
|
+
schemas.add(name)
|
|
161
|
+
elif kind_of(name) == 'catalog':
|
|
162
|
+
try:
|
|
163
|
+
listed = reader.pages(f'{UC}/schemas', 'schemas', {'catalog_name': name})
|
|
164
|
+
except _ReadError as exc:
|
|
165
|
+
gaps.append(Gap('unreadable_object', f'schemas in {name} ({exc})'))
|
|
166
|
+
continue
|
|
167
|
+
schemas |= {
|
|
168
|
+
s['full_name'].lower() for s in listed if isinstance(s, dict) and isinstance(s.get('full_name'), str)
|
|
169
|
+
}
|
|
170
|
+
found = set(schemas)
|
|
171
|
+
for schema in sorted(schemas):
|
|
172
|
+
catalog, schema_name = schema.split('.')
|
|
173
|
+
try:
|
|
174
|
+
listed = reader.pages(f'{UC}/tables', 'tables', {'catalog_name': catalog, 'schema_name': schema_name})
|
|
175
|
+
except _ReadError as exc:
|
|
176
|
+
gaps.append(Gap('unreadable_object', f'tables in {schema} ({exc})'))
|
|
177
|
+
continue
|
|
178
|
+
found |= {t['full_name'].lower() for t in listed if isinstance(t, dict) and isinstance(t.get('full_name'), str)}
|
|
179
|
+
if len(found) > limit:
|
|
180
|
+
gaps.append(Gap('limit_reached', f'{len(found)} child objects under --deep; only the first {limit} were read'))
|
|
181
|
+
found = set(sorted(found)[:limit])
|
|
182
|
+
return found
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _check_since_plan(plan: Plan, live: dict[str, dict[str, frozenset[str]]], gaps: list[Gap]) -> None:
|
|
186
|
+
for resource in plan.grants:
|
|
187
|
+
if resource.before is None or resource.securable not in live:
|
|
188
|
+
continue
|
|
189
|
+
current = live[resource.securable]
|
|
190
|
+
observed = current if resource.authoritative else {p: current.get(p, frozenset()) for p in resource.before}
|
|
191
|
+
if observed != resource.before:
|
|
192
|
+
gaps.append(
|
|
193
|
+
Gap(
|
|
194
|
+
'changed_since_plan',
|
|
195
|
+
f"grants on {resource.securable} differ from the plan's refreshed state ({resource.address})",
|
|
196
|
+
)
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _lookup_name(reader: _Reader, name: str, gaps: list[Gap]) -> Principal | None:
|
|
201
|
+
if '"' in name or '\\' in name:
|
|
202
|
+
gaps.append(Gap('unresolved_name', f'{name} cannot be looked up safely'))
|
|
203
|
+
return None
|
|
204
|
+
kinds = ('user', 'group', 'service_principal') if '@' in name else ('group', 'service_principal', 'user')
|
|
205
|
+
for kind in kinds:
|
|
206
|
+
resource, attribute = LOOKUPS[kind]
|
|
207
|
+
try:
|
|
208
|
+
page = reader.get(f'{SCIM}/{resource}', {'filter': f'{attribute} eq "{name}"', 'attributes': 'id'})
|
|
209
|
+
except _ReadError as exc:
|
|
210
|
+
gaps.append(Gap('unresolved_name', f'{name} ({exc})'))
|
|
211
|
+
return None
|
|
212
|
+
ids = [r['id'] for r in page.get('Resources') or [] if isinstance(r, dict) and isinstance(r.get('id'), str)]
|
|
213
|
+
if len(ids) > 1:
|
|
214
|
+
gaps.append(Gap('unresolved_name', f'{name} matches more than one {kind}'))
|
|
215
|
+
return None
|
|
216
|
+
if ids:
|
|
217
|
+
return Principal(name, kind, ids[0])
|
|
218
|
+
gaps.append(Gap('unresolved_name', f'{name} was not found in workspace SCIM'))
|
|
219
|
+
return None
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _lookup_id(reader: _Reader, scim_id: str, gaps: list[Gap]) -> Principal | None:
|
|
223
|
+
refused = False
|
|
224
|
+
for kind in ('user', 'service_principal', 'group'):
|
|
225
|
+
resource, attribute = LOOKUPS[kind]
|
|
226
|
+
try:
|
|
227
|
+
record = reader.get(f'{SCIM}/{resource}/{quote(scim_id, safe="")}')
|
|
228
|
+
except _ReadError as exc:
|
|
229
|
+
refused = refused or str(exc) == 'PermissionDenied'
|
|
230
|
+
continue
|
|
231
|
+
if record.get('id') == scim_id and isinstance(record.get(attribute), str):
|
|
232
|
+
return Principal(record[attribute], kind, scim_id)
|
|
233
|
+
gaps.append(
|
|
234
|
+
Gap(
|
|
235
|
+
'unresolved_name',
|
|
236
|
+
f'SCIM ID {scim_id} from the plan (lookup needs workspace admin)'
|
|
237
|
+
if refused
|
|
238
|
+
else f'SCIM ID {scim_id} from the plan was not found',
|
|
239
|
+
)
|
|
240
|
+
)
|
|
241
|
+
return None
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _member_kind(member: dict) -> str:
|
|
245
|
+
kind = member.get('type')
|
|
246
|
+
if not isinstance(kind, str):
|
|
247
|
+
parts = str(member.get('$ref', '')).rstrip('/').split('/')
|
|
248
|
+
kind = parts[-2] if len(parts) >= 2 else ''
|
|
249
|
+
return MEMBER_KINDS.get(kind.lower().replace('_', ''), 'unknown')
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _read_identities(
|
|
253
|
+
reader: _Reader, plan: Plan, grants: list[Grant], limit: int, gaps: list[Gap]
|
|
254
|
+
) -> tuple[dict[str, Principal], list[Membership], set[str]]:
|
|
255
|
+
names = {g.principal for g in grants}
|
|
256
|
+
for resource in plan.changed_grants():
|
|
257
|
+
names |= set(resource.before or {}) | set(resource.after or {})
|
|
258
|
+
for resource in plan.changed_owners():
|
|
259
|
+
names |= {x for x in (resource.before, resource.after) if x}
|
|
260
|
+
principals: dict[str, Principal] = {}
|
|
261
|
+
by_id: dict[str, Principal] = {}
|
|
262
|
+
for name in sorted(names):
|
|
263
|
+
found = Principal(name, 'group') if name in BUILTIN_GROUPS else _lookup_name(reader, name, gaps)
|
|
264
|
+
if found:
|
|
265
|
+
principals[name] = found
|
|
266
|
+
if found.scim_id:
|
|
267
|
+
by_id[found.scim_id] = found
|
|
268
|
+
pending = [p.scim_id for p in principals.values() if p.kind == 'group' and p.scim_id]
|
|
269
|
+
for edge in plan.changed_memberships():
|
|
270
|
+
pending.append(edge.group_id)
|
|
271
|
+
gaps.append(
|
|
272
|
+
Gap(
|
|
273
|
+
'membership_scope',
|
|
274
|
+
f'{edge.address}: grants the group holds outside the objects this plan touches were not read',
|
|
275
|
+
)
|
|
276
|
+
)
|
|
277
|
+
if edge.member_id not in by_id and (found := _lookup_id(reader, edge.member_id, gaps)):
|
|
278
|
+
principals.setdefault(found.name, found)
|
|
279
|
+
by_id[edge.member_id] = found
|
|
280
|
+
memberships: list[Membership] = []
|
|
281
|
+
unread: set[str] = set()
|
|
282
|
+
expanded: set[str] = set()
|
|
283
|
+
count = 0
|
|
284
|
+
while pending:
|
|
285
|
+
group_id = pending.pop(0)
|
|
286
|
+
if group_id in expanded:
|
|
287
|
+
continue
|
|
288
|
+
expanded.add(group_id)
|
|
289
|
+
path = f'{SCIM}/Groups/{quote(group_id, safe="")}'
|
|
290
|
+
try:
|
|
291
|
+
record = reader.get(path)
|
|
292
|
+
name, members = record.get('displayName'), record.get('members', [])
|
|
293
|
+
if record.get('id') != group_id or not isinstance(name, str) or not isinstance(members, list):
|
|
294
|
+
raise _ReadError('InvalidGroup')
|
|
295
|
+
except _ReadError as exc:
|
|
296
|
+
if str(exc) == 'PermissionDenied':
|
|
297
|
+
# Workspace SCIM shows members only to admins; every further group would be refused too.
|
|
298
|
+
gaps.append(
|
|
299
|
+
Gap(
|
|
300
|
+
'membership_incomplete',
|
|
301
|
+
'group members not read: workspace SCIM shows members '
|
|
302
|
+
'only to workspace admins (PermissionDenied)',
|
|
303
|
+
)
|
|
304
|
+
)
|
|
305
|
+
unread |= {by_id[g].name for g in [group_id, *pending] if g in by_id}
|
|
306
|
+
return principals, memberships, unread
|
|
307
|
+
gaps.append(Gap('membership_incomplete', f'group {group_id} ({exc})'))
|
|
308
|
+
if group_id in by_id:
|
|
309
|
+
unread.add(by_id[group_id].name)
|
|
310
|
+
continue
|
|
311
|
+
group = by_id.get(group_id) or Principal(name, 'group', group_id)
|
|
312
|
+
by_id[group_id] = group
|
|
313
|
+
principals.setdefault(group.name, group)
|
|
314
|
+
for member in members:
|
|
315
|
+
member_id = member.get('value') if isinstance(member, dict) else None
|
|
316
|
+
if not isinstance(member_id, str):
|
|
317
|
+
gaps.append(Gap('membership_incomplete', f'malformed member entry in {group.name}'))
|
|
318
|
+
continue
|
|
319
|
+
count += 1
|
|
320
|
+
if count > limit:
|
|
321
|
+
gaps.append(Gap('limit_reached', f'group members beyond {limit} were not read'))
|
|
322
|
+
return principals, memberships, unread
|
|
323
|
+
known = by_id.get(member_id)
|
|
324
|
+
if known is None:
|
|
325
|
+
kind = _member_kind(member)
|
|
326
|
+
display = member['display'] if isinstance(member.get('display'), str) else member_id
|
|
327
|
+
known = Principal(display if kind == 'group' else f'{display} ({kind} {member_id})', kind, member_id)
|
|
328
|
+
by_id[member_id] = known
|
|
329
|
+
principals.setdefault(known.name, known)
|
|
330
|
+
memberships.append(Membership(known.name, group.name, f'GET {path}: members.value={member_id}'))
|
|
331
|
+
if known.kind == 'group':
|
|
332
|
+
pending.append(member_id)
|
|
333
|
+
return principals, memberships, unread
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _caller(reader: _Reader, gaps: list[Gap]) -> tuple[str, frozenset[str]] | None:
|
|
337
|
+
"""The caller's grant name (a service principal's application ID) and that name plus its direct groups."""
|
|
338
|
+
try:
|
|
339
|
+
me = reader.get(f'{SCIM}/Me')
|
|
340
|
+
except _ReadError as exc:
|
|
341
|
+
gaps.append(Gap('unreadable_object', f"the caller's identity (SCIM Me: {exc})"))
|
|
342
|
+
return None
|
|
343
|
+
user = me.get('userName')
|
|
344
|
+
if not isinstance(user, str) or not user:
|
|
345
|
+
gaps.append(Gap('unreadable_object', "the caller's identity (SCIM Me: InvalidResponse)"))
|
|
346
|
+
return None
|
|
347
|
+
groups = {g['display'] for g in me.get('groups') or [] if isinstance(g, dict) and isinstance(g.get('display'), str)}
|
|
348
|
+
return user, frozenset({user} | groups)
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _metastore_admin(reader: _Reader, names: frozenset[str]) -> bool:
|
|
352
|
+
try:
|
|
353
|
+
return reader.get(f'{UC}/metastore_summary').get('owner') in names
|
|
354
|
+
except _ReadError:
|
|
355
|
+
return False
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _hidden(
|
|
359
|
+
reader: _Reader, name: str, caller, admin: bool, securables: dict[str, Securable], cleared: set[str]
|
|
360
|
+
) -> str | None:
|
|
361
|
+
"""None when the caller sees every grant on the object; otherwise why not.
|
|
362
|
+
|
|
363
|
+
Databricks shows other callers only their own grants, without an error. Objects are checked parents first, so a
|
|
364
|
+
cleared parent clears its children (READ METADATA and MANAGE inherit; ownership covers children).
|
|
365
|
+
"""
|
|
366
|
+
if caller is None:
|
|
367
|
+
return 'the caller could not be identified, so the list may hold only its own grants'
|
|
368
|
+
user, names = caller
|
|
369
|
+
chain = ancestors(name)
|
|
370
|
+
if (
|
|
371
|
+
admin
|
|
372
|
+
or any(n in cleared for n in chain[1:])
|
|
373
|
+
or any(n in securables and securables[n].owner in names for n in chain)
|
|
374
|
+
):
|
|
375
|
+
cleared.add(name)
|
|
376
|
+
return None
|
|
377
|
+
path = f'{UC}/effective-permissions/{kind_of(name)}/{quote(name, safe="")}'
|
|
378
|
+
try:
|
|
379
|
+
items = reader.pages(path, 'privilege_assignments', {'principal': user})
|
|
380
|
+
except _ReadError as exc:
|
|
381
|
+
if str(exc) == 'PermissionDenied': # the caller cannot read the object at all
|
|
382
|
+
return f'the caller cannot see this object (PermissionDenied); {NEEDS}'
|
|
383
|
+
return f'visibility check failed ({exc})'
|
|
384
|
+
held = {
|
|
385
|
+
normalize_privilege(p['privilege'])
|
|
386
|
+
for item in items
|
|
387
|
+
if isinstance(item, dict)
|
|
388
|
+
for p in item.get('privileges') or []
|
|
389
|
+
if isinstance(p, dict) and isinstance(p.get('privilege'), str)
|
|
390
|
+
}
|
|
391
|
+
if held & SEES_ALL:
|
|
392
|
+
cleared.add(name)
|
|
393
|
+
return None
|
|
394
|
+
return HIDDEN
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def collect(plan: Plan, client, config: Config, *, deep: bool) -> State:
|
|
398
|
+
host = str(client.config.host).rstrip('/')
|
|
399
|
+
if plan.hosts and host not in plan.hosts:
|
|
400
|
+
raise ValueError('the plan targets a different workspace host than the authenticated one')
|
|
401
|
+
notes = () if plan.hosts else ('Target workspace is not stated in the plan.',)
|
|
402
|
+
reader, gaps = _Reader(client), []
|
|
403
|
+
changed = changed_objects(plan)
|
|
404
|
+
base = sorted(with_ancestors(changed)) # always read: the object limit truncates only deep children
|
|
405
|
+
children = sorted(_children(reader, changed, config.limit_deep_children, gaps) - set(base)) if deep else []
|
|
406
|
+
dropped = min(len(children), max(len(base) + len(children) - config.limit_objects, 0))
|
|
407
|
+
if dropped:
|
|
408
|
+
gaps.append(
|
|
409
|
+
Gap(
|
|
410
|
+
'limit_reached',
|
|
411
|
+
f'{len(base) + len(children)} objects in scope; {dropped} child objects '
|
|
412
|
+
f'under --deep were not read (limit {config.limit_objects})',
|
|
413
|
+
)
|
|
414
|
+
)
|
|
415
|
+
children = children[: len(children) - dropped]
|
|
416
|
+
names = base + children
|
|
417
|
+
caller = _caller(reader, gaps)
|
|
418
|
+
admin = caller is not None and _metastore_admin(reader, caller[1])
|
|
419
|
+
securables, grants, live, cleared = {}, [], {}, set()
|
|
420
|
+
for name in names: # parents first: names are sorted, and a name sorts before its children
|
|
421
|
+
securables[name] = _read_securable(reader, name, name in changed and kind_of(name) == 'table', gaps)
|
|
422
|
+
reason = _hidden(reader, name, caller, admin, securables, cleared)
|
|
423
|
+
if reason:
|
|
424
|
+
gaps.append(Gap('unreadable_object', f'grants on {name} ({reason}); partial data discarded'))
|
|
425
|
+
continue
|
|
426
|
+
object_grants = _read_grants(reader, name, gaps)
|
|
427
|
+
if object_grants is not None:
|
|
428
|
+
grants += object_grants
|
|
429
|
+
held = live.setdefault(name, {})
|
|
430
|
+
for grant in object_grants:
|
|
431
|
+
held[grant.principal] = held.get(grant.principal, frozenset()) | grant.privileges
|
|
432
|
+
_check_since_plan(plan, live, gaps)
|
|
433
|
+
principals, memberships, unread = _read_identities(reader, plan, grants, config.limit_members, gaps)
|
|
434
|
+
if reader.attempted and not reader.succeeded:
|
|
435
|
+
raise RuntimeError(
|
|
436
|
+
f'no Databricks read succeeded; first failure: {reader.first_failure}. Check the '
|
|
437
|
+
'workspace entitlement (e.g. workspace-consume) and the READ METADATA grant of the '
|
|
438
|
+
'identity reachdiff runs as'
|
|
439
|
+
)
|
|
440
|
+
return State(
|
|
441
|
+
'live',
|
|
442
|
+
principals,
|
|
443
|
+
securables,
|
|
444
|
+
tuple(grants),
|
|
445
|
+
tuple(memberships),
|
|
446
|
+
tuple(sorted(set(gaps))),
|
|
447
|
+
notes,
|
|
448
|
+
unread_groups=frozenset(unread),
|
|
449
|
+
)
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def collect_live(plan: Plan, config: Config, *, deep: bool, profile: str | None = None) -> State:
|
|
453
|
+
try:
|
|
454
|
+
from databricks.sdk import WorkspaceClient
|
|
455
|
+
except ImportError:
|
|
456
|
+
raise RuntimeError(
|
|
457
|
+
'live mode needs the Databricks SDK: pip install "reachdiff[databricks]", or pass --offline'
|
|
458
|
+
) from None
|
|
459
|
+
try:
|
|
460
|
+
client = WorkspaceClient(profile=profile) if profile else WorkspaceClient()
|
|
461
|
+
except Exception as exc:
|
|
462
|
+
raise RuntimeError(f'Databricks login failed ({type(exc).__name__}). {LOGIN_HINT}') from None
|
|
463
|
+
return collect(plan, client, config, deep=deep)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Current state from plan data only. Everything outside Terraform's view is a gap."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from reachdiff.scope import with_ancestors
|
|
6
|
+
from reachdiff.state import Gap, Grant, Membership, Securable, State
|
|
7
|
+
from reachdiff.tfplan import Plan
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def build_offline(plan: Plan) -> State:
|
|
11
|
+
gaps = list(plan.gaps) + [
|
|
12
|
+
Gap('offline_blind_spot', 'grants, memberships and tags not managed by this plan were not read (offline mode)')
|
|
13
|
+
]
|
|
14
|
+
grants = []
|
|
15
|
+
for resource in plan.grants:
|
|
16
|
+
for principal, privileges in sorted((resource.before or {}).items()):
|
|
17
|
+
grants.append(Grant(principal, resource.securable, privileges, f'plan before: {resource.address}'))
|
|
18
|
+
memberships = []
|
|
19
|
+
for edge in plan.memberships:
|
|
20
|
+
if not edge.before:
|
|
21
|
+
continue
|
|
22
|
+
group, member = plan.identities.get(edge.group_id), plan.identities.get(edge.member_id)
|
|
23
|
+
if group is None or member is None:
|
|
24
|
+
gaps.append(Gap('unresolved_name', f'{edge.address}: SCIM IDs are not named in the plan'))
|
|
25
|
+
continue
|
|
26
|
+
memberships.append(Membership(member.name, group.name, f'plan before: {edge.address}'))
|
|
27
|
+
names = with_ancestors([g.securable for g in plan.grants] + [o.securable for o in plan.owners])
|
|
28
|
+
securables = {name: Securable(name) for name in sorted(names)}
|
|
29
|
+
for resource in plan.owners:
|
|
30
|
+
securables[resource.securable] = Securable(resource.securable, resource.before)
|
|
31
|
+
principals = {p.name: p for p in plan.identities.values()}
|
|
32
|
+
return State('offline', principals, securables, tuple(grants), tuple(memberships), tuple(sorted(set(gaps))))
|