steve-cli 1.0.3__tar.gz → 1.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {steve_cli-1.0.3 → steve_cli-1.1.2}/PKG-INFO +2 -2
- {steve_cli-1.0.3 → steve_cli-1.1.2}/pyproject.toml +2 -2
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/auth.py +2 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/cli.py +277 -7
- steve_cli-1.1.2/steve_cli/policies/client.py +205 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/branch.py +18 -2
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/trino.py +173 -64
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli.egg-info/PKG-INFO +2 -2
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli.egg-info/requires.txt +1 -1
- steve_cli-1.0.3/steve_cli/policies/client.py +0 -93
- {steve_cli-1.0.3 → steve_cli-1.1.2}/README.md +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/setup.cfg +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/__init__.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/decorators/__init__.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/decorators/lineage_job.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/__init__.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/adapters/__init__.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/adapters/composite.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/adapters/dataproductregistry.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/adapters/logging.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/adapters/null.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/adapters/openlineage.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/collector.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/port.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/registry.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/storage.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/ontology.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/policies/__init__.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/__init__.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/__init__.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/__init__.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/csv.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/excel.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/generic.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/json.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/parquet.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/port.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/registry.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/parquet.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/protocol.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/s3.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/validation/__init__.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/validation/adapters/__init__.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/validation/adapters/great_expectations.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/validation/adapters/null.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/validation/adapters/validoopsie.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/validation/port.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/validation/registry.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/vkg.py +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli.egg-info/SOURCES.txt +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli.egg-info/dependency_links.txt +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli.egg-info/entry_points.txt +0 -0
- {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: steve-cli
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.1.2
|
|
4
4
|
Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
|
|
5
5
|
Author: Frank
|
|
6
6
|
License: MIT
|
|
@@ -49,7 +49,7 @@ Provides-Extra: trino
|
|
|
49
49
|
Requires-Dist: trino>=0.330; extra == "trino"
|
|
50
50
|
Requires-Dist: pyarrow>=17.0.0; extra == "trino"
|
|
51
51
|
Requires-Dist: requests>=2.31.0; extra == "trino"
|
|
52
|
-
Requires-Dist: pyiceberg>=0.
|
|
52
|
+
Requires-Dist: pyiceberg>=0.12.0; extra == "trino"
|
|
53
53
|
Requires-Dist: s3fs>=2024.1.0; extra == "trino"
|
|
54
54
|
Provides-Extra: visidata
|
|
55
55
|
Requires-Dist: visidata>=3.0; extra == "visidata"
|
|
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
|
|
|
5
5
|
|
|
6
6
|
[project]
|
|
7
7
|
name = "steve-cli"
|
|
8
|
-
version = "1.
|
|
8
|
+
version = "1.1.2"
|
|
9
9
|
description = "A simple CLI tool to run jobs from jobs.yaml with proper environment setup"
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
license = {text = "MIT"}
|
|
@@ -65,7 +65,7 @@ trino = [
|
|
|
65
65
|
"trino>=0.330",
|
|
66
66
|
"pyarrow>=17.0.0",
|
|
67
67
|
"requests>=2.31.0",
|
|
68
|
-
"pyiceberg>=0.
|
|
68
|
+
"pyiceberg>=0.12.0",
|
|
69
69
|
"s3fs>=2024.1.0",
|
|
70
70
|
]
|
|
71
71
|
visidata = [
|
|
@@ -74,6 +74,8 @@ def get_s3_endpoint(workspace_id: Optional[str] = None) -> Optional[str]:
|
|
|
74
74
|
|
|
75
75
|
|
|
76
76
|
def get_service_url(service: str, workspace_id: Optional[str] = None) -> Optional[str]:
|
|
77
|
+
import os
|
|
78
|
+
workspace_id = workspace_id or os.environ.get("WORKSPACE_ID")
|
|
77
79
|
entry = _get_workspace_entry(workspace_id)
|
|
78
80
|
base_url = entry.get("base_url")
|
|
79
81
|
if not base_url or not workspace_id:
|
|
@@ -940,7 +940,9 @@ def policies():
|
|
|
940
940
|
@policies.command("apply")
|
|
941
941
|
@click.option('--file', '-f', type=click.Path(path_type=Path),
|
|
942
942
|
help='Path to policies YAML file (default: ./policies/access.yaml)')
|
|
943
|
-
|
|
943
|
+
@click.option('--validate', type=click.Choice(["warn", "error"]), default=None,
|
|
944
|
+
help='Validate schema/table references against Trino after applying (warn: print issues, error: fail if any found)')
|
|
945
|
+
def policies_apply(file: Path | None, validate: str | None):
|
|
944
946
|
"""Apply access policies from a YAML file to the Policy Control Plane."""
|
|
945
947
|
from dotenv import load_dotenv
|
|
946
948
|
load_dotenv(".env", override=False)
|
|
@@ -948,7 +950,8 @@ def policies_apply(file: Path | None):
|
|
|
948
950
|
from steve_cli.policies import PolicyClient
|
|
949
951
|
policies_file = Path(file) if file else None
|
|
950
952
|
try:
|
|
951
|
-
PolicyClient().apply_from_file(policies_file)
|
|
953
|
+
branch, sanitized = PolicyClient().apply_from_file(policies_file, validate=validate)
|
|
954
|
+
click.secho(f"Branch: {branch} → schema suffix __{sanitized}", fg="cyan")
|
|
952
955
|
click.secho("Policies applied successfully.", fg="green")
|
|
953
956
|
except FileNotFoundError as e:
|
|
954
957
|
click.secho(str(e), fg="red", err=True)
|
|
@@ -958,6 +961,262 @@ def policies_apply(file: Path | None):
|
|
|
958
961
|
sys.exit(1)
|
|
959
962
|
|
|
960
963
|
|
|
964
|
+
@policies.command("ls")
|
|
965
|
+
def policies_ls():
|
|
966
|
+
"""List all registered PDPs and their attached workspaces."""
|
|
967
|
+
from dotenv import load_dotenv
|
|
968
|
+
load_dotenv(".env", override=False)
|
|
969
|
+
from steve_cli.policies import PolicyClient
|
|
970
|
+
try:
|
|
971
|
+
pdps = PolicyClient().list_pdps()
|
|
972
|
+
except Exception as e:
|
|
973
|
+
click.secho(f"ERROR: {e}", fg="red", bold=True, err=True)
|
|
974
|
+
sys.exit(1)
|
|
975
|
+
if not pdps:
|
|
976
|
+
click.echo("No PDPs registered.")
|
|
977
|
+
return
|
|
978
|
+
for pdp in pdps:
|
|
979
|
+
workspaces = [w.get("workspace", "") for w in pdp.get("workspaces", [])]
|
|
980
|
+
policies_count = len(pdp.get("policies", []))
|
|
981
|
+
click.echo(
|
|
982
|
+
f" {click.style(pdp['name'], fg='blue', bold=True)}"
|
|
983
|
+
f" {click.style(pdp['type'], fg='cyan')}"
|
|
984
|
+
f" {click.style(pdp['endpoint'], fg='bright_black')}"
|
|
985
|
+
)
|
|
986
|
+
if workspaces:
|
|
987
|
+
click.echo(f" workspaces: {', '.join(workspaces)}")
|
|
988
|
+
if pdp.get("policies"):
|
|
989
|
+
names = [p.get("name", p.get("id", "?")) for p in pdp["policies"]]
|
|
990
|
+
click.echo(f" policies: {', '.join(names)}")
|
|
991
|
+
else:
|
|
992
|
+
click.echo(f" policies: (none)")
|
|
993
|
+
|
|
994
|
+
|
|
995
|
+
@policies.command("show")
|
|
996
|
+
@click.argument("pdp_name", required=False, default=None)
|
|
997
|
+
def policies_show(pdp_name: str | None):
|
|
998
|
+
"""Show full policy content for a PDP (interactive if name omitted)."""
|
|
999
|
+
from dotenv import load_dotenv
|
|
1000
|
+
load_dotenv(".env", override=False)
|
|
1001
|
+
from steve_cli.policies import PolicyClient
|
|
1002
|
+
try:
|
|
1003
|
+
pdps = PolicyClient().list_pdps()
|
|
1004
|
+
except Exception as e:
|
|
1005
|
+
click.secho(f"ERROR: {e}", fg="red", bold=True, err=True)
|
|
1006
|
+
sys.exit(1)
|
|
1007
|
+
if not pdps:
|
|
1008
|
+
click.echo("No PDPs registered.")
|
|
1009
|
+
return
|
|
1010
|
+
if pdp_name is None:
|
|
1011
|
+
pdp_name = questionary.select(
|
|
1012
|
+
"Select a PDP:",
|
|
1013
|
+
choices=[p["name"] for p in pdps],
|
|
1014
|
+
).ask()
|
|
1015
|
+
if pdp_name is None:
|
|
1016
|
+
sys.exit(0)
|
|
1017
|
+
pdp = next((p for p in pdps if p["name"] == pdp_name), None)
|
|
1018
|
+
if not pdp:
|
|
1019
|
+
click.secho(f"PDP '{pdp_name}' not found.", fg="red", err=True)
|
|
1020
|
+
sys.exit(1)
|
|
1021
|
+
click.echo(f"\n{click.style(pdp['name'], fg='blue', bold=True)} {pdp['type']} {pdp['endpoint']}")
|
|
1022
|
+
import json as _json
|
|
1023
|
+
policies_list = pdp.get("policies", [])
|
|
1024
|
+
fragments_list = pdp.get("fragments", [])
|
|
1025
|
+
if not policies_list and not fragments_list:
|
|
1026
|
+
click.echo(" (no policies)")
|
|
1027
|
+
return
|
|
1028
|
+
for pol in policies_list:
|
|
1029
|
+
click.echo(f"\n {click.style(pol.get('name', pol.get('id', '?')), fg='cyan')} {click.style('rego', fg='bright_black')}")
|
|
1030
|
+
content = pol.get("content", pol.get("rawContent", ""))
|
|
1031
|
+
if isinstance(content, dict):
|
|
1032
|
+
content = _json.dumps(content, indent=2)
|
|
1033
|
+
for line in str(content).splitlines():
|
|
1034
|
+
click.echo(f" {line}")
|
|
1035
|
+
for frag in fragments_list:
|
|
1036
|
+
updated = frag.get("updatedAt", frag.get("createdAt", ""))
|
|
1037
|
+
click.echo(f"\n {click.style(frag.get('workspace', '?'), fg='cyan')} {click.style(frag.get('syntax', '?'), fg='magenta')} {click.style(updated, fg='bright_black')}")
|
|
1038
|
+
content = frag.get("content", {})
|
|
1039
|
+
for line in _json.dumps(content, indent=2).splitlines():
|
|
1040
|
+
click.echo(f" {line}")
|
|
1041
|
+
|
|
1042
|
+
|
|
1043
|
+
@policies.command("diff")
|
|
1044
|
+
@click.option('--file', '-f', type=click.Path(path_type=Path),
|
|
1045
|
+
help='Path to policies YAML file (default: ./policies/access.yaml)')
|
|
1046
|
+
@click.argument("pdp_name", required=False, default=None)
|
|
1047
|
+
def policies_diff(file: Path | None, pdp_name: str | None):
|
|
1048
|
+
"""Diff local access.yaml (expanded) against what is applied on a PDP."""
|
|
1049
|
+
from dotenv import load_dotenv
|
|
1050
|
+
load_dotenv(".env", override=False)
|
|
1051
|
+
import json as _json
|
|
1052
|
+
import difflib
|
|
1053
|
+
import yaml
|
|
1054
|
+
from steve_cli.policies import PolicyClient
|
|
1055
|
+
from steve_cli.storage.branch import get_branch_prefix
|
|
1056
|
+
|
|
1057
|
+
path = Path(file) if file else Path.cwd() / "policies" / "access.yaml"
|
|
1058
|
+
if not path.exists():
|
|
1059
|
+
click.secho(f"Policies file not found: {path}", fg="red", err=True)
|
|
1060
|
+
sys.exit(1)
|
|
1061
|
+
|
|
1062
|
+
config = yaml.safe_load(path.read_text())
|
|
1063
|
+
local_rules = config.get("rules", [])
|
|
1064
|
+
sanitized = get_branch_prefix()
|
|
1065
|
+
from steve_cli.policies.client import _expand_rules
|
|
1066
|
+
expanded = _expand_rules(local_rules, sanitized)
|
|
1067
|
+
local_text = _json.dumps({"rules": expanded}, indent=2)
|
|
1068
|
+
|
|
1069
|
+
try:
|
|
1070
|
+
client = PolicyClient()
|
|
1071
|
+
pdps = client.list_pdps()
|
|
1072
|
+
except Exception as e:
|
|
1073
|
+
click.secho(f"ERROR: {e}", fg="red", bold=True, err=True)
|
|
1074
|
+
sys.exit(1)
|
|
1075
|
+
|
|
1076
|
+
if not pdps:
|
|
1077
|
+
click.echo("No PDPs registered.")
|
|
1078
|
+
return
|
|
1079
|
+
|
|
1080
|
+
if pdp_name is None:
|
|
1081
|
+
pdp_cfg = config.get("pdp", {})
|
|
1082
|
+
pdp_name = pdp_cfg.get("name")
|
|
1083
|
+
if pdp_name not in [p["name"] for p in pdps]:
|
|
1084
|
+
pdp_name = questionary.select(
|
|
1085
|
+
"Select a PDP to diff against:",
|
|
1086
|
+
choices=[p["name"] for p in pdps],
|
|
1087
|
+
).ask()
|
|
1088
|
+
if pdp_name is None:
|
|
1089
|
+
sys.exit(0)
|
|
1090
|
+
|
|
1091
|
+
pdp = next((p for p in pdps if p["name"] == pdp_name), None)
|
|
1092
|
+
if not pdp:
|
|
1093
|
+
click.secho(f"PDP '{pdp_name}' not found.", fg="red", err=True)
|
|
1094
|
+
sys.exit(1)
|
|
1095
|
+
|
|
1096
|
+
remote_rules = []
|
|
1097
|
+
for frag in pdp.get("fragments", []):
|
|
1098
|
+
content = frag.get("content", {})
|
|
1099
|
+
if isinstance(content, dict) and "rules" in content:
|
|
1100
|
+
remote_rules.extend(content["rules"])
|
|
1101
|
+
for pol in pdp.get("policies", []):
|
|
1102
|
+
content = pol.get("content", {})
|
|
1103
|
+
if isinstance(content, dict) and "rules" in content:
|
|
1104
|
+
remote_rules.extend(content["rules"])
|
|
1105
|
+
remote_text = _json.dumps({"rules": remote_rules}, indent=2)
|
|
1106
|
+
|
|
1107
|
+
diff = list(difflib.unified_diff(
|
|
1108
|
+
remote_text.splitlines(),
|
|
1109
|
+
local_text.splitlines(),
|
|
1110
|
+
fromfile=f"remote/{pdp_name}",
|
|
1111
|
+
tofile=f"local/{path.name} (branch: {sanitized})",
|
|
1112
|
+
lineterm="",
|
|
1113
|
+
))
|
|
1114
|
+
|
|
1115
|
+
if not diff:
|
|
1116
|
+
click.secho("No differences.", fg="green")
|
|
1117
|
+
return
|
|
1118
|
+
|
|
1119
|
+
for line in diff:
|
|
1120
|
+
if line.startswith("+"):
|
|
1121
|
+
click.secho(line, fg="green")
|
|
1122
|
+
elif line.startswith("-"):
|
|
1123
|
+
click.secho(line, fg="red")
|
|
1124
|
+
else:
|
|
1125
|
+
click.echo(line)
|
|
1126
|
+
|
|
1127
|
+
|
|
1128
|
+
@policies.command("rm")
|
|
1129
|
+
@click.argument("pdp_name", required=False, default=None)
|
|
1130
|
+
@click.option("--yes", "-y", is_flag=True, default=False, help="Skip confirmation prompt.")
|
|
1131
|
+
def policies_rm(pdp_name: str | None, yes: bool):
|
|
1132
|
+
"""Delete a registered PDP (interactive if name omitted)."""
|
|
1133
|
+
from dotenv import load_dotenv
|
|
1134
|
+
load_dotenv(".env", override=False)
|
|
1135
|
+
from steve_cli.policies import PolicyClient
|
|
1136
|
+
try:
|
|
1137
|
+
client = PolicyClient()
|
|
1138
|
+
pdps = client.list_pdps()
|
|
1139
|
+
except Exception as e:
|
|
1140
|
+
click.secho(f"ERROR: {e}", fg="red", bold=True, err=True)
|
|
1141
|
+
sys.exit(1)
|
|
1142
|
+
if not pdps:
|
|
1143
|
+
click.echo("No PDPs registered.")
|
|
1144
|
+
return
|
|
1145
|
+
if pdp_name is None:
|
|
1146
|
+
pdp_name = questionary.select(
|
|
1147
|
+
"Select a PDP to delete:",
|
|
1148
|
+
choices=[p["name"] for p in pdps],
|
|
1149
|
+
).ask()
|
|
1150
|
+
if pdp_name is None:
|
|
1151
|
+
sys.exit(0)
|
|
1152
|
+
pdp = next((p for p in pdps if p["name"] == pdp_name), None)
|
|
1153
|
+
if not pdp:
|
|
1154
|
+
click.secho(f"PDP '{pdp_name}' not found.", fg="red", err=True)
|
|
1155
|
+
sys.exit(1)
|
|
1156
|
+
if not yes:
|
|
1157
|
+
if not questionary.confirm(f"Delete PDP '{pdp_name}'?", default=False).ask():
|
|
1158
|
+
click.echo("Aborted.")
|
|
1159
|
+
return
|
|
1160
|
+
try:
|
|
1161
|
+
client.delete_pdp(pdp["id"])
|
|
1162
|
+
click.secho(f"Deleted PDP '{pdp_name}'.", fg="green")
|
|
1163
|
+
except Exception as e:
|
|
1164
|
+
click.secho(f"ERROR: {e}", fg="red", bold=True, err=True)
|
|
1165
|
+
sys.exit(1)
|
|
1166
|
+
|
|
1167
|
+
|
|
1168
|
+
@policies.command("policy-rm")
|
|
1169
|
+
@click.argument("pdp_name", required=False, default=None)
|
|
1170
|
+
@click.option("--yes", "-y", is_flag=True, default=False, help="Skip confirmation prompt.")
|
|
1171
|
+
def policies_policy_rm(pdp_name: str | None, yes: bool):
|
|
1172
|
+
"""Delete a raw rego policy from a PDP (interactive if name omitted)."""
|
|
1173
|
+
from dotenv import load_dotenv
|
|
1174
|
+
load_dotenv(".env", override=False)
|
|
1175
|
+
from steve_cli.policies import PolicyClient
|
|
1176
|
+
try:
|
|
1177
|
+
client = PolicyClient()
|
|
1178
|
+
pdps = client.list_pdps()
|
|
1179
|
+
except Exception as e:
|
|
1180
|
+
click.secho(f"ERROR: {e}", fg="red", bold=True, err=True)
|
|
1181
|
+
sys.exit(1)
|
|
1182
|
+
if not pdps:
|
|
1183
|
+
click.echo("No PDPs registered.")
|
|
1184
|
+
return
|
|
1185
|
+
if pdp_name is None:
|
|
1186
|
+
pdp_name = questionary.select(
|
|
1187
|
+
"Select a PDP:",
|
|
1188
|
+
choices=[p["name"] for p in pdps],
|
|
1189
|
+
).ask()
|
|
1190
|
+
if pdp_name is None:
|
|
1191
|
+
sys.exit(0)
|
|
1192
|
+
pdp = next((p for p in pdps if p["name"] == pdp_name), None)
|
|
1193
|
+
if not pdp:
|
|
1194
|
+
click.secho(f"PDP '{pdp_name}' not found.", fg="red", err=True)
|
|
1195
|
+
sys.exit(1)
|
|
1196
|
+
policies_list = pdp.get("policies", [])
|
|
1197
|
+
if not policies_list:
|
|
1198
|
+
click.echo(f"No policies on PDP '{pdp_name}'.")
|
|
1199
|
+
return
|
|
1200
|
+
choices = [p.get("name", p.get("id", "?")) for p in policies_list]
|
|
1201
|
+
policy_name = questionary.select(
|
|
1202
|
+
"Select a policy to delete:",
|
|
1203
|
+
choices=choices,
|
|
1204
|
+
).ask()
|
|
1205
|
+
if policy_name is None:
|
|
1206
|
+
sys.exit(0)
|
|
1207
|
+
policy = next((p for p in policies_list if p.get("name", p.get("id")) == policy_name), None)
|
|
1208
|
+
if not yes:
|
|
1209
|
+
if not questionary.confirm(f"Delete policy '{policy_name}' from PDP '{pdp_name}'?", default=False).ask():
|
|
1210
|
+
click.echo("Aborted.")
|
|
1211
|
+
return
|
|
1212
|
+
try:
|
|
1213
|
+
client.delete_pdp_policy(policy["id"])
|
|
1214
|
+
click.secho(f"Deleted policy '{policy_name}' from PDP '{pdp_name}'.", fg="green")
|
|
1215
|
+
except Exception as e:
|
|
1216
|
+
click.secho(f"ERROR: {e}", fg="red", bold=True, err=True)
|
|
1217
|
+
sys.exit(1)
|
|
1218
|
+
|
|
1219
|
+
|
|
961
1220
|
@main.command("login")
|
|
962
1221
|
@click.option("--token", default=None, help="Service principal token (stp_...) to save")
|
|
963
1222
|
@click.option("--workspace", "workspace_id", default=None, help="Workspace ID (defaults to WORKSPACE_ID from .env)")
|
|
@@ -1112,6 +1371,7 @@ def status(env_file: tuple, no_check: bool):
|
|
|
1112
1371
|
expires_at = entry.get("expires_at", "")
|
|
1113
1372
|
if token:
|
|
1114
1373
|
expiry_warn = False
|
|
1374
|
+
token_expired = False
|
|
1115
1375
|
expiry_str = ", expires never" if not expires_at else ""
|
|
1116
1376
|
if expires_at:
|
|
1117
1377
|
try:
|
|
@@ -1129,6 +1389,7 @@ def status(env_file: tuple, no_check: bool):
|
|
|
1129
1389
|
ago = f"{s // 86400}d ago"
|
|
1130
1390
|
expiry_str = f", expired {ago}"
|
|
1131
1391
|
expiry_warn = True
|
|
1392
|
+
token_expired = True
|
|
1132
1393
|
elif total_seconds < 3600:
|
|
1133
1394
|
expiry_str = f", expires in {total_seconds // 60}m ⚠"
|
|
1134
1395
|
expiry_warn = True
|
|
@@ -1163,12 +1424,18 @@ def status(env_file: tuple, no_check: bool):
|
|
|
1163
1424
|
)
|
|
1164
1425
|
with urllib.request.urlopen(_req, timeout=4) as _r:
|
|
1165
1426
|
_r.read()
|
|
1166
|
-
|
|
1427
|
+
if token_expired:
|
|
1428
|
+
_row("Auth", f"✗ token expired (stp_***{token[-4:]}{expiry_str}) — run: steve login", "red")
|
|
1429
|
+
else:
|
|
1430
|
+
_row("Auth", f"✓ token valid (stp_***{token[-4:]}{expiry_str})", "green" if not expiry_warn else "yellow")
|
|
1167
1431
|
except urllib.error.HTTPError as _e:
|
|
1168
1432
|
if _e.code == 401:
|
|
1169
1433
|
_row("Auth", f"✗ token invalid/revoked (stp_***{token[-4:]}{expiry_str})", "red")
|
|
1170
1434
|
else:
|
|
1171
|
-
|
|
1435
|
+
if token_expired:
|
|
1436
|
+
_row("Auth", f"✗ token expired (stp_***{token[-4:]}{expiry_str}) — run: steve login", "red")
|
|
1437
|
+
else:
|
|
1438
|
+
_row("Auth", f"✓ token valid (stp_***{token[-4:]}{expiry_str})", "green" if not expiry_warn else "yellow")
|
|
1172
1439
|
except Exception as _e:
|
|
1173
1440
|
_row("Auth", f"{token_display} (validation failed: {_e})", token_color)
|
|
1174
1441
|
else:
|
|
@@ -1212,8 +1479,7 @@ def status(env_file: tuple, no_check: bool):
|
|
|
1212
1479
|
if trino_schema_explicit:
|
|
1213
1480
|
schema = trino_schema_explicit
|
|
1214
1481
|
elif resolved_workspace:
|
|
1215
|
-
|
|
1216
|
-
schema = f"{ws_clean}_{tier}__{branch}"
|
|
1482
|
+
schema = branch
|
|
1217
1483
|
else:
|
|
1218
1484
|
schema = f"(workspace not set — set WORKSPACE_NAME)"
|
|
1219
1485
|
_row(f"Schema {tier}", schema, "cyan" if resolved_workspace or trino_schema_explicit else "yellow")
|
|
@@ -1282,7 +1548,11 @@ def upgrade():
|
|
|
1282
1548
|
env=os.environ.copy(),
|
|
1283
1549
|
check=False,
|
|
1284
1550
|
)
|
|
1285
|
-
|
|
1551
|
+
if result.returncode != 0:
|
|
1552
|
+
sys.exit(result.returncode)
|
|
1553
|
+
click.echo("🔒 Syncing lockfile...")
|
|
1554
|
+
sync = subprocess.run(["uv", "sync"], env=os.environ.copy(), check=False)
|
|
1555
|
+
sys.exit(sync.returncode)
|
|
1286
1556
|
|
|
1287
1557
|
|
|
1288
1558
|
from steve_cli.ontology import ontology
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
from copy import deepcopy
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Literal
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
|
|
12
|
+
_EMAIL_RE = re.compile(r'^[^@\s]+@[^@\s]+\.[^@\s]+$')
|
|
13
|
+
|
|
14
|
+
_POLICY_VERSION = "v0"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _expand_schema(schema: str, sanitized_branch: str) -> str:
|
|
18
|
+
if schema == sanitized_branch or not schema:
|
|
19
|
+
return sanitized_branch
|
|
20
|
+
return schema
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _expand_rules(rules: list[dict[str, Any]], sanitized_branch: str, workspace: str = "") -> list[dict[str, Any]]:
|
|
24
|
+
expanded = []
|
|
25
|
+
for rule in rules:
|
|
26
|
+
rule = deepcopy(rule)
|
|
27
|
+
for entry in rule.get("tables", []):
|
|
28
|
+
if "schema" in entry:
|
|
29
|
+
entry["schema"] = _expand_schema(entry["schema"], sanitized_branch)
|
|
30
|
+
if workspace and "catalogs" not in entry:
|
|
31
|
+
tier = entry.get("tier", "bronze")
|
|
32
|
+
entry["catalogs"] = [f"{workspace}-{tier}"]
|
|
33
|
+
for entry in rule.get("columns", []):
|
|
34
|
+
if "schema" in entry:
|
|
35
|
+
entry["schema"] = _expand_schema(entry["schema"], sanitized_branch)
|
|
36
|
+
expanded.append(rule)
|
|
37
|
+
return expanded
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class PolicyClient:
|
|
41
|
+
def __init__(
|
|
42
|
+
self,
|
|
43
|
+
endpoint: str | None = None,
|
|
44
|
+
token: str | None = None,
|
|
45
|
+
workspace_id: str | None = None,
|
|
46
|
+
):
|
|
47
|
+
from steve_cli.auth import get_service_url, get_token
|
|
48
|
+
if endpoint is None:
|
|
49
|
+
endpoint = get_service_url("pcp", workspace_id) or "http://policy-control-plane:3010"
|
|
50
|
+
self.endpoint = endpoint.rstrip("/")
|
|
51
|
+
self.token = token or get_token(workspace_id) or ""
|
|
52
|
+
|
|
53
|
+
def __trpc(self, method: str, path: str, payload: dict | None = None) -> dict:
|
|
54
|
+
url = f"{self.endpoint}/api/trpc/{path}"
|
|
55
|
+
headers = {"Authorization": f"Bearer {self.token}"} if self.token else {}
|
|
56
|
+
if method == "query":
|
|
57
|
+
resp = httpx.get(url, params={"input": json.dumps(payload or {})}, headers=headers, timeout=10)
|
|
58
|
+
else:
|
|
59
|
+
resp = httpx.post(url, json=payload or {}, headers=headers, timeout=10)
|
|
60
|
+
resp.raise_for_status()
|
|
61
|
+
data = resp.json()
|
|
62
|
+
if "error" in data:
|
|
63
|
+
raise RuntimeError(f"tRPC error on {path}: {data['error']}")
|
|
64
|
+
result = data.get("result", {})
|
|
65
|
+
if isinstance(result, list):
|
|
66
|
+
return result
|
|
67
|
+
data_inner = result.get("data", result)
|
|
68
|
+
if isinstance(data_inner, list):
|
|
69
|
+
return data_inner
|
|
70
|
+
if isinstance(data_inner, dict):
|
|
71
|
+
return data_inner.get("json", data_inner)
|
|
72
|
+
return data_inner
|
|
73
|
+
|
|
74
|
+
def _trpc(self, method: str, path: str, payload: dict | None = None) -> dict:
|
|
75
|
+
try:
|
|
76
|
+
return self.__trpc(method, path, payload)
|
|
77
|
+
except httpx.ConnectError as e:
|
|
78
|
+
raise EnvironmentError(
|
|
79
|
+
f"Cannot reach Policy Control Plane at {self.endpoint} — "
|
|
80
|
+
f"run `steve login` to configure credentials"
|
|
81
|
+
) from e
|
|
82
|
+
|
|
83
|
+
def list_pdps(self) -> list[dict]:
|
|
84
|
+
return self._trpc("query", "pdp.list") or []
|
|
85
|
+
|
|
86
|
+
def list_workspaces(self) -> list[str]:
|
|
87
|
+
return self._trpc("query", "workspace.list") or []
|
|
88
|
+
|
|
89
|
+
def list_workspace_pdps(self, workspace: str) -> list[dict]:
|
|
90
|
+
return self._trpc("query", "workspace.listPdps", {"workspace": workspace}) or []
|
|
91
|
+
|
|
92
|
+
def delete_pdp(self, pdp_id: str) -> dict:
|
|
93
|
+
return self._trpc("mutation", "pdp.delete", {"id": pdp_id})
|
|
94
|
+
|
|
95
|
+
def delete_pdp_policy(self, policy_id: str) -> dict:
|
|
96
|
+
return self._trpc("mutation", "pdp.policy.delete", {"id": policy_id})
|
|
97
|
+
|
|
98
|
+
def register_pdp(self, name: str, endpoint: str, pdp_type: str = "opa", is_default: bool = True) -> dict:
|
|
99
|
+
return self._trpc("mutation", "pdp.register", {
|
|
100
|
+
"name": name,
|
|
101
|
+
"type": pdp_type,
|
|
102
|
+
"endpoint": endpoint,
|
|
103
|
+
"isDefault": is_default,
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
def attach_pdp(self, workspace: str, pdp_id: str) -> None:
|
|
107
|
+
self._trpc("mutation", "workspace.attachPdp", {"workspace": workspace, "pdpId": pdp_id})
|
|
108
|
+
|
|
109
|
+
def apply_policies(self, workspace: str, pdp_name: str, rules: list[dict[str, Any]]) -> dict:
|
|
110
|
+
return self._trpc("mutation", "workspace.applyPolicies", {
|
|
111
|
+
"workspace": workspace,
|
|
112
|
+
"fragments": [
|
|
113
|
+
{
|
|
114
|
+
"syntax": "dsl",
|
|
115
|
+
"target": pdp_name,
|
|
116
|
+
"content": {"rules": rules},
|
|
117
|
+
}
|
|
118
|
+
],
|
|
119
|
+
})
|
|
120
|
+
|
|
121
|
+
def _validate_against_trino(self, rules: list[dict[str, Any]], sanitized_branch: str) -> list[str]:
|
|
122
|
+
workspace_id = os.environ.get("WORKSPACE_ID")
|
|
123
|
+
workspace_name = os.environ.get("WORKSPACE_NAME", "")
|
|
124
|
+
|
|
125
|
+
seen: dict[str, list[str]] = {}
|
|
126
|
+
for rule in rules:
|
|
127
|
+
for entry in list(rule.get("tables", [])) + list(rule.get("columns", [])):
|
|
128
|
+
schema = entry.get("schema", "")
|
|
129
|
+
table = entry.get("table", "")
|
|
130
|
+
if schema and table:
|
|
131
|
+
seen.setdefault(schema, [])
|
|
132
|
+
if table not in seen[schema]:
|
|
133
|
+
seen[schema].append(table)
|
|
134
|
+
|
|
135
|
+
results: list[str] = []
|
|
136
|
+
for schema, tables in seen.items():
|
|
137
|
+
tier = "bronze"
|
|
138
|
+
ws_clean = workspace_name.lower().replace("-", "_") if workspace_name else ""
|
|
139
|
+
catalog = f"{workspace_name}-{tier}" if workspace_name else f"minio-{tier}"
|
|
140
|
+
|
|
141
|
+
try:
|
|
142
|
+
from steve_cli.storage.trino import TrinoStorage
|
|
143
|
+
storage = TrinoStorage(tier=tier, workspace=workspace_name or None)
|
|
144
|
+
existing = storage._execute(f'SHOW TABLES FROM "{storage.catalog}".{schema}')
|
|
145
|
+
existing_names = {r.get("Table", "") for r in existing}
|
|
146
|
+
catalog = storage.catalog
|
|
147
|
+
except Exception as exc:
|
|
148
|
+
results.append(f"⚠ cannot reach Trino to validate schema '{schema}': {exc}")
|
|
149
|
+
continue
|
|
150
|
+
|
|
151
|
+
for table in tables:
|
|
152
|
+
if table in existing_names:
|
|
153
|
+
results.append(f"✓ {catalog}.{schema}.{table}")
|
|
154
|
+
else:
|
|
155
|
+
results.append(
|
|
156
|
+
f"⚠ table '{table}' not found in '{catalog}.{schema}'"
|
|
157
|
+
+ (f" (found: {sorted(existing_names)})" if existing_names else " (schema empty or not found)")
|
|
158
|
+
)
|
|
159
|
+
return results
|
|
160
|
+
|
|
161
|
+
def apply_from_file(self, policies_file: Path | None = None, validate: Literal[None, "warn", "error"] = None) -> tuple[str, str]:
|
|
162
|
+
path = policies_file or Path.cwd() / "policies" / "access.yaml"
|
|
163
|
+
if not path.exists():
|
|
164
|
+
raise FileNotFoundError(f"Policies file not found: {path}")
|
|
165
|
+
|
|
166
|
+
import yaml
|
|
167
|
+
config: dict[str, Any] = yaml.safe_load(path.read_text())
|
|
168
|
+
workspace = config["workspace"]
|
|
169
|
+
pdp_cfg = config["pdp"]
|
|
170
|
+
rules = config.get("rules", [])
|
|
171
|
+
|
|
172
|
+
invalid_users = [r["user"] for r in rules if not _EMAIL_RE.match(r.get("user", ""))]
|
|
173
|
+
if invalid_users:
|
|
174
|
+
raise ValueError(
|
|
175
|
+
f"Invalid user(s) in rules — expected email addresses, got: {invalid_users}. "
|
|
176
|
+
f"Use group-based access for wildcard permissions instead."
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
from steve_cli.storage.branch import get_branch_prefix
|
|
180
|
+
sanitized = get_branch_prefix()
|
|
181
|
+
branch = sanitized
|
|
182
|
+
rules = _expand_rules(rules, sanitized, workspace)
|
|
183
|
+
|
|
184
|
+
pdp = self.register_pdp(
|
|
185
|
+
name=pdp_cfg["name"],
|
|
186
|
+
endpoint=pdp_cfg["endpoint"],
|
|
187
|
+
pdp_type=pdp_cfg.get("type", "opa"),
|
|
188
|
+
)
|
|
189
|
+
self.attach_pdp(workspace, pdp["id"])
|
|
190
|
+
result = self.apply_policies(workspace, pdp_cfg["name"], rules)
|
|
191
|
+
|
|
192
|
+
errors = result.get("errors", [])
|
|
193
|
+
if errors:
|
|
194
|
+
reasons = "; ".join(e["reason"] for e in errors)
|
|
195
|
+
raise RuntimeError(f"Failed to apply {len(errors)} policy fragment(s): {reasons}")
|
|
196
|
+
|
|
197
|
+
if validate is not None:
|
|
198
|
+
issues = self._validate_against_trino(rules, sanitized)
|
|
199
|
+
failed = [msg for msg in issues if msg.startswith("⚠")]
|
|
200
|
+
for msg in issues:
|
|
201
|
+
print(msg)
|
|
202
|
+
if failed and validate == "error":
|
|
203
|
+
raise RuntimeError(f"Validation failed: {len(failed)} issue(s) found")
|
|
204
|
+
|
|
205
|
+
return branch, sanitized
|
|
@@ -33,8 +33,24 @@ def get_branch_prefix() -> str:
|
|
|
33
33
|
branch = result.stdout.strip()
|
|
34
34
|
if branch and branch != "HEAD":
|
|
35
35
|
return _sanitize(branch)
|
|
36
|
+
if branch == "HEAD":
|
|
37
|
+
sha_result = subprocess.run(
|
|
38
|
+
["git", "rev-parse", "--short", "HEAD"],
|
|
39
|
+
capture_output=True,
|
|
40
|
+
text=True,
|
|
41
|
+
timeout=3,
|
|
42
|
+
)
|
|
43
|
+
if sha_result.returncode == 0 and sha_result.stdout.strip():
|
|
44
|
+
short_sha = sha_result.stdout.strip()
|
|
45
|
+
logger.warning(
|
|
46
|
+
"Detached HEAD: using commit SHA as branch prefix (sha_%s). "
|
|
47
|
+
"Set STEVE_BRANCH to override.",
|
|
48
|
+
short_sha,
|
|
49
|
+
)
|
|
50
|
+
return f"sha_{short_sha}"
|
|
36
51
|
except Exception as exc:
|
|
37
52
|
logger.debug("git branch detection failed: %s", exc)
|
|
38
53
|
|
|
39
|
-
|
|
40
|
-
|
|
54
|
+
raise RuntimeError(
|
|
55
|
+
"Could not detect git branch. Set STEVE_BRANCH=<branch-name> and retry."
|
|
56
|
+
)
|
|
@@ -15,13 +15,64 @@ from .branch import get_branch_prefix
|
|
|
15
15
|
logger = logging.getLogger(__name__)
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
def _derive_schema(
|
|
19
|
-
|
|
20
|
-
|
|
18
|
+
def _derive_schema(_tier: str, _workspace: str | None, branch: str) -> str:
|
|
19
|
+
return branch
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _patch_pyiceberg_get_fs_once() -> None:
|
|
23
|
+
"""Fix pyiceberg race: per-thread get_fs() causes repeated _s3() unregister+register_last
|
|
24
|
+
cycles, leaving a window where S3V4RestSigner is absent from before-sign.s3 handlers.
|
|
25
|
+
Replace get_fs() with an instance-level shared cache so _s3() runs only once per
|
|
26
|
+
FsspecFileIO instance (from the calling thread), eliminating the re-registration race."""
|
|
27
|
+
logger.warning(
|
|
28
|
+
"Applied pyiceberg get_fs monkey-patch (instance-level cache) to work around "
|
|
29
|
+
"S3V4RestSigner race condition. Remove once upstream fixes are released: "
|
|
30
|
+
"https://github.com/apache/iceberg-python/issues/3625 "
|
|
31
|
+
"https://github.com/apache/iceberg-python/issues/3896"
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
try:
|
|
35
|
+
import threading as _t
|
|
36
|
+
from pyiceberg.io.fsspec import FsspecFileIO
|
|
37
|
+
|
|
38
|
+
_caches: dict[int, dict] = {}
|
|
39
|
+
_lock = _t.Lock()
|
|
40
|
+
_orig = FsspecFileIO.get_fs
|
|
41
|
+
|
|
42
|
+
def _get_fs_shared(self, scheme: str, hostname=None):
|
|
43
|
+
fio_id = id(self)
|
|
44
|
+
key = (scheme, hostname)
|
|
45
|
+
cache = _caches.get(fio_id)
|
|
46
|
+
if cache is not None and key in cache:
|
|
47
|
+
return cache[key]
|
|
48
|
+
with _lock:
|
|
49
|
+
if fio_id not in _caches:
|
|
50
|
+
_caches[fio_id] = {}
|
|
51
|
+
cache = _caches[fio_id]
|
|
52
|
+
if key not in cache:
|
|
53
|
+
cache[key] = _orig(self, scheme, hostname)
|
|
54
|
+
return cache[key]
|
|
55
|
+
|
|
56
|
+
FsspecFileIO.get_fs = _get_fs_shared
|
|
57
|
+
|
|
58
|
+
except Exception as exc:
|
|
59
|
+
logger.warning("Could not patch pyiceberg FsspecFileIO.get_fs: %s", exc)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
_patch_pyiceberg_get_fs_once()
|
|
21
63
|
|
|
22
64
|
|
|
23
65
|
class TrinoStorage:
|
|
24
66
|
def __init__(self, tier: str = "bronze", workspace: str | None = None):
|
|
67
|
+
# aiobotocore 3.9+ defaults to STREAMING-UNSIGNED-PAYLOAD-TRAILER (flexible checksums)
|
|
68
|
+
# which Lakekeeper's remote signing proxy and MinIO don't support.
|
|
69
|
+
os.environ.setdefault('AWS_REQUEST_CHECKSUM_CALCULATION', 'when_required')
|
|
70
|
+
# Reduce keepalive from 12s to 3s: MinIO closes server-side connections faster than
|
|
71
|
+
# aiobotocore's default, causing stale pooled connections to return bare HTTP 403
|
|
72
|
+
# ("Forbidden", no S3 XML) on reuse. 3s ensures idle connections are evicted before
|
|
73
|
+
# the next put_bytes call reaches the S3 write phase.
|
|
74
|
+
import aiobotocore.httpsession
|
|
75
|
+
aiobotocore.httpsession.DEFAULT_KEEPALIVE_TIMEOUT = 3
|
|
25
76
|
from steve_cli.auth import get_service_url
|
|
26
77
|
self._branch = get_branch_prefix()
|
|
27
78
|
self._tier = tier.lower()
|
|
@@ -47,61 +98,39 @@ class TrinoStorage:
|
|
|
47
98
|
_lk_env = os.getenv("LAKEKEEPER_ENDPOINT")
|
|
48
99
|
_lk_svc = get_service_url("lakekeeper", self._workspace_id)
|
|
49
100
|
self._lakekeeper_endpoint = _lk_env.rstrip("/") if _lk_env else (_lk_svc.rstrip("/") + "/catalog" if _lk_svc else None)
|
|
101
|
+
self._lakekeeper_base = (self._lakekeeper_endpoint or "http://lakekeeper:8181/catalog").rstrip("/")
|
|
50
102
|
self._lakekeeper_warehouse = os.getenv("LAKEKEEPER_WAREHOUSE", default_catalog)
|
|
51
103
|
_iceberg_bucket = os.getenv(f"{tier.upper()}_BUCKET", "")
|
|
52
104
|
if _iceberg_bucket:
|
|
53
|
-
self._iceberg_location_prefix = f"s3://{_iceberg_bucket}/_iceberg/{self._branch}"
|
|
105
|
+
self._iceberg_location_prefix: str | None = f"s3://{_iceberg_bucket}/_iceberg/{self._branch}"
|
|
54
106
|
else:
|
|
55
|
-
|
|
56
|
-
f"TrinoStorage: missing bucket configuration for tier '{tier}' — "
|
|
57
|
-
f"{tier.upper()}_BUCKET is not set.\n"
|
|
58
|
-
f" Required env vars: {tier.upper()}_BUCKET, {tier.upper()}_ACCESS_KEY, {tier.upper()}_SECRET_KEY\n"
|
|
59
|
-
f" This workspace may only have read access to the '{tier}' tier. "
|
|
60
|
-
f"Write access requires provisioning credentials (run 'uv run steve setup env' or check workspace provisioning)."
|
|
61
|
-
)
|
|
62
|
-
self.__iceberg_catalog = None
|
|
107
|
+
self._iceberg_location_prefix = None
|
|
63
108
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
109
|
+
if os.getenv("STEVE_DIRECT_S3") == "1":
|
|
110
|
+
if not self._iceberg_location_prefix:
|
|
111
|
+
raise OSError(
|
|
112
|
+
f"TrinoStorage (STEVE_DIRECT_S3=1): missing bucket configuration for tier '{tier}' — "
|
|
113
|
+
f"{tier.upper()}_BUCKET is not set.\n"
|
|
114
|
+
f" Direct S3 mode requires explicit bucket credentials "
|
|
115
|
+
f"(run 'uv run steve setup env' or check workspace provisioning)."
|
|
116
|
+
)
|
|
67
117
|
from pyiceberg.catalog.rest import RestCatalog
|
|
68
118
|
from pyiceberg.io import load_file_io
|
|
69
119
|
from pyiceberg.table import Table
|
|
70
|
-
|
|
71
|
-
# WORKAROUND: Lakekeeper's remote S3 presigner is bypassed for external access.
|
|
72
|
-
#
|
|
73
|
-
# Lakekeeper returns `s3.signer` / `s3.signer.uri` in the table config, which
|
|
74
|
-
# pyiceberg uses to sign S3 requests via Lakekeeper's signing endpoint. This works
|
|
75
|
-
# inside the cluster but fails from a laptop because:
|
|
76
|
-
# 1. The signed URI Lakekeeper produces contains the cluster-internal hostname
|
|
77
|
-
# (e.g. http://minio:9000) which is unreachable externally.
|
|
78
|
-
# 2. Even after overriding the signer URI to the public lakekeeper subdomain,
|
|
79
|
-
# Lakekeeper's signer returns HTTP 500 for URIs with the public minio hostname
|
|
80
|
-
# because it only knows the internal minio endpoint.
|
|
81
|
-
#
|
|
82
|
-
# Current workaround: strip all signer config from the merged table properties and
|
|
83
|
-
# use direct S3 credentials (access-key + secret-key) with the public minio endpoint
|
|
84
|
-
# from ~/.steve/credentials.json. This bypasses Lakekeeper's access control for S3
|
|
85
|
-
# writes — credentials live on disk rather than being vended by the cluster.
|
|
86
|
-
#
|
|
87
|
-
# To revert: configure Lakekeeper with its public S3 endpoint so presigned URIs
|
|
88
|
-
# contain a resolvable hostname, then remove the merged.pop() calls below and
|
|
89
|
-
# restore `s3.signer.uri: lakekeeper_base` in s3_overrides.
|
|
90
|
-
tier = self._tier.upper()
|
|
91
|
-
lakekeeper_base = (self._lakekeeper_endpoint or "http://lakekeeper:8181/catalog").rstrip("/")
|
|
92
120
|
from steve_cli.auth import get_s3_endpoint as _get_s3_ep
|
|
121
|
+
_tier = tier.upper()
|
|
93
122
|
_s3_ep = _get_s3_ep(self._workspace_id) or os.getenv("S3_ENDPOINT", "http://minio:9000")
|
|
94
|
-
|
|
123
|
+
_s3_overrides = {
|
|
95
124
|
"s3.endpoint": _s3_ep,
|
|
96
|
-
"s3.access-key-id": os.getenv(f"{
|
|
97
|
-
"s3.secret-access-key": os.getenv(f"{
|
|
125
|
+
"s3.access-key-id": os.getenv(f"{_tier}_ACCESS_KEY") or os.getenv("AWS_ACCESS_KEY_ID", ""),
|
|
126
|
+
"s3.secret-access-key": os.getenv(f"{_tier}_SECRET_KEY") or os.getenv("AWS_SECRET_ACCESS_KEY", ""),
|
|
98
127
|
"s3.path-style-access": "true",
|
|
99
128
|
} if self._lakekeeper_endpoint else {}
|
|
129
|
+
_base = self._lakekeeper_base
|
|
100
130
|
|
|
101
131
|
class _PatchedRestCatalog(RestCatalog):
|
|
102
132
|
def _response_to_table(self_, identifier_tuple, table_response):
|
|
103
|
-
merged = {**table_response.metadata.properties, **table_response.config, **
|
|
104
|
-
# Strip remote-signer config — see WORKAROUND comment above.
|
|
133
|
+
merged = {**table_response.metadata.properties, **table_response.config, **_s3_overrides, "uri": _base}
|
|
105
134
|
merged.pop("s3.signer", None)
|
|
106
135
|
merged.pop("s3.signer.uri", None)
|
|
107
136
|
merged.pop("s3.signer.endpoint", None)
|
|
@@ -114,27 +143,40 @@ class TrinoStorage:
|
|
|
114
143
|
config=table_response.config,
|
|
115
144
|
)
|
|
116
145
|
|
|
146
|
+
self._catalog_cls = _PatchedRestCatalog
|
|
147
|
+
self._extra_catalog_kwargs: dict = {**_s3_overrides}
|
|
148
|
+
else:
|
|
149
|
+
self._catalog_cls = None
|
|
150
|
+
self._extra_catalog_kwargs = {}
|
|
151
|
+
|
|
152
|
+
self.__iceberg_catalog = None
|
|
153
|
+
|
|
154
|
+
@property
|
|
155
|
+
def _iceberg_catalog(self):
|
|
156
|
+
if self.__iceberg_catalog is None:
|
|
157
|
+
from pyiceberg.catalog.rest import RestCatalog
|
|
117
158
|
from steve_cli.auth import get_token
|
|
118
159
|
stp_token = get_token(self._workspace_id)
|
|
119
160
|
token_kwargs = {"token": stp_token} if stp_token else {}
|
|
161
|
+
catalog_cls = self._catalog_cls or RestCatalog
|
|
120
162
|
try:
|
|
121
|
-
catalog =
|
|
163
|
+
catalog = catalog_cls(
|
|
122
164
|
"lakekeeper",
|
|
123
|
-
uri=
|
|
165
|
+
uri=self._lakekeeper_base,
|
|
124
166
|
warehouse=self._lakekeeper_warehouse,
|
|
125
|
-
**
|
|
167
|
+
**self._extra_catalog_kwargs,
|
|
126
168
|
**token_kwargs,
|
|
127
169
|
)
|
|
128
170
|
except Exception as exc:
|
|
129
171
|
msg = str(exc)
|
|
130
172
|
if "NoSuchWarehouseException" in msg or "does not exist" in msg:
|
|
131
173
|
raise RuntimeError(
|
|
132
|
-
f"Lakekeeper warehouse '{self._lakekeeper_warehouse}' not found at {
|
|
174
|
+
f"Lakekeeper warehouse '{self._lakekeeper_warehouse}' not found at {self._lakekeeper_base}.\n"
|
|
133
175
|
f"The workspace may not have been provisioned yet — ensure the workspace was created in the data mesh before running this job."
|
|
134
176
|
) from None
|
|
135
177
|
raise
|
|
136
|
-
if self._lakekeeper_endpoint:
|
|
137
|
-
catalog.uri =
|
|
178
|
+
if self._catalog_cls and self._lakekeeper_endpoint:
|
|
179
|
+
catalog.uri = self._lakekeeper_base
|
|
138
180
|
self.__iceberg_catalog = catalog
|
|
139
181
|
return self.__iceberg_catalog
|
|
140
182
|
|
|
@@ -278,8 +320,7 @@ class TrinoStorage:
|
|
|
278
320
|
import datetime
|
|
279
321
|
|
|
280
322
|
from pyiceberg.exceptions import NamespaceAlreadyExistsError
|
|
281
|
-
|
|
282
|
-
namespace_props = {"location": self._iceberg_location_prefix}
|
|
323
|
+
namespace_props = {"location": self._iceberg_location_prefix} if self._iceberg_location_prefix else {}
|
|
283
324
|
created = False
|
|
284
325
|
try:
|
|
285
326
|
self._iceberg_catalog.create_namespace(self.schema, properties=namespace_props)
|
|
@@ -292,12 +333,12 @@ class TrinoStorage:
|
|
|
292
333
|
if "403" in msg or "Access Denied" in msg or "Forbidden" in msg:
|
|
293
334
|
raise OSError(
|
|
294
335
|
f"TrinoStorage: namespace '{self.schema}' creation denied at Lakekeeper "
|
|
295
|
-
f"[{
|
|
336
|
+
f"[{self._lakekeeper_base}] — check that the token has write access to warehouse "
|
|
296
337
|
f"'{self._lakekeeper_warehouse}'"
|
|
297
338
|
) from exc
|
|
298
339
|
raise OSError(
|
|
299
340
|
f"TrinoStorage: namespace '{self.schema}' creation failed at Lakekeeper "
|
|
300
|
-
f"[{
|
|
341
|
+
f"[{self._lakekeeper_base}] — {type(exc).__name__}: {exc}"
|
|
301
342
|
) from exc
|
|
302
343
|
|
|
303
344
|
if created:
|
|
@@ -317,7 +358,7 @@ class TrinoStorage:
|
|
|
317
358
|
"branch": [self._branch],
|
|
318
359
|
"tier": [self._tier],
|
|
319
360
|
"schema_name": [self.schema],
|
|
320
|
-
"iceberg_location": [self._iceberg_location_prefix],
|
|
361
|
+
"iceberg_location": [self._iceberg_location_prefix or ""],
|
|
321
362
|
"created_at": [datetime.datetime.now(datetime.timezone.utc).isoformat()],
|
|
322
363
|
}))
|
|
323
364
|
logger.info("Created _meta table in namespace %s", self.schema)
|
|
@@ -325,39 +366,101 @@ class TrinoStorage:
|
|
|
325
366
|
logger.warning("Could not create _meta table in %s: %s", self.schema, exc)
|
|
326
367
|
|
|
327
368
|
try:
|
|
369
|
+
_location_clause = f" WITH (location = '{self._iceberg_location_prefix}')" if self._iceberg_location_prefix else ""
|
|
328
370
|
self._execute(
|
|
329
|
-
f'CREATE SCHEMA IF NOT EXISTS "{self.catalog}".{self.schema}
|
|
330
|
-
f"WITH (location = '{self._iceberg_location_prefix}')"
|
|
371
|
+
f'CREATE SCHEMA IF NOT EXISTS "{self.catalog}".{self.schema}{_location_clause}'
|
|
331
372
|
)
|
|
332
373
|
logger.debug("Ensured Trino schema %s.%s", self.catalog, self.schema)
|
|
333
374
|
except Exception as exc:
|
|
334
375
|
logger.warning("Could not ensure Trino schema %s.%s: %s", self.catalog, self.schema, exc)
|
|
335
376
|
|
|
336
377
|
def put_bytes(self, data: bytes, path: str) -> None:
|
|
337
|
-
from pyiceberg.exceptions import NoSuchTableError
|
|
378
|
+
from pyiceberg.exceptions import NoSuchTableError, SignError
|
|
338
379
|
|
|
339
380
|
arrow_table = pq.read_table(io.BytesIO(data))
|
|
340
381
|
table_id = f"{self.schema}.{self._table_name(path)}"
|
|
341
382
|
catalog = self._iceberg_catalog
|
|
342
|
-
lakekeeper_base = self.
|
|
383
|
+
lakekeeper_base = self._lakekeeper_base
|
|
343
384
|
|
|
344
385
|
self._ensure_namespace()
|
|
345
386
|
|
|
346
387
|
def _append(iceberg_table, arrow_table, table_id):
|
|
388
|
+
if os.getenv("STEVE_TRINO_TRACE") == "1":
|
|
389
|
+
io_props = getattr(iceberg_table.io, "properties", {})
|
|
390
|
+
logger.warning(
|
|
391
|
+
"[trino] append: table=%s, location=%s, s3.endpoint=%s, signer.uri=%s",
|
|
392
|
+
table_id,
|
|
393
|
+
iceberg_table.location(),
|
|
394
|
+
io_props.get("s3.endpoint", "n/a"),
|
|
395
|
+
io_props.get("s3.signer.uri", "n/a"),
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
s3_client = None
|
|
399
|
+
if os.getenv("STEVE_TRINO_TRACE") == "1":
|
|
400
|
+
try:
|
|
401
|
+
import threading as _threading
|
|
402
|
+
s3_fs = iceberg_table.io._get_fs("s3")
|
|
403
|
+
s3_client = s3_fs.s3
|
|
404
|
+
|
|
405
|
+
def _trace_sign(request, **kwargs):
|
|
406
|
+
logger.warning("[trino] s3 SIGN %s req_id=%x (thread=%s)",
|
|
407
|
+
request.method, id(request),
|
|
408
|
+
_threading.current_thread().name)
|
|
409
|
+
|
|
410
|
+
def _trace_send(request, **kwargs):
|
|
411
|
+
hdrs = dict(getattr(request, "headers", {}))
|
|
412
|
+
auth = next((v for k, v in hdrs.items() if k.lower() == "authorization"), "<MISSING>")
|
|
413
|
+
logger.warning("[trino] s3.%s req_id=%x auth=%s (thread=%s)",
|
|
414
|
+
request.method, id(request),
|
|
415
|
+
auth[:24] if auth != "<MISSING>" else "<MISSING>",
|
|
416
|
+
_threading.current_thread().name)
|
|
417
|
+
|
|
418
|
+
def _trace_retry(response=None, operation=None, request_dict=None, **kwargs):
|
|
419
|
+
if not response:
|
|
420
|
+
return
|
|
421
|
+
status = getattr(response[0], "status_code", "?")
|
|
422
|
+
if status >= 300:
|
|
423
|
+
url = (request_dict or {}).get("url", "?")
|
|
424
|
+
logger.warning("[trino] s3 <- %s %s %s", operation or "?", status, url)
|
|
425
|
+
|
|
426
|
+
s3_client.meta.events.register("before-sign.s3", _trace_sign, unique_id="steve-trino-s3-sign")
|
|
427
|
+
s3_client.meta.events.register("before-send.s3", _trace_send, unique_id="steve-trino-s3-trace")
|
|
428
|
+
s3_client.meta.events.register("needs-retry.s3", _trace_retry, unique_id="steve-trino-s3-retry")
|
|
429
|
+
except Exception as hook_exc:
|
|
430
|
+
logger.warning("[trino] TRACE hook failed: %s", hook_exc)
|
|
431
|
+
s3_client = None
|
|
432
|
+
|
|
347
433
|
try:
|
|
348
434
|
iceberg_table.append(arrow_table)
|
|
435
|
+
except SignError as exc:
|
|
436
|
+
raise OSError(
|
|
437
|
+
f"TrinoStorage: Lakekeeper refused to sign write for '{table_id}' "
|
|
438
|
+
f"[signer={lakekeeper_base}, warehouse={self._lakekeeper_warehouse}] — {exc}\n"
|
|
439
|
+
f" Hint: the token may lack write permissions on this warehouse."
|
|
440
|
+
) from exc
|
|
349
441
|
except Exception as exc:
|
|
350
442
|
msg = str(exc)
|
|
351
443
|
if "403" in msg or "Access Denied" in msg or "Forbidden" in msg:
|
|
352
444
|
raise OSError(
|
|
353
|
-
f"TrinoStorage: write to '{table_id}' denied
|
|
354
|
-
f"
|
|
355
|
-
f"(
|
|
445
|
+
f"TrinoStorage: write to '{table_id}' denied by MinIO "
|
|
446
|
+
f"[location={iceberg_table.location()}, warehouse={self._lakekeeper_warehouse}] — "
|
|
447
|
+
f"{type(exc).__name__}: {exc}"
|
|
356
448
|
) from exc
|
|
357
449
|
raise OSError(
|
|
358
450
|
f"TrinoStorage: write to '{table_id}' failed via Lakekeeper [{lakekeeper_base}] — "
|
|
359
451
|
f"{type(exc).__name__}: {exc}"
|
|
360
452
|
) from exc
|
|
453
|
+
finally:
|
|
454
|
+
if s3_client is not None:
|
|
455
|
+
for event, uid in (
|
|
456
|
+
("before-sign.s3", "steve-trino-s3-sign"),
|
|
457
|
+
("before-send.s3", "steve-trino-s3-trace"),
|
|
458
|
+
("needs-retry.s3", "steve-trino-s3-retry"),
|
|
459
|
+
):
|
|
460
|
+
try:
|
|
461
|
+
s3_client.meta.events.unregister(event, unique_id=uid)
|
|
462
|
+
except Exception:
|
|
463
|
+
pass
|
|
361
464
|
|
|
362
465
|
try:
|
|
363
466
|
iceberg_table = catalog.load_table(table_id)
|
|
@@ -400,13 +503,19 @@ class TrinoStorage:
|
|
|
400
503
|
iceberg_schema = Schema(*fields)
|
|
401
504
|
try:
|
|
402
505
|
iceberg_table = catalog.create_table(table_id, schema=iceberg_schema)
|
|
506
|
+
except SignError as exc:
|
|
507
|
+
raise OSError(
|
|
508
|
+
f"TrinoStorage: Lakekeeper refused to sign table creation for '{table_id}' "
|
|
509
|
+
f"[signer={lakekeeper_base}, warehouse={self._lakekeeper_warehouse}] — {exc}\n"
|
|
510
|
+
f" Hint: the token may lack write permissions on this warehouse."
|
|
511
|
+
) from exc
|
|
403
512
|
except Exception as exc:
|
|
404
513
|
msg = str(exc)
|
|
405
514
|
if "403" in msg or "Access Denied" in msg or "Forbidden" in msg:
|
|
406
515
|
raise OSError(
|
|
407
|
-
f"TrinoStorage: create table '{table_id}' denied
|
|
408
|
-
f"
|
|
409
|
-
f"(
|
|
516
|
+
f"TrinoStorage: create table '{table_id}' denied by MinIO "
|
|
517
|
+
f"[location={self._iceberg_location_prefix or '<lakekeeper-managed>'}, warehouse={self._lakekeeper_warehouse}] — "
|
|
518
|
+
f"{type(exc).__name__}: {exc}"
|
|
410
519
|
) from exc
|
|
411
520
|
raise OSError(
|
|
412
521
|
f"TrinoStorage: create table '{table_id}' failed via Lakekeeper [{lakekeeper_base}] — "
|
|
@@ -423,7 +532,7 @@ class TrinoStorage:
|
|
|
423
532
|
logger.warning(
|
|
424
533
|
"[trino] put_bytes: wrote %d rows → %s.%s (warehouse=%s, location=%s%s)",
|
|
425
534
|
len(arrow_table), self.catalog, table_id,
|
|
426
|
-
self._lakekeeper_warehouse, self._iceberg_location_prefix,
|
|
535
|
+
self._lakekeeper_warehouse, self._iceberg_location_prefix or "<lakekeeper-managed>",
|
|
427
536
|
f", snapshot={snapshot.snapshot_id}" if snapshot else "",
|
|
428
537
|
)
|
|
429
538
|
logger.info("Written %d rows to %s", len(arrow_table), table_id)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: steve-cli
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.1.2
|
|
4
4
|
Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
|
|
5
5
|
Author: Frank
|
|
6
6
|
License: MIT
|
|
@@ -49,7 +49,7 @@ Provides-Extra: trino
|
|
|
49
49
|
Requires-Dist: trino>=0.330; extra == "trino"
|
|
50
50
|
Requires-Dist: pyarrow>=17.0.0; extra == "trino"
|
|
51
51
|
Requires-Dist: requests>=2.31.0; extra == "trino"
|
|
52
|
-
Requires-Dist: pyiceberg>=0.
|
|
52
|
+
Requires-Dist: pyiceberg>=0.12.0; extra == "trino"
|
|
53
53
|
Requires-Dist: s3fs>=2024.1.0; extra == "trino"
|
|
54
54
|
Provides-Extra: visidata
|
|
55
55
|
Requires-Dist: visidata>=3.0; extra == "visidata"
|
|
@@ -1,93 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import json
|
|
4
|
-
from pathlib import Path
|
|
5
|
-
from typing import Any
|
|
6
|
-
|
|
7
|
-
import httpx
|
|
8
|
-
|
|
9
|
-
_POLICY_VERSION = "v0"
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
class PolicyClient:
|
|
13
|
-
def __init__(
|
|
14
|
-
self,
|
|
15
|
-
endpoint: str | None = None,
|
|
16
|
-
token: str | None = None,
|
|
17
|
-
workspace_id: str | None = None,
|
|
18
|
-
):
|
|
19
|
-
from steve_cli.auth import get_service_url, get_token
|
|
20
|
-
if endpoint is None:
|
|
21
|
-
endpoint = get_service_url("pcp", workspace_id) or "http://policy-control-plane:3010"
|
|
22
|
-
self.endpoint = endpoint.rstrip("/")
|
|
23
|
-
self.token = token or get_token(workspace_id) or ""
|
|
24
|
-
|
|
25
|
-
def __trpc(self, method: str, path: str, payload: dict | None = None) -> dict:
|
|
26
|
-
url = f"{self.endpoint}/api/trpc/{path}"
|
|
27
|
-
headers = {"Authorization": f"Bearer {self.token}"} if self.token else {}
|
|
28
|
-
if method == "query":
|
|
29
|
-
resp = httpx.get(url, params={"input": json.dumps(payload or {})}, headers=headers, timeout=10)
|
|
30
|
-
else:
|
|
31
|
-
resp = httpx.post(url, json=payload or {}, headers=headers, timeout=10)
|
|
32
|
-
resp.raise_for_status()
|
|
33
|
-
data = resp.json()
|
|
34
|
-
if "error" in data:
|
|
35
|
-
raise RuntimeError(f"tRPC error on {path}: {data['error']}")
|
|
36
|
-
result = data.get("result", {})
|
|
37
|
-
return result.get("data", {}).get("json", result.get("data", result))
|
|
38
|
-
|
|
39
|
-
def _trpc(self, method: str, path: str, payload: dict | None = None) -> dict:
|
|
40
|
-
try:
|
|
41
|
-
return self.__trpc(method, path, payload)
|
|
42
|
-
except httpx.ConnectError as e:
|
|
43
|
-
raise EnvironmentError(
|
|
44
|
-
f"Cannot reach Policy Control Plane at {self.endpoint} — "
|
|
45
|
-
f"run `steve login` to configure credentials"
|
|
46
|
-
) from e
|
|
47
|
-
|
|
48
|
-
def register_pdp(self, name: str, endpoint: str, pdp_type: str = "opa", is_default: bool = True) -> dict:
|
|
49
|
-
return self._trpc("mutation", "pdp.register", {
|
|
50
|
-
"name": name,
|
|
51
|
-
"type": pdp_type,
|
|
52
|
-
"endpoint": endpoint,
|
|
53
|
-
"isDefault": is_default,
|
|
54
|
-
})
|
|
55
|
-
|
|
56
|
-
def attach_pdp(self, workspace: str, pdp_id: str) -> None:
|
|
57
|
-
self._trpc("mutation", "workspace.attachPdp", {"workspace": workspace, "pdpId": pdp_id})
|
|
58
|
-
|
|
59
|
-
def apply_policies(self, workspace: str, pdp_name: str, rules: list[dict[str, Any]]) -> dict:
|
|
60
|
-
return self._trpc("mutation", "workspace.applyPolicies", {
|
|
61
|
-
"workspace": workspace,
|
|
62
|
-
"fragments": [
|
|
63
|
-
{
|
|
64
|
-
"syntax": "dsl",
|
|
65
|
-
"target": pdp_name,
|
|
66
|
-
"content": {"rules": rules},
|
|
67
|
-
}
|
|
68
|
-
],
|
|
69
|
-
})
|
|
70
|
-
|
|
71
|
-
def apply_from_file(self, policies_file: Path | None = None) -> None:
|
|
72
|
-
path = policies_file or Path.cwd() / "policies" / "access.yaml"
|
|
73
|
-
if not path.exists():
|
|
74
|
-
raise FileNotFoundError(f"Policies file not found: {path}")
|
|
75
|
-
|
|
76
|
-
import yaml
|
|
77
|
-
config: dict[str, Any] = yaml.safe_load(path.read_text())
|
|
78
|
-
workspace = config["workspace"]
|
|
79
|
-
pdp_cfg = config["pdp"]
|
|
80
|
-
rules = config.get("rules", [])
|
|
81
|
-
|
|
82
|
-
pdp = self.register_pdp(
|
|
83
|
-
name=pdp_cfg["name"],
|
|
84
|
-
endpoint=pdp_cfg["endpoint"],
|
|
85
|
-
pdp_type=pdp_cfg.get("type", "opa"),
|
|
86
|
-
)
|
|
87
|
-
self.attach_pdp(workspace, pdp["id"])
|
|
88
|
-
result = self.apply_policies(workspace, pdp_cfg["name"], rules)
|
|
89
|
-
|
|
90
|
-
errors = result.get("errors", [])
|
|
91
|
-
if errors:
|
|
92
|
-
reasons = "; ".join(e["reason"] for e in errors)
|
|
93
|
-
raise RuntimeError(f"Failed to apply {len(errors)} policy fragment(s): {reasons}")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|