steve-cli 1.0.3__tar.gz → 1.1.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. {steve_cli-1.0.3 → steve_cli-1.1.2}/PKG-INFO +2 -2
  2. {steve_cli-1.0.3 → steve_cli-1.1.2}/pyproject.toml +2 -2
  3. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/auth.py +2 -0
  4. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/cli.py +277 -7
  5. steve_cli-1.1.2/steve_cli/policies/client.py +205 -0
  6. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/branch.py +18 -2
  7. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/trino.py +173 -64
  8. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli.egg-info/PKG-INFO +2 -2
  9. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli.egg-info/requires.txt +1 -1
  10. steve_cli-1.0.3/steve_cli/policies/client.py +0 -93
  11. {steve_cli-1.0.3 → steve_cli-1.1.2}/README.md +0 -0
  12. {steve_cli-1.0.3 → steve_cli-1.1.2}/setup.cfg +0 -0
  13. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/__init__.py +0 -0
  14. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/decorators/__init__.py +0 -0
  15. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/decorators/lineage_job.py +0 -0
  16. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/__init__.py +0 -0
  17. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/adapters/__init__.py +0 -0
  18. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/adapters/composite.py +0 -0
  19. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/adapters/dataproductregistry.py +0 -0
  20. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/adapters/logging.py +0 -0
  21. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/adapters/null.py +0 -0
  22. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/adapters/openlineage.py +0 -0
  23. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/collector.py +0 -0
  24. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/port.py +0 -0
  25. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/registry.py +0 -0
  26. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/lineage/storage.py +0 -0
  27. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/ontology.py +0 -0
  28. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/policies/__init__.py +0 -0
  29. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/__init__.py +0 -0
  30. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/__init__.py +0 -0
  31. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/__init__.py +0 -0
  32. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/csv.py +0 -0
  33. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/excel.py +0 -0
  34. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/generic.py +0 -0
  35. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/json.py +0 -0
  36. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/parquet.py +0 -0
  37. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/port.py +0 -0
  38. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/metadata/registry.py +0 -0
  39. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/parquet.py +0 -0
  40. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/protocol.py +0 -0
  41. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage/s3.py +0 -0
  42. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/storage.py +0 -0
  43. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/validation/__init__.py +0 -0
  44. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/validation/adapters/__init__.py +0 -0
  45. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/validation/adapters/great_expectations.py +0 -0
  46. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/validation/adapters/null.py +0 -0
  47. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/validation/adapters/validoopsie.py +0 -0
  48. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/validation/port.py +0 -0
  49. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/validation/registry.py +0 -0
  50. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli/vkg.py +0 -0
  51. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli.egg-info/SOURCES.txt +0 -0
  52. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli.egg-info/dependency_links.txt +0 -0
  53. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli.egg-info/entry_points.txt +0 -0
  54. {steve_cli-1.0.3 → steve_cli-1.1.2}/steve_cli.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: steve-cli
3
- Version: 1.0.3
3
+ Version: 1.1.2
4
4
  Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
5
5
  Author: Frank
6
6
  License: MIT
@@ -49,7 +49,7 @@ Provides-Extra: trino
49
49
  Requires-Dist: trino>=0.330; extra == "trino"
50
50
  Requires-Dist: pyarrow>=17.0.0; extra == "trino"
51
51
  Requires-Dist: requests>=2.31.0; extra == "trino"
52
- Requires-Dist: pyiceberg>=0.7.0; extra == "trino"
52
+ Requires-Dist: pyiceberg>=0.12.0; extra == "trino"
53
53
  Requires-Dist: s3fs>=2024.1.0; extra == "trino"
54
54
  Provides-Extra: visidata
55
55
  Requires-Dist: visidata>=3.0; extra == "visidata"
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
5
5
 
6
6
  [project]
7
7
  name = "steve-cli"
8
- version = "1.0.3"
8
+ version = "1.1.2"
9
9
  description = "A simple CLI tool to run jobs from jobs.yaml with proper environment setup"
10
10
  readme = "README.md"
11
11
  license = {text = "MIT"}
@@ -65,7 +65,7 @@ trino = [
65
65
  "trino>=0.330",
66
66
  "pyarrow>=17.0.0",
67
67
  "requests>=2.31.0",
68
- "pyiceberg>=0.7.0",
68
+ "pyiceberg>=0.12.0",
69
69
  "s3fs>=2024.1.0",
70
70
  ]
71
71
  visidata = [
@@ -74,6 +74,8 @@ def get_s3_endpoint(workspace_id: Optional[str] = None) -> Optional[str]:
74
74
 
75
75
 
76
76
  def get_service_url(service: str, workspace_id: Optional[str] = None) -> Optional[str]:
77
+ import os
78
+ workspace_id = workspace_id or os.environ.get("WORKSPACE_ID")
77
79
  entry = _get_workspace_entry(workspace_id)
78
80
  base_url = entry.get("base_url")
79
81
  if not base_url or not workspace_id:
@@ -940,7 +940,9 @@ def policies():
940
940
  @policies.command("apply")
941
941
  @click.option('--file', '-f', type=click.Path(path_type=Path),
942
942
  help='Path to policies YAML file (default: ./policies/access.yaml)')
943
- def policies_apply(file: Path | None):
943
+ @click.option('--validate', type=click.Choice(["warn", "error"]), default=None,
944
+ help='Validate schema/table references against Trino after applying (warn: print issues, error: fail if any found)')
945
+ def policies_apply(file: Path | None, validate: str | None):
944
946
  """Apply access policies from a YAML file to the Policy Control Plane."""
945
947
  from dotenv import load_dotenv
946
948
  load_dotenv(".env", override=False)
@@ -948,7 +950,8 @@ def policies_apply(file: Path | None):
948
950
  from steve_cli.policies import PolicyClient
949
951
  policies_file = Path(file) if file else None
950
952
  try:
951
- PolicyClient().apply_from_file(policies_file)
953
+ branch, sanitized = PolicyClient().apply_from_file(policies_file, validate=validate)
954
+ click.secho(f"Branch: {branch} → schema suffix __{sanitized}", fg="cyan")
952
955
  click.secho("Policies applied successfully.", fg="green")
953
956
  except FileNotFoundError as e:
954
957
  click.secho(str(e), fg="red", err=True)
@@ -958,6 +961,262 @@ def policies_apply(file: Path | None):
958
961
  sys.exit(1)
959
962
 
960
963
 
964
+ @policies.command("ls")
965
+ def policies_ls():
966
+ """List all registered PDPs and their attached workspaces."""
967
+ from dotenv import load_dotenv
968
+ load_dotenv(".env", override=False)
969
+ from steve_cli.policies import PolicyClient
970
+ try:
971
+ pdps = PolicyClient().list_pdps()
972
+ except Exception as e:
973
+ click.secho(f"ERROR: {e}", fg="red", bold=True, err=True)
974
+ sys.exit(1)
975
+ if not pdps:
976
+ click.echo("No PDPs registered.")
977
+ return
978
+ for pdp in pdps:
979
+ workspaces = [w.get("workspace", "") for w in pdp.get("workspaces", [])]
980
+ policies_count = len(pdp.get("policies", []))
981
+ click.echo(
982
+ f" {click.style(pdp['name'], fg='blue', bold=True)}"
983
+ f" {click.style(pdp['type'], fg='cyan')}"
984
+ f" {click.style(pdp['endpoint'], fg='bright_black')}"
985
+ )
986
+ if workspaces:
987
+ click.echo(f" workspaces: {', '.join(workspaces)}")
988
+ if pdp.get("policies"):
989
+ names = [p.get("name", p.get("id", "?")) for p in pdp["policies"]]
990
+ click.echo(f" policies: {', '.join(names)}")
991
+ else:
992
+ click.echo(f" policies: (none)")
993
+
994
+
995
+ @policies.command("show")
996
+ @click.argument("pdp_name", required=False, default=None)
997
+ def policies_show(pdp_name: str | None):
998
+ """Show full policy content for a PDP (interactive if name omitted)."""
999
+ from dotenv import load_dotenv
1000
+ load_dotenv(".env", override=False)
1001
+ from steve_cli.policies import PolicyClient
1002
+ try:
1003
+ pdps = PolicyClient().list_pdps()
1004
+ except Exception as e:
1005
+ click.secho(f"ERROR: {e}", fg="red", bold=True, err=True)
1006
+ sys.exit(1)
1007
+ if not pdps:
1008
+ click.echo("No PDPs registered.")
1009
+ return
1010
+ if pdp_name is None:
1011
+ pdp_name = questionary.select(
1012
+ "Select a PDP:",
1013
+ choices=[p["name"] for p in pdps],
1014
+ ).ask()
1015
+ if pdp_name is None:
1016
+ sys.exit(0)
1017
+ pdp = next((p for p in pdps if p["name"] == pdp_name), None)
1018
+ if not pdp:
1019
+ click.secho(f"PDP '{pdp_name}' not found.", fg="red", err=True)
1020
+ sys.exit(1)
1021
+ click.echo(f"\n{click.style(pdp['name'], fg='blue', bold=True)} {pdp['type']} {pdp['endpoint']}")
1022
+ import json as _json
1023
+ policies_list = pdp.get("policies", [])
1024
+ fragments_list = pdp.get("fragments", [])
1025
+ if not policies_list and not fragments_list:
1026
+ click.echo(" (no policies)")
1027
+ return
1028
+ for pol in policies_list:
1029
+ click.echo(f"\n {click.style(pol.get('name', pol.get('id', '?')), fg='cyan')} {click.style('rego', fg='bright_black')}")
1030
+ content = pol.get("content", pol.get("rawContent", ""))
1031
+ if isinstance(content, dict):
1032
+ content = _json.dumps(content, indent=2)
1033
+ for line in str(content).splitlines():
1034
+ click.echo(f" {line}")
1035
+ for frag in fragments_list:
1036
+ updated = frag.get("updatedAt", frag.get("createdAt", ""))
1037
+ click.echo(f"\n {click.style(frag.get('workspace', '?'), fg='cyan')} {click.style(frag.get('syntax', '?'), fg='magenta')} {click.style(updated, fg='bright_black')}")
1038
+ content = frag.get("content", {})
1039
+ for line in _json.dumps(content, indent=2).splitlines():
1040
+ click.echo(f" {line}")
1041
+
1042
+
1043
+ @policies.command("diff")
1044
+ @click.option('--file', '-f', type=click.Path(path_type=Path),
1045
+ help='Path to policies YAML file (default: ./policies/access.yaml)')
1046
+ @click.argument("pdp_name", required=False, default=None)
1047
+ def policies_diff(file: Path | None, pdp_name: str | None):
1048
+ """Diff local access.yaml (expanded) against what is applied on a PDP."""
1049
+ from dotenv import load_dotenv
1050
+ load_dotenv(".env", override=False)
1051
+ import json as _json
1052
+ import difflib
1053
+ import yaml
1054
+ from steve_cli.policies import PolicyClient
1055
+ from steve_cli.storage.branch import get_branch_prefix
1056
+
1057
+ path = Path(file) if file else Path.cwd() / "policies" / "access.yaml"
1058
+ if not path.exists():
1059
+ click.secho(f"Policies file not found: {path}", fg="red", err=True)
1060
+ sys.exit(1)
1061
+
1062
+ config = yaml.safe_load(path.read_text())
1063
+ local_rules = config.get("rules", [])
1064
+ sanitized = get_branch_prefix()
1065
+ from steve_cli.policies.client import _expand_rules
1066
+ expanded = _expand_rules(local_rules, sanitized)
1067
+ local_text = _json.dumps({"rules": expanded}, indent=2)
1068
+
1069
+ try:
1070
+ client = PolicyClient()
1071
+ pdps = client.list_pdps()
1072
+ except Exception as e:
1073
+ click.secho(f"ERROR: {e}", fg="red", bold=True, err=True)
1074
+ sys.exit(1)
1075
+
1076
+ if not pdps:
1077
+ click.echo("No PDPs registered.")
1078
+ return
1079
+
1080
+ if pdp_name is None:
1081
+ pdp_cfg = config.get("pdp", {})
1082
+ pdp_name = pdp_cfg.get("name")
1083
+ if pdp_name not in [p["name"] for p in pdps]:
1084
+ pdp_name = questionary.select(
1085
+ "Select a PDP to diff against:",
1086
+ choices=[p["name"] for p in pdps],
1087
+ ).ask()
1088
+ if pdp_name is None:
1089
+ sys.exit(0)
1090
+
1091
+ pdp = next((p for p in pdps if p["name"] == pdp_name), None)
1092
+ if not pdp:
1093
+ click.secho(f"PDP '{pdp_name}' not found.", fg="red", err=True)
1094
+ sys.exit(1)
1095
+
1096
+ remote_rules = []
1097
+ for frag in pdp.get("fragments", []):
1098
+ content = frag.get("content", {})
1099
+ if isinstance(content, dict) and "rules" in content:
1100
+ remote_rules.extend(content["rules"])
1101
+ for pol in pdp.get("policies", []):
1102
+ content = pol.get("content", {})
1103
+ if isinstance(content, dict) and "rules" in content:
1104
+ remote_rules.extend(content["rules"])
1105
+ remote_text = _json.dumps({"rules": remote_rules}, indent=2)
1106
+
1107
+ diff = list(difflib.unified_diff(
1108
+ remote_text.splitlines(),
1109
+ local_text.splitlines(),
1110
+ fromfile=f"remote/{pdp_name}",
1111
+ tofile=f"local/{path.name} (branch: {sanitized})",
1112
+ lineterm="",
1113
+ ))
1114
+
1115
+ if not diff:
1116
+ click.secho("No differences.", fg="green")
1117
+ return
1118
+
1119
+ for line in diff:
1120
+ if line.startswith("+"):
1121
+ click.secho(line, fg="green")
1122
+ elif line.startswith("-"):
1123
+ click.secho(line, fg="red")
1124
+ else:
1125
+ click.echo(line)
1126
+
1127
+
1128
+ @policies.command("rm")
1129
+ @click.argument("pdp_name", required=False, default=None)
1130
+ @click.option("--yes", "-y", is_flag=True, default=False, help="Skip confirmation prompt.")
1131
+ def policies_rm(pdp_name: str | None, yes: bool):
1132
+ """Delete a registered PDP (interactive if name omitted)."""
1133
+ from dotenv import load_dotenv
1134
+ load_dotenv(".env", override=False)
1135
+ from steve_cli.policies import PolicyClient
1136
+ try:
1137
+ client = PolicyClient()
1138
+ pdps = client.list_pdps()
1139
+ except Exception as e:
1140
+ click.secho(f"ERROR: {e}", fg="red", bold=True, err=True)
1141
+ sys.exit(1)
1142
+ if not pdps:
1143
+ click.echo("No PDPs registered.")
1144
+ return
1145
+ if pdp_name is None:
1146
+ pdp_name = questionary.select(
1147
+ "Select a PDP to delete:",
1148
+ choices=[p["name"] for p in pdps],
1149
+ ).ask()
1150
+ if pdp_name is None:
1151
+ sys.exit(0)
1152
+ pdp = next((p for p in pdps if p["name"] == pdp_name), None)
1153
+ if not pdp:
1154
+ click.secho(f"PDP '{pdp_name}' not found.", fg="red", err=True)
1155
+ sys.exit(1)
1156
+ if not yes:
1157
+ if not questionary.confirm(f"Delete PDP '{pdp_name}'?", default=False).ask():
1158
+ click.echo("Aborted.")
1159
+ return
1160
+ try:
1161
+ client.delete_pdp(pdp["id"])
1162
+ click.secho(f"Deleted PDP '{pdp_name}'.", fg="green")
1163
+ except Exception as e:
1164
+ click.secho(f"ERROR: {e}", fg="red", bold=True, err=True)
1165
+ sys.exit(1)
1166
+
1167
+
1168
+ @policies.command("policy-rm")
1169
+ @click.argument("pdp_name", required=False, default=None)
1170
+ @click.option("--yes", "-y", is_flag=True, default=False, help="Skip confirmation prompt.")
1171
+ def policies_policy_rm(pdp_name: str | None, yes: bool):
1172
+ """Delete a raw rego policy from a PDP (interactive if name omitted)."""
1173
+ from dotenv import load_dotenv
1174
+ load_dotenv(".env", override=False)
1175
+ from steve_cli.policies import PolicyClient
1176
+ try:
1177
+ client = PolicyClient()
1178
+ pdps = client.list_pdps()
1179
+ except Exception as e:
1180
+ click.secho(f"ERROR: {e}", fg="red", bold=True, err=True)
1181
+ sys.exit(1)
1182
+ if not pdps:
1183
+ click.echo("No PDPs registered.")
1184
+ return
1185
+ if pdp_name is None:
1186
+ pdp_name = questionary.select(
1187
+ "Select a PDP:",
1188
+ choices=[p["name"] for p in pdps],
1189
+ ).ask()
1190
+ if pdp_name is None:
1191
+ sys.exit(0)
1192
+ pdp = next((p for p in pdps if p["name"] == pdp_name), None)
1193
+ if not pdp:
1194
+ click.secho(f"PDP '{pdp_name}' not found.", fg="red", err=True)
1195
+ sys.exit(1)
1196
+ policies_list = pdp.get("policies", [])
1197
+ if not policies_list:
1198
+ click.echo(f"No policies on PDP '{pdp_name}'.")
1199
+ return
1200
+ choices = [p.get("name", p.get("id", "?")) for p in policies_list]
1201
+ policy_name = questionary.select(
1202
+ "Select a policy to delete:",
1203
+ choices=choices,
1204
+ ).ask()
1205
+ if policy_name is None:
1206
+ sys.exit(0)
1207
+ policy = next((p for p in policies_list if p.get("name", p.get("id")) == policy_name), None)
1208
+ if not yes:
1209
+ if not questionary.confirm(f"Delete policy '{policy_name}' from PDP '{pdp_name}'?", default=False).ask():
1210
+ click.echo("Aborted.")
1211
+ return
1212
+ try:
1213
+ client.delete_pdp_policy(policy["id"])
1214
+ click.secho(f"Deleted policy '{policy_name}' from PDP '{pdp_name}'.", fg="green")
1215
+ except Exception as e:
1216
+ click.secho(f"ERROR: {e}", fg="red", bold=True, err=True)
1217
+ sys.exit(1)
1218
+
1219
+
961
1220
  @main.command("login")
962
1221
  @click.option("--token", default=None, help="Service principal token (stp_...) to save")
963
1222
  @click.option("--workspace", "workspace_id", default=None, help="Workspace ID (defaults to WORKSPACE_ID from .env)")
@@ -1112,6 +1371,7 @@ def status(env_file: tuple, no_check: bool):
1112
1371
  expires_at = entry.get("expires_at", "")
1113
1372
  if token:
1114
1373
  expiry_warn = False
1374
+ token_expired = False
1115
1375
  expiry_str = ", expires never" if not expires_at else ""
1116
1376
  if expires_at:
1117
1377
  try:
@@ -1129,6 +1389,7 @@ def status(env_file: tuple, no_check: bool):
1129
1389
  ago = f"{s // 86400}d ago"
1130
1390
  expiry_str = f", expired {ago}"
1131
1391
  expiry_warn = True
1392
+ token_expired = True
1132
1393
  elif total_seconds < 3600:
1133
1394
  expiry_str = f", expires in {total_seconds // 60}m ⚠"
1134
1395
  expiry_warn = True
@@ -1163,12 +1424,18 @@ def status(env_file: tuple, no_check: bool):
1163
1424
  )
1164
1425
  with urllib.request.urlopen(_req, timeout=4) as _r:
1165
1426
  _r.read()
1166
- _row("Auth", f"✓ token valid (stp_***{token[-4:]}{expiry_str})", "green" if not expiry_warn else "yellow")
1427
+ if token_expired:
1428
+ _row("Auth", f"✗ token expired (stp_***{token[-4:]}{expiry_str}) — run: steve login", "red")
1429
+ else:
1430
+ _row("Auth", f"✓ token valid (stp_***{token[-4:]}{expiry_str})", "green" if not expiry_warn else "yellow")
1167
1431
  except urllib.error.HTTPError as _e:
1168
1432
  if _e.code == 401:
1169
1433
  _row("Auth", f"✗ token invalid/revoked (stp_***{token[-4:]}{expiry_str})", "red")
1170
1434
  else:
1171
- _row("Auth", f"✓ token valid (stp_***{token[-4:]}{expiry_str})", "green" if not expiry_warn else "yellow")
1435
+ if token_expired:
1436
+ _row("Auth", f"✗ token expired (stp_***{token[-4:]}{expiry_str}) — run: steve login", "red")
1437
+ else:
1438
+ _row("Auth", f"✓ token valid (stp_***{token[-4:]}{expiry_str})", "green" if not expiry_warn else "yellow")
1172
1439
  except Exception as _e:
1173
1440
  _row("Auth", f"{token_display} (validation failed: {_e})", token_color)
1174
1441
  else:
@@ -1212,8 +1479,7 @@ def status(env_file: tuple, no_check: bool):
1212
1479
  if trino_schema_explicit:
1213
1480
  schema = trino_schema_explicit
1214
1481
  elif resolved_workspace:
1215
- ws_clean = resolved_workspace.lower().replace("-", "_")
1216
- schema = f"{ws_clean}_{tier}__{branch}"
1482
+ schema = branch
1217
1483
  else:
1218
1484
  schema = f"(workspace not set — set WORKSPACE_NAME)"
1219
1485
  _row(f"Schema {tier}", schema, "cyan" if resolved_workspace or trino_schema_explicit else "yellow")
@@ -1282,7 +1548,11 @@ def upgrade():
1282
1548
  env=os.environ.copy(),
1283
1549
  check=False,
1284
1550
  )
1285
- sys.exit(result.returncode)
1551
+ if result.returncode != 0:
1552
+ sys.exit(result.returncode)
1553
+ click.echo("🔒 Syncing lockfile...")
1554
+ sync = subprocess.run(["uv", "sync"], env=os.environ.copy(), check=False)
1555
+ sys.exit(sync.returncode)
1286
1556
 
1287
1557
 
1288
1558
  from steve_cli.ontology import ontology
@@ -0,0 +1,205 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import os
5
+ import re
6
+ from copy import deepcopy
7
+ from pathlib import Path
8
+ from typing import Any, Literal
9
+
10
+ import httpx
11
+
12
+ _EMAIL_RE = re.compile(r'^[^@\s]+@[^@\s]+\.[^@\s]+$')
13
+
14
+ _POLICY_VERSION = "v0"
15
+
16
+
17
+ def _expand_schema(schema: str, sanitized_branch: str) -> str:
18
+ if schema == sanitized_branch or not schema:
19
+ return sanitized_branch
20
+ return schema
21
+
22
+
23
+ def _expand_rules(rules: list[dict[str, Any]], sanitized_branch: str, workspace: str = "") -> list[dict[str, Any]]:
24
+ expanded = []
25
+ for rule in rules:
26
+ rule = deepcopy(rule)
27
+ for entry in rule.get("tables", []):
28
+ if "schema" in entry:
29
+ entry["schema"] = _expand_schema(entry["schema"], sanitized_branch)
30
+ if workspace and "catalogs" not in entry:
31
+ tier = entry.get("tier", "bronze")
32
+ entry["catalogs"] = [f"{workspace}-{tier}"]
33
+ for entry in rule.get("columns", []):
34
+ if "schema" in entry:
35
+ entry["schema"] = _expand_schema(entry["schema"], sanitized_branch)
36
+ expanded.append(rule)
37
+ return expanded
38
+
39
+
40
+ class PolicyClient:
41
+ def __init__(
42
+ self,
43
+ endpoint: str | None = None,
44
+ token: str | None = None,
45
+ workspace_id: str | None = None,
46
+ ):
47
+ from steve_cli.auth import get_service_url, get_token
48
+ if endpoint is None:
49
+ endpoint = get_service_url("pcp", workspace_id) or "http://policy-control-plane:3010"
50
+ self.endpoint = endpoint.rstrip("/")
51
+ self.token = token or get_token(workspace_id) or ""
52
+
53
+ def __trpc(self, method: str, path: str, payload: dict | None = None) -> dict:
54
+ url = f"{self.endpoint}/api/trpc/{path}"
55
+ headers = {"Authorization": f"Bearer {self.token}"} if self.token else {}
56
+ if method == "query":
57
+ resp = httpx.get(url, params={"input": json.dumps(payload or {})}, headers=headers, timeout=10)
58
+ else:
59
+ resp = httpx.post(url, json=payload or {}, headers=headers, timeout=10)
60
+ resp.raise_for_status()
61
+ data = resp.json()
62
+ if "error" in data:
63
+ raise RuntimeError(f"tRPC error on {path}: {data['error']}")
64
+ result = data.get("result", {})
65
+ if isinstance(result, list):
66
+ return result
67
+ data_inner = result.get("data", result)
68
+ if isinstance(data_inner, list):
69
+ return data_inner
70
+ if isinstance(data_inner, dict):
71
+ return data_inner.get("json", data_inner)
72
+ return data_inner
73
+
74
+ def _trpc(self, method: str, path: str, payload: dict | None = None) -> dict:
75
+ try:
76
+ return self.__trpc(method, path, payload)
77
+ except httpx.ConnectError as e:
78
+ raise EnvironmentError(
79
+ f"Cannot reach Policy Control Plane at {self.endpoint} — "
80
+ f"run `steve login` to configure credentials"
81
+ ) from e
82
+
83
+ def list_pdps(self) -> list[dict]:
84
+ return self._trpc("query", "pdp.list") or []
85
+
86
+ def list_workspaces(self) -> list[str]:
87
+ return self._trpc("query", "workspace.list") or []
88
+
89
+ def list_workspace_pdps(self, workspace: str) -> list[dict]:
90
+ return self._trpc("query", "workspace.listPdps", {"workspace": workspace}) or []
91
+
92
+ def delete_pdp(self, pdp_id: str) -> dict:
93
+ return self._trpc("mutation", "pdp.delete", {"id": pdp_id})
94
+
95
+ def delete_pdp_policy(self, policy_id: str) -> dict:
96
+ return self._trpc("mutation", "pdp.policy.delete", {"id": policy_id})
97
+
98
+ def register_pdp(self, name: str, endpoint: str, pdp_type: str = "opa", is_default: bool = True) -> dict:
99
+ return self._trpc("mutation", "pdp.register", {
100
+ "name": name,
101
+ "type": pdp_type,
102
+ "endpoint": endpoint,
103
+ "isDefault": is_default,
104
+ })
105
+
106
+ def attach_pdp(self, workspace: str, pdp_id: str) -> None:
107
+ self._trpc("mutation", "workspace.attachPdp", {"workspace": workspace, "pdpId": pdp_id})
108
+
109
+ def apply_policies(self, workspace: str, pdp_name: str, rules: list[dict[str, Any]]) -> dict:
110
+ return self._trpc("mutation", "workspace.applyPolicies", {
111
+ "workspace": workspace,
112
+ "fragments": [
113
+ {
114
+ "syntax": "dsl",
115
+ "target": pdp_name,
116
+ "content": {"rules": rules},
117
+ }
118
+ ],
119
+ })
120
+
121
+ def _validate_against_trino(self, rules: list[dict[str, Any]], sanitized_branch: str) -> list[str]:
122
+ workspace_id = os.environ.get("WORKSPACE_ID")
123
+ workspace_name = os.environ.get("WORKSPACE_NAME", "")
124
+
125
+ seen: dict[str, list[str]] = {}
126
+ for rule in rules:
127
+ for entry in list(rule.get("tables", [])) + list(rule.get("columns", [])):
128
+ schema = entry.get("schema", "")
129
+ table = entry.get("table", "")
130
+ if schema and table:
131
+ seen.setdefault(schema, [])
132
+ if table not in seen[schema]:
133
+ seen[schema].append(table)
134
+
135
+ results: list[str] = []
136
+ for schema, tables in seen.items():
137
+ tier = "bronze"
138
+ ws_clean = workspace_name.lower().replace("-", "_") if workspace_name else ""
139
+ catalog = f"{workspace_name}-{tier}" if workspace_name else f"minio-{tier}"
140
+
141
+ try:
142
+ from steve_cli.storage.trino import TrinoStorage
143
+ storage = TrinoStorage(tier=tier, workspace=workspace_name or None)
144
+ existing = storage._execute(f'SHOW TABLES FROM "{storage.catalog}".{schema}')
145
+ existing_names = {r.get("Table", "") for r in existing}
146
+ catalog = storage.catalog
147
+ except Exception as exc:
148
+ results.append(f"⚠ cannot reach Trino to validate schema '{schema}': {exc}")
149
+ continue
150
+
151
+ for table in tables:
152
+ if table in existing_names:
153
+ results.append(f"✓ {catalog}.{schema}.{table}")
154
+ else:
155
+ results.append(
156
+ f"⚠ table '{table}' not found in '{catalog}.{schema}'"
157
+ + (f" (found: {sorted(existing_names)})" if existing_names else " (schema empty or not found)")
158
+ )
159
+ return results
160
+
161
+ def apply_from_file(self, policies_file: Path | None = None, validate: Literal[None, "warn", "error"] = None) -> tuple[str, str]:
162
+ path = policies_file or Path.cwd() / "policies" / "access.yaml"
163
+ if not path.exists():
164
+ raise FileNotFoundError(f"Policies file not found: {path}")
165
+
166
+ import yaml
167
+ config: dict[str, Any] = yaml.safe_load(path.read_text())
168
+ workspace = config["workspace"]
169
+ pdp_cfg = config["pdp"]
170
+ rules = config.get("rules", [])
171
+
172
+ invalid_users = [r["user"] for r in rules if not _EMAIL_RE.match(r.get("user", ""))]
173
+ if invalid_users:
174
+ raise ValueError(
175
+ f"Invalid user(s) in rules — expected email addresses, got: {invalid_users}. "
176
+ f"Use group-based access for wildcard permissions instead."
177
+ )
178
+
179
+ from steve_cli.storage.branch import get_branch_prefix
180
+ sanitized = get_branch_prefix()
181
+ branch = sanitized
182
+ rules = _expand_rules(rules, sanitized, workspace)
183
+
184
+ pdp = self.register_pdp(
185
+ name=pdp_cfg["name"],
186
+ endpoint=pdp_cfg["endpoint"],
187
+ pdp_type=pdp_cfg.get("type", "opa"),
188
+ )
189
+ self.attach_pdp(workspace, pdp["id"])
190
+ result = self.apply_policies(workspace, pdp_cfg["name"], rules)
191
+
192
+ errors = result.get("errors", [])
193
+ if errors:
194
+ reasons = "; ".join(e["reason"] for e in errors)
195
+ raise RuntimeError(f"Failed to apply {len(errors)} policy fragment(s): {reasons}")
196
+
197
+ if validate is not None:
198
+ issues = self._validate_against_trino(rules, sanitized)
199
+ failed = [msg for msg in issues if msg.startswith("⚠")]
200
+ for msg in issues:
201
+ print(msg)
202
+ if failed and validate == "error":
203
+ raise RuntimeError(f"Validation failed: {len(failed)} issue(s) found")
204
+
205
+ return branch, sanitized
@@ -33,8 +33,24 @@ def get_branch_prefix() -> str:
33
33
  branch = result.stdout.strip()
34
34
  if branch and branch != "HEAD":
35
35
  return _sanitize(branch)
36
+ if branch == "HEAD":
37
+ sha_result = subprocess.run(
38
+ ["git", "rev-parse", "--short", "HEAD"],
39
+ capture_output=True,
40
+ text=True,
41
+ timeout=3,
42
+ )
43
+ if sha_result.returncode == 0 and sha_result.stdout.strip():
44
+ short_sha = sha_result.stdout.strip()
45
+ logger.warning(
46
+ "Detached HEAD: using commit SHA as branch prefix (sha_%s). "
47
+ "Set STEVE_BRANCH to override.",
48
+ short_sha,
49
+ )
50
+ return f"sha_{short_sha}"
36
51
  except Exception as exc:
37
52
  logger.debug("git branch detection failed: %s", exc)
38
53
 
39
- logger.warning("Could not detect git branch; defaulting to 'main'")
40
- return "main"
54
+ raise RuntimeError(
55
+ "Could not detect git branch. Set STEVE_BRANCH=<branch-name> and retry."
56
+ )
@@ -15,13 +15,64 @@ from .branch import get_branch_prefix
15
15
  logger = logging.getLogger(__name__)
16
16
 
17
17
 
18
- def _derive_schema(tier: str, workspace: str | None, branch: str) -> str:
19
- base = f"{workspace.lower().replace('-', '_')}_{tier.lower()}" if workspace else tier.lower()
20
- return f"{base}__{branch}"
18
+ def _derive_schema(_tier: str, _workspace: str | None, branch: str) -> str:
19
+ return branch
20
+
21
+
22
+ def _patch_pyiceberg_get_fs_once() -> None:
23
+ """Fix pyiceberg race: per-thread get_fs() causes repeated _s3() unregister+register_last
24
+ cycles, leaving a window where S3V4RestSigner is absent from before-sign.s3 handlers.
25
+ Replace get_fs() with an instance-level shared cache so _s3() runs only once per
26
+ FsspecFileIO instance (from the calling thread), eliminating the re-registration race."""
27
+ logger.warning(
28
+ "Applied pyiceberg get_fs monkey-patch (instance-level cache) to work around "
29
+ "S3V4RestSigner race condition. Remove once upstream fixes are released: "
30
+ "https://github.com/apache/iceberg-python/issues/3625 "
31
+ "https://github.com/apache/iceberg-python/issues/3896"
32
+ )
33
+
34
+ try:
35
+ import threading as _t
36
+ from pyiceberg.io.fsspec import FsspecFileIO
37
+
38
+ _caches: dict[int, dict] = {}
39
+ _lock = _t.Lock()
40
+ _orig = FsspecFileIO.get_fs
41
+
42
+ def _get_fs_shared(self, scheme: str, hostname=None):
43
+ fio_id = id(self)
44
+ key = (scheme, hostname)
45
+ cache = _caches.get(fio_id)
46
+ if cache is not None and key in cache:
47
+ return cache[key]
48
+ with _lock:
49
+ if fio_id not in _caches:
50
+ _caches[fio_id] = {}
51
+ cache = _caches[fio_id]
52
+ if key not in cache:
53
+ cache[key] = _orig(self, scheme, hostname)
54
+ return cache[key]
55
+
56
+ FsspecFileIO.get_fs = _get_fs_shared
57
+
58
+ except Exception as exc:
59
+ logger.warning("Could not patch pyiceberg FsspecFileIO.get_fs: %s", exc)
60
+
61
+
62
+ _patch_pyiceberg_get_fs_once()
21
63
 
22
64
 
23
65
  class TrinoStorage:
24
66
  def __init__(self, tier: str = "bronze", workspace: str | None = None):
67
+ # aiobotocore 3.9+ defaults to STREAMING-UNSIGNED-PAYLOAD-TRAILER (flexible checksums)
68
+ # which Lakekeeper's remote signing proxy and MinIO don't support.
69
+ os.environ.setdefault('AWS_REQUEST_CHECKSUM_CALCULATION', 'when_required')
70
+ # Reduce keepalive from 12s to 3s: MinIO closes server-side connections faster than
71
+ # aiobotocore's default, causing stale pooled connections to return bare HTTP 403
72
+ # ("Forbidden", no S3 XML) on reuse. 3s ensures idle connections are evicted before
73
+ # the next put_bytes call reaches the S3 write phase.
74
+ import aiobotocore.httpsession
75
+ aiobotocore.httpsession.DEFAULT_KEEPALIVE_TIMEOUT = 3
25
76
  from steve_cli.auth import get_service_url
26
77
  self._branch = get_branch_prefix()
27
78
  self._tier = tier.lower()
@@ -47,61 +98,39 @@ class TrinoStorage:
47
98
  _lk_env = os.getenv("LAKEKEEPER_ENDPOINT")
48
99
  _lk_svc = get_service_url("lakekeeper", self._workspace_id)
49
100
  self._lakekeeper_endpoint = _lk_env.rstrip("/") if _lk_env else (_lk_svc.rstrip("/") + "/catalog" if _lk_svc else None)
101
+ self._lakekeeper_base = (self._lakekeeper_endpoint or "http://lakekeeper:8181/catalog").rstrip("/")
50
102
  self._lakekeeper_warehouse = os.getenv("LAKEKEEPER_WAREHOUSE", default_catalog)
51
103
  _iceberg_bucket = os.getenv(f"{tier.upper()}_BUCKET", "")
52
104
  if _iceberg_bucket:
53
- self._iceberg_location_prefix = f"s3://{_iceberg_bucket}/_iceberg/{self._branch}"
105
+ self._iceberg_location_prefix: str | None = f"s3://{_iceberg_bucket}/_iceberg/{self._branch}"
54
106
  else:
55
- raise OSError(
56
- f"TrinoStorage: missing bucket configuration for tier '{tier}' — "
57
- f"{tier.upper()}_BUCKET is not set.\n"
58
- f" Required env vars: {tier.upper()}_BUCKET, {tier.upper()}_ACCESS_KEY, {tier.upper()}_SECRET_KEY\n"
59
- f" This workspace may only have read access to the '{tier}' tier. "
60
- f"Write access requires provisioning credentials (run 'uv run steve setup env' or check workspace provisioning)."
61
- )
62
- self.__iceberg_catalog = None
107
+ self._iceberg_location_prefix = None
63
108
 
64
- @property
65
- def _iceberg_catalog(self):
66
- if self.__iceberg_catalog is None:
109
+ if os.getenv("STEVE_DIRECT_S3") == "1":
110
+ if not self._iceberg_location_prefix:
111
+ raise OSError(
112
+ f"TrinoStorage (STEVE_DIRECT_S3=1): missing bucket configuration for tier '{tier}' — "
113
+ f"{tier.upper()}_BUCKET is not set.\n"
114
+ f" Direct S3 mode requires explicit bucket credentials "
115
+ f"(run 'uv run steve setup env' or check workspace provisioning)."
116
+ )
67
117
  from pyiceberg.catalog.rest import RestCatalog
68
118
  from pyiceberg.io import load_file_io
69
119
  from pyiceberg.table import Table
70
-
71
- # WORKAROUND: Lakekeeper's remote S3 presigner is bypassed for external access.
72
- #
73
- # Lakekeeper returns `s3.signer` / `s3.signer.uri` in the table config, which
74
- # pyiceberg uses to sign S3 requests via Lakekeeper's signing endpoint. This works
75
- # inside the cluster but fails from a laptop because:
76
- # 1. The signed URI Lakekeeper produces contains the cluster-internal hostname
77
- # (e.g. http://minio:9000) which is unreachable externally.
78
- # 2. Even after overriding the signer URI to the public lakekeeper subdomain,
79
- # Lakekeeper's signer returns HTTP 500 for URIs with the public minio hostname
80
- # because it only knows the internal minio endpoint.
81
- #
82
- # Current workaround: strip all signer config from the merged table properties and
83
- # use direct S3 credentials (access-key + secret-key) with the public minio endpoint
84
- # from ~/.steve/credentials.json. This bypasses Lakekeeper's access control for S3
85
- # writes — credentials live on disk rather than being vended by the cluster.
86
- #
87
- # To revert: configure Lakekeeper with its public S3 endpoint so presigned URIs
88
- # contain a resolvable hostname, then remove the merged.pop() calls below and
89
- # restore `s3.signer.uri: lakekeeper_base` in s3_overrides.
90
- tier = self._tier.upper()
91
- lakekeeper_base = (self._lakekeeper_endpoint or "http://lakekeeper:8181/catalog").rstrip("/")
92
120
  from steve_cli.auth import get_s3_endpoint as _get_s3_ep
121
+ _tier = tier.upper()
93
122
  _s3_ep = _get_s3_ep(self._workspace_id) or os.getenv("S3_ENDPOINT", "http://minio:9000")
94
- s3_overrides = {
123
+ _s3_overrides = {
95
124
  "s3.endpoint": _s3_ep,
96
- "s3.access-key-id": os.getenv(f"{tier}_ACCESS_KEY") or os.getenv("AWS_ACCESS_KEY_ID", ""),
97
- "s3.secret-access-key": os.getenv(f"{tier}_SECRET_KEY") or os.getenv("AWS_SECRET_ACCESS_KEY", ""),
125
+ "s3.access-key-id": os.getenv(f"{_tier}_ACCESS_KEY") or os.getenv("AWS_ACCESS_KEY_ID", ""),
126
+ "s3.secret-access-key": os.getenv(f"{_tier}_SECRET_KEY") or os.getenv("AWS_SECRET_ACCESS_KEY", ""),
98
127
  "s3.path-style-access": "true",
99
128
  } if self._lakekeeper_endpoint else {}
129
+ _base = self._lakekeeper_base
100
130
 
101
131
  class _PatchedRestCatalog(RestCatalog):
102
132
  def _response_to_table(self_, identifier_tuple, table_response):
103
- merged = {**table_response.metadata.properties, **table_response.config, **s3_overrides, "uri": lakekeeper_base}
104
- # Strip remote-signer config — see WORKAROUND comment above.
133
+ merged = {**table_response.metadata.properties, **table_response.config, **_s3_overrides, "uri": _base}
105
134
  merged.pop("s3.signer", None)
106
135
  merged.pop("s3.signer.uri", None)
107
136
  merged.pop("s3.signer.endpoint", None)
@@ -114,27 +143,40 @@ class TrinoStorage:
114
143
  config=table_response.config,
115
144
  )
116
145
 
146
+ self._catalog_cls = _PatchedRestCatalog
147
+ self._extra_catalog_kwargs: dict = {**_s3_overrides}
148
+ else:
149
+ self._catalog_cls = None
150
+ self._extra_catalog_kwargs = {}
151
+
152
+ self.__iceberg_catalog = None
153
+
154
+ @property
155
+ def _iceberg_catalog(self):
156
+ if self.__iceberg_catalog is None:
157
+ from pyiceberg.catalog.rest import RestCatalog
117
158
  from steve_cli.auth import get_token
118
159
  stp_token = get_token(self._workspace_id)
119
160
  token_kwargs = {"token": stp_token} if stp_token else {}
161
+ catalog_cls = self._catalog_cls or RestCatalog
120
162
  try:
121
- catalog = _PatchedRestCatalog(
163
+ catalog = catalog_cls(
122
164
  "lakekeeper",
123
- uri=lakekeeper_base,
165
+ uri=self._lakekeeper_base,
124
166
  warehouse=self._lakekeeper_warehouse,
125
- **s3_overrides,
167
+ **self._extra_catalog_kwargs,
126
168
  **token_kwargs,
127
169
  )
128
170
  except Exception as exc:
129
171
  msg = str(exc)
130
172
  if "NoSuchWarehouseException" in msg or "does not exist" in msg:
131
173
  raise RuntimeError(
132
- f"Lakekeeper warehouse '{self._lakekeeper_warehouse}' not found at {lakekeeper_base}.\n"
174
+ f"Lakekeeper warehouse '{self._lakekeeper_warehouse}' not found at {self._lakekeeper_base}.\n"
133
175
  f"The workspace may not have been provisioned yet — ensure the workspace was created in the data mesh before running this job."
134
176
  ) from None
135
177
  raise
136
- if self._lakekeeper_endpoint:
137
- catalog.uri = lakekeeper_base
178
+ if self._catalog_cls and self._lakekeeper_endpoint:
179
+ catalog.uri = self._lakekeeper_base
138
180
  self.__iceberg_catalog = catalog
139
181
  return self.__iceberg_catalog
140
182
 
@@ -278,8 +320,7 @@ class TrinoStorage:
278
320
  import datetime
279
321
 
280
322
  from pyiceberg.exceptions import NamespaceAlreadyExistsError
281
- lakekeeper_base = (self._lakekeeper_endpoint or "http://lakekeeper:8181/catalog").rstrip("/")
282
- namespace_props = {"location": self._iceberg_location_prefix}
323
+ namespace_props = {"location": self._iceberg_location_prefix} if self._iceberg_location_prefix else {}
283
324
  created = False
284
325
  try:
285
326
  self._iceberg_catalog.create_namespace(self.schema, properties=namespace_props)
@@ -292,12 +333,12 @@ class TrinoStorage:
292
333
  if "403" in msg or "Access Denied" in msg or "Forbidden" in msg:
293
334
  raise OSError(
294
335
  f"TrinoStorage: namespace '{self.schema}' creation denied at Lakekeeper "
295
- f"[{lakekeeper_base}] — check that the token has write access to warehouse "
336
+ f"[{self._lakekeeper_base}] — check that the token has write access to warehouse "
296
337
  f"'{self._lakekeeper_warehouse}'"
297
338
  ) from exc
298
339
  raise OSError(
299
340
  f"TrinoStorage: namespace '{self.schema}' creation failed at Lakekeeper "
300
- f"[{lakekeeper_base}] — {type(exc).__name__}: {exc}"
341
+ f"[{self._lakekeeper_base}] — {type(exc).__name__}: {exc}"
301
342
  ) from exc
302
343
 
303
344
  if created:
@@ -317,7 +358,7 @@ class TrinoStorage:
317
358
  "branch": [self._branch],
318
359
  "tier": [self._tier],
319
360
  "schema_name": [self.schema],
320
- "iceberg_location": [self._iceberg_location_prefix],
361
+ "iceberg_location": [self._iceberg_location_prefix or ""],
321
362
  "created_at": [datetime.datetime.now(datetime.timezone.utc).isoformat()],
322
363
  }))
323
364
  logger.info("Created _meta table in namespace %s", self.schema)
@@ -325,39 +366,101 @@ class TrinoStorage:
325
366
  logger.warning("Could not create _meta table in %s: %s", self.schema, exc)
326
367
 
327
368
  try:
369
+ _location_clause = f" WITH (location = '{self._iceberg_location_prefix}')" if self._iceberg_location_prefix else ""
328
370
  self._execute(
329
- f'CREATE SCHEMA IF NOT EXISTS "{self.catalog}".{self.schema} '
330
- f"WITH (location = '{self._iceberg_location_prefix}')"
371
+ f'CREATE SCHEMA IF NOT EXISTS "{self.catalog}".{self.schema}{_location_clause}'
331
372
  )
332
373
  logger.debug("Ensured Trino schema %s.%s", self.catalog, self.schema)
333
374
  except Exception as exc:
334
375
  logger.warning("Could not ensure Trino schema %s.%s: %s", self.catalog, self.schema, exc)
335
376
 
336
377
  def put_bytes(self, data: bytes, path: str) -> None:
337
- from pyiceberg.exceptions import NoSuchTableError
378
+ from pyiceberg.exceptions import NoSuchTableError, SignError
338
379
 
339
380
  arrow_table = pq.read_table(io.BytesIO(data))
340
381
  table_id = f"{self.schema}.{self._table_name(path)}"
341
382
  catalog = self._iceberg_catalog
342
- lakekeeper_base = self._lakekeeper_endpoint or "http://lakekeeper:8181/catalog"
383
+ lakekeeper_base = self._lakekeeper_base
343
384
 
344
385
  self._ensure_namespace()
345
386
 
346
387
  def _append(iceberg_table, arrow_table, table_id):
388
+ if os.getenv("STEVE_TRINO_TRACE") == "1":
389
+ io_props = getattr(iceberg_table.io, "properties", {})
390
+ logger.warning(
391
+ "[trino] append: table=%s, location=%s, s3.endpoint=%s, signer.uri=%s",
392
+ table_id,
393
+ iceberg_table.location(),
394
+ io_props.get("s3.endpoint", "n/a"),
395
+ io_props.get("s3.signer.uri", "n/a"),
396
+ )
397
+
398
+ s3_client = None
399
+ if os.getenv("STEVE_TRINO_TRACE") == "1":
400
+ try:
401
+ import threading as _threading
402
+ s3_fs = iceberg_table.io._get_fs("s3")
403
+ s3_client = s3_fs.s3
404
+
405
+ def _trace_sign(request, **kwargs):
406
+ logger.warning("[trino] s3 SIGN %s req_id=%x (thread=%s)",
407
+ request.method, id(request),
408
+ _threading.current_thread().name)
409
+
410
+ def _trace_send(request, **kwargs):
411
+ hdrs = dict(getattr(request, "headers", {}))
412
+ auth = next((v for k, v in hdrs.items() if k.lower() == "authorization"), "<MISSING>")
413
+ logger.warning("[trino] s3.%s req_id=%x auth=%s (thread=%s)",
414
+ request.method, id(request),
415
+ auth[:24] if auth != "<MISSING>" else "<MISSING>",
416
+ _threading.current_thread().name)
417
+
418
+ def _trace_retry(response=None, operation=None, request_dict=None, **kwargs):
419
+ if not response:
420
+ return
421
+ status = getattr(response[0], "status_code", "?")
422
+ if status >= 300:
423
+ url = (request_dict or {}).get("url", "?")
424
+ logger.warning("[trino] s3 <- %s %s %s", operation or "?", status, url)
425
+
426
+ s3_client.meta.events.register("before-sign.s3", _trace_sign, unique_id="steve-trino-s3-sign")
427
+ s3_client.meta.events.register("before-send.s3", _trace_send, unique_id="steve-trino-s3-trace")
428
+ s3_client.meta.events.register("needs-retry.s3", _trace_retry, unique_id="steve-trino-s3-retry")
429
+ except Exception as hook_exc:
430
+ logger.warning("[trino] TRACE hook failed: %s", hook_exc)
431
+ s3_client = None
432
+
347
433
  try:
348
434
  iceberg_table.append(arrow_table)
435
+ except SignError as exc:
436
+ raise OSError(
437
+ f"TrinoStorage: Lakekeeper refused to sign write for '{table_id}' "
438
+ f"[signer={lakekeeper_base}, warehouse={self._lakekeeper_warehouse}] — {exc}\n"
439
+ f" Hint: the token may lack write permissions on this warehouse."
440
+ ) from exc
349
441
  except Exception as exc:
350
442
  msg = str(exc)
351
443
  if "403" in msg or "Access Denied" in msg or "Forbidden" in msg:
352
444
  raise OSError(
353
- f"TrinoStorage: write to '{table_id}' denied via Lakekeeper [{lakekeeper_base}] — "
354
- f"Access Denied on warehouse '{self._lakekeeper_warehouse}' "
355
- f"(check S3 bucket policy and token permissions)"
445
+ f"TrinoStorage: write to '{table_id}' denied by MinIO "
446
+ f"[location={iceberg_table.location()}, warehouse={self._lakekeeper_warehouse}] — "
447
+ f"{type(exc).__name__}: {exc}"
356
448
  ) from exc
357
449
  raise OSError(
358
450
  f"TrinoStorage: write to '{table_id}' failed via Lakekeeper [{lakekeeper_base}] — "
359
451
  f"{type(exc).__name__}: {exc}"
360
452
  ) from exc
453
+ finally:
454
+ if s3_client is not None:
455
+ for event, uid in (
456
+ ("before-sign.s3", "steve-trino-s3-sign"),
457
+ ("before-send.s3", "steve-trino-s3-trace"),
458
+ ("needs-retry.s3", "steve-trino-s3-retry"),
459
+ ):
460
+ try:
461
+ s3_client.meta.events.unregister(event, unique_id=uid)
462
+ except Exception:
463
+ pass
361
464
 
362
465
  try:
363
466
  iceberg_table = catalog.load_table(table_id)
@@ -400,13 +503,19 @@ class TrinoStorage:
400
503
  iceberg_schema = Schema(*fields)
401
504
  try:
402
505
  iceberg_table = catalog.create_table(table_id, schema=iceberg_schema)
506
+ except SignError as exc:
507
+ raise OSError(
508
+ f"TrinoStorage: Lakekeeper refused to sign table creation for '{table_id}' "
509
+ f"[signer={lakekeeper_base}, warehouse={self._lakekeeper_warehouse}] — {exc}\n"
510
+ f" Hint: the token may lack write permissions on this warehouse."
511
+ ) from exc
403
512
  except Exception as exc:
404
513
  msg = str(exc)
405
514
  if "403" in msg or "Access Denied" in msg or "Forbidden" in msg:
406
515
  raise OSError(
407
- f"TrinoStorage: create table '{table_id}' denied via Lakekeeper [{lakekeeper_base}] — "
408
- f"Access Denied on warehouse '{self._lakekeeper_warehouse}' "
409
- f"(check S3 bucket policy and token permissions)"
516
+ f"TrinoStorage: create table '{table_id}' denied by MinIO "
517
+ f"[location={self._iceberg_location_prefix or '<lakekeeper-managed>'}, warehouse={self._lakekeeper_warehouse}] — "
518
+ f"{type(exc).__name__}: {exc}"
410
519
  ) from exc
411
520
  raise OSError(
412
521
  f"TrinoStorage: create table '{table_id}' failed via Lakekeeper [{lakekeeper_base}] — "
@@ -423,7 +532,7 @@ class TrinoStorage:
423
532
  logger.warning(
424
533
  "[trino] put_bytes: wrote %d rows → %s.%s (warehouse=%s, location=%s%s)",
425
534
  len(arrow_table), self.catalog, table_id,
426
- self._lakekeeper_warehouse, self._iceberg_location_prefix,
535
+ self._lakekeeper_warehouse, self._iceberg_location_prefix or "<lakekeeper-managed>",
427
536
  f", snapshot={snapshot.snapshot_id}" if snapshot else "",
428
537
  )
429
538
  logger.info("Written %d rows to %s", len(arrow_table), table_id)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: steve-cli
3
- Version: 1.0.3
3
+ Version: 1.1.2
4
4
  Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
5
5
  Author: Frank
6
6
  License: MIT
@@ -49,7 +49,7 @@ Provides-Extra: trino
49
49
  Requires-Dist: trino>=0.330; extra == "trino"
50
50
  Requires-Dist: pyarrow>=17.0.0; extra == "trino"
51
51
  Requires-Dist: requests>=2.31.0; extra == "trino"
52
- Requires-Dist: pyiceberg>=0.7.0; extra == "trino"
52
+ Requires-Dist: pyiceberg>=0.12.0; extra == "trino"
53
53
  Requires-Dist: s3fs>=2024.1.0; extra == "trino"
54
54
  Provides-Extra: visidata
55
55
  Requires-Dist: visidata>=3.0; extra == "visidata"
@@ -46,7 +46,7 @@ pyarrow>=17.0.0
46
46
  trino>=0.330
47
47
  pyarrow>=17.0.0
48
48
  requests>=2.31.0
49
- pyiceberg>=0.7.0
49
+ pyiceberg>=0.12.0
50
50
  s3fs>=2024.1.0
51
51
 
52
52
  [validoopsie]
@@ -1,93 +0,0 @@
1
- from __future__ import annotations
2
-
3
- import json
4
- from pathlib import Path
5
- from typing import Any
6
-
7
- import httpx
8
-
9
- _POLICY_VERSION = "v0"
10
-
11
-
12
- class PolicyClient:
13
- def __init__(
14
- self,
15
- endpoint: str | None = None,
16
- token: str | None = None,
17
- workspace_id: str | None = None,
18
- ):
19
- from steve_cli.auth import get_service_url, get_token
20
- if endpoint is None:
21
- endpoint = get_service_url("pcp", workspace_id) or "http://policy-control-plane:3010"
22
- self.endpoint = endpoint.rstrip("/")
23
- self.token = token or get_token(workspace_id) or ""
24
-
25
- def __trpc(self, method: str, path: str, payload: dict | None = None) -> dict:
26
- url = f"{self.endpoint}/api/trpc/{path}"
27
- headers = {"Authorization": f"Bearer {self.token}"} if self.token else {}
28
- if method == "query":
29
- resp = httpx.get(url, params={"input": json.dumps(payload or {})}, headers=headers, timeout=10)
30
- else:
31
- resp = httpx.post(url, json=payload or {}, headers=headers, timeout=10)
32
- resp.raise_for_status()
33
- data = resp.json()
34
- if "error" in data:
35
- raise RuntimeError(f"tRPC error on {path}: {data['error']}")
36
- result = data.get("result", {})
37
- return result.get("data", {}).get("json", result.get("data", result))
38
-
39
- def _trpc(self, method: str, path: str, payload: dict | None = None) -> dict:
40
- try:
41
- return self.__trpc(method, path, payload)
42
- except httpx.ConnectError as e:
43
- raise EnvironmentError(
44
- f"Cannot reach Policy Control Plane at {self.endpoint} — "
45
- f"run `steve login` to configure credentials"
46
- ) from e
47
-
48
- def register_pdp(self, name: str, endpoint: str, pdp_type: str = "opa", is_default: bool = True) -> dict:
49
- return self._trpc("mutation", "pdp.register", {
50
- "name": name,
51
- "type": pdp_type,
52
- "endpoint": endpoint,
53
- "isDefault": is_default,
54
- })
55
-
56
- def attach_pdp(self, workspace: str, pdp_id: str) -> None:
57
- self._trpc("mutation", "workspace.attachPdp", {"workspace": workspace, "pdpId": pdp_id})
58
-
59
- def apply_policies(self, workspace: str, pdp_name: str, rules: list[dict[str, Any]]) -> dict:
60
- return self._trpc("mutation", "workspace.applyPolicies", {
61
- "workspace": workspace,
62
- "fragments": [
63
- {
64
- "syntax": "dsl",
65
- "target": pdp_name,
66
- "content": {"rules": rules},
67
- }
68
- ],
69
- })
70
-
71
- def apply_from_file(self, policies_file: Path | None = None) -> None:
72
- path = policies_file or Path.cwd() / "policies" / "access.yaml"
73
- if not path.exists():
74
- raise FileNotFoundError(f"Policies file not found: {path}")
75
-
76
- import yaml
77
- config: dict[str, Any] = yaml.safe_load(path.read_text())
78
- workspace = config["workspace"]
79
- pdp_cfg = config["pdp"]
80
- rules = config.get("rules", [])
81
-
82
- pdp = self.register_pdp(
83
- name=pdp_cfg["name"],
84
- endpoint=pdp_cfg["endpoint"],
85
- pdp_type=pdp_cfg.get("type", "opa"),
86
- )
87
- self.attach_pdp(workspace, pdp["id"])
88
- result = self.apply_policies(workspace, pdp_cfg["name"], rules)
89
-
90
- errors = result.get("errors", [])
91
- if errors:
92
- reasons = "; ".join(e["reason"] for e in errors)
93
- raise RuntimeError(f"Failed to apply {len(errors)} policy fragment(s): {reasons}")
File without changes
File without changes
File without changes