steve-cli 1.1.1__tar.gz → 1.1.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. {steve_cli-1.1.1 → steve_cli-1.1.2}/PKG-INFO +2 -2
  2. {steve_cli-1.1.1 → steve_cli-1.1.2}/pyproject.toml +2 -2
  3. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/cli.py +5 -1
  4. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/storage/trino.py +171 -61
  5. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli.egg-info/PKG-INFO +2 -2
  6. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli.egg-info/requires.txt +1 -1
  7. {steve_cli-1.1.1 → steve_cli-1.1.2}/README.md +0 -0
  8. {steve_cli-1.1.1 → steve_cli-1.1.2}/setup.cfg +0 -0
  9. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/__init__.py +0 -0
  10. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/auth.py +0 -0
  11. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/decorators/__init__.py +0 -0
  12. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/decorators/lineage_job.py +0 -0
  13. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/lineage/__init__.py +0 -0
  14. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/lineage/adapters/__init__.py +0 -0
  15. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/lineage/adapters/composite.py +0 -0
  16. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/lineage/adapters/dataproductregistry.py +0 -0
  17. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/lineage/adapters/logging.py +0 -0
  18. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/lineage/adapters/null.py +0 -0
  19. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/lineage/adapters/openlineage.py +0 -0
  20. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/lineage/collector.py +0 -0
  21. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/lineage/port.py +0 -0
  22. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/lineage/registry.py +0 -0
  23. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/lineage/storage.py +0 -0
  24. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/ontology.py +0 -0
  25. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/policies/__init__.py +0 -0
  26. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/policies/client.py +0 -0
  27. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/storage/__init__.py +0 -0
  28. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/storage/branch.py +0 -0
  29. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/storage/metadata/__init__.py +0 -0
  30. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/__init__.py +0 -0
  31. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/csv.py +0 -0
  32. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/excel.py +0 -0
  33. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/generic.py +0 -0
  34. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/json.py +0 -0
  35. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/storage/metadata/extractors/parquet.py +0 -0
  36. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/storage/metadata/port.py +0 -0
  37. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/storage/metadata/registry.py +0 -0
  38. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/storage/parquet.py +0 -0
  39. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/storage/protocol.py +0 -0
  40. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/storage/s3.py +0 -0
  41. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/storage.py +0 -0
  42. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/validation/__init__.py +0 -0
  43. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/validation/adapters/__init__.py +0 -0
  44. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/validation/adapters/great_expectations.py +0 -0
  45. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/validation/adapters/null.py +0 -0
  46. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/validation/adapters/validoopsie.py +0 -0
  47. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/validation/port.py +0 -0
  48. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/validation/registry.py +0 -0
  49. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli/vkg.py +0 -0
  50. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli.egg-info/SOURCES.txt +0 -0
  51. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli.egg-info/dependency_links.txt +0 -0
  52. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli.egg-info/entry_points.txt +0 -0
  53. {steve_cli-1.1.1 → steve_cli-1.1.2}/steve_cli.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: steve-cli
3
- Version: 1.1.1
3
+ Version: 1.1.2
4
4
  Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
5
5
  Author: Frank
6
6
  License: MIT
@@ -49,7 +49,7 @@ Provides-Extra: trino
49
49
  Requires-Dist: trino>=0.330; extra == "trino"
50
50
  Requires-Dist: pyarrow>=17.0.0; extra == "trino"
51
51
  Requires-Dist: requests>=2.31.0; extra == "trino"
52
- Requires-Dist: pyiceberg>=0.7.0; extra == "trino"
52
+ Requires-Dist: pyiceberg>=0.12.0; extra == "trino"
53
53
  Requires-Dist: s3fs>=2024.1.0; extra == "trino"
54
54
  Provides-Extra: visidata
55
55
  Requires-Dist: visidata>=3.0; extra == "visidata"
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
5
5
 
6
6
  [project]
7
7
  name = "steve-cli"
8
- version = "1.1.1"
8
+ version = "1.1.2"
9
9
  description = "A simple CLI tool to run jobs from jobs.yaml with proper environment setup"
10
10
  readme = "README.md"
11
11
  license = {text = "MIT"}
@@ -65,7 +65,7 @@ trino = [
65
65
  "trino>=0.330",
66
66
  "pyarrow>=17.0.0",
67
67
  "requests>=2.31.0",
68
- "pyiceberg>=0.7.0",
68
+ "pyiceberg>=0.12.0",
69
69
  "s3fs>=2024.1.0",
70
70
  ]
71
71
  visidata = [
@@ -1548,7 +1548,11 @@ def upgrade():
1548
1548
  env=os.environ.copy(),
1549
1549
  check=False,
1550
1550
  )
1551
- sys.exit(result.returncode)
1551
+ if result.returncode != 0:
1552
+ sys.exit(result.returncode)
1553
+ click.echo("🔒 Syncing lockfile...")
1554
+ sync = subprocess.run(["uv", "sync"], env=os.environ.copy(), check=False)
1555
+ sys.exit(sync.returncode)
1552
1556
 
1553
1557
 
1554
1558
  from steve_cli.ontology import ontology
@@ -19,8 +19,60 @@ def _derive_schema(_tier: str, _workspace: str | None, branch: str) -> str:
19
19
  return branch
20
20
 
21
21
 
22
+ def _patch_pyiceberg_get_fs_once() -> None:
23
+ """Fix pyiceberg race: per-thread get_fs() causes repeated _s3() unregister+register_last
24
+ cycles, leaving a window where S3V4RestSigner is absent from before-sign.s3 handlers.
25
+ Replace get_fs() with an instance-level shared cache so _s3() runs only once per
26
+ FsspecFileIO instance (from the calling thread), eliminating the re-registration race."""
27
+ logger.warning(
28
+ "Applied pyiceberg get_fs monkey-patch (instance-level cache) to work around "
29
+ "S3V4RestSigner race condition. Remove once upstream fixes are released: "
30
+ "https://github.com/apache/iceberg-python/issues/3625 "
31
+ "https://github.com/apache/iceberg-python/issues/3896"
32
+ )
33
+
34
+ try:
35
+ import threading as _t
36
+ from pyiceberg.io.fsspec import FsspecFileIO
37
+
38
+ _caches: dict[int, dict] = {}
39
+ _lock = _t.Lock()
40
+ _orig = FsspecFileIO.get_fs
41
+
42
+ def _get_fs_shared(self, scheme: str, hostname=None):
43
+ fio_id = id(self)
44
+ key = (scheme, hostname)
45
+ cache = _caches.get(fio_id)
46
+ if cache is not None and key in cache:
47
+ return cache[key]
48
+ with _lock:
49
+ if fio_id not in _caches:
50
+ _caches[fio_id] = {}
51
+ cache = _caches[fio_id]
52
+ if key not in cache:
53
+ cache[key] = _orig(self, scheme, hostname)
54
+ return cache[key]
55
+
56
+ FsspecFileIO.get_fs = _get_fs_shared
57
+
58
+ except Exception as exc:
59
+ logger.warning("Could not patch pyiceberg FsspecFileIO.get_fs: %s", exc)
60
+
61
+
62
+ _patch_pyiceberg_get_fs_once()
63
+
64
+
22
65
  class TrinoStorage:
23
66
  def __init__(self, tier: str = "bronze", workspace: str | None = None):
67
+ # aiobotocore 3.9+ defaults to STREAMING-UNSIGNED-PAYLOAD-TRAILER (flexible checksums)
68
+ # which Lakekeeper's remote signing proxy and MinIO don't support.
69
+ os.environ.setdefault('AWS_REQUEST_CHECKSUM_CALCULATION', 'when_required')
70
+ # Reduce keepalive from 12s to 3s: MinIO closes server-side connections faster than
71
+ # aiobotocore's default, causing stale pooled connections to return bare HTTP 403
72
+ # ("Forbidden", no S3 XML) on reuse. 3s ensures idle connections are evicted before
73
+ # the next put_bytes call reaches the S3 write phase.
74
+ import aiobotocore.httpsession
75
+ aiobotocore.httpsession.DEFAULT_KEEPALIVE_TIMEOUT = 3
24
76
  from steve_cli.auth import get_service_url
25
77
  self._branch = get_branch_prefix()
26
78
  self._tier = tier.lower()
@@ -46,61 +98,39 @@ class TrinoStorage:
46
98
  _lk_env = os.getenv("LAKEKEEPER_ENDPOINT")
47
99
  _lk_svc = get_service_url("lakekeeper", self._workspace_id)
48
100
  self._lakekeeper_endpoint = _lk_env.rstrip("/") if _lk_env else (_lk_svc.rstrip("/") + "/catalog" if _lk_svc else None)
101
+ self._lakekeeper_base = (self._lakekeeper_endpoint or "http://lakekeeper:8181/catalog").rstrip("/")
49
102
  self._lakekeeper_warehouse = os.getenv("LAKEKEEPER_WAREHOUSE", default_catalog)
50
103
  _iceberg_bucket = os.getenv(f"{tier.upper()}_BUCKET", "")
51
104
  if _iceberg_bucket:
52
- self._iceberg_location_prefix = f"s3://{_iceberg_bucket}/_iceberg/{self._branch}"
105
+ self._iceberg_location_prefix: str | None = f"s3://{_iceberg_bucket}/_iceberg/{self._branch}"
53
106
  else:
54
- raise OSError(
55
- f"TrinoStorage: missing bucket configuration for tier '{tier}' — "
56
- f"{tier.upper()}_BUCKET is not set.\n"
57
- f" Required env vars: {tier.upper()}_BUCKET, {tier.upper()}_ACCESS_KEY, {tier.upper()}_SECRET_KEY\n"
58
- f" This workspace may only have read access to the '{tier}' tier. "
59
- f"Write access requires provisioning credentials (run 'uv run steve setup env' or check workspace provisioning)."
60
- )
61
- self.__iceberg_catalog = None
107
+ self._iceberg_location_prefix = None
62
108
 
63
- @property
64
- def _iceberg_catalog(self):
65
- if self.__iceberg_catalog is None:
109
+ if os.getenv("STEVE_DIRECT_S3") == "1":
110
+ if not self._iceberg_location_prefix:
111
+ raise OSError(
112
+ f"TrinoStorage (STEVE_DIRECT_S3=1): missing bucket configuration for tier '{tier}' — "
113
+ f"{tier.upper()}_BUCKET is not set.\n"
114
+ f" Direct S3 mode requires explicit bucket credentials "
115
+ f"(run 'uv run steve setup env' or check workspace provisioning)."
116
+ )
66
117
  from pyiceberg.catalog.rest import RestCatalog
67
118
  from pyiceberg.io import load_file_io
68
119
  from pyiceberg.table import Table
69
-
70
- # WORKAROUND: Lakekeeper's remote S3 presigner is bypassed for external access.
71
- #
72
- # Lakekeeper returns `s3.signer` / `s3.signer.uri` in the table config, which
73
- # pyiceberg uses to sign S3 requests via Lakekeeper's signing endpoint. This works
74
- # inside the cluster but fails from a laptop because:
75
- # 1. The signed URI Lakekeeper produces contains the cluster-internal hostname
76
- # (e.g. http://minio:9000) which is unreachable externally.
77
- # 2. Even after overriding the signer URI to the public lakekeeper subdomain,
78
- # Lakekeeper's signer returns HTTP 500 for URIs with the public minio hostname
79
- # because it only knows the internal minio endpoint.
80
- #
81
- # Current workaround: strip all signer config from the merged table properties and
82
- # use direct S3 credentials (access-key + secret-key) with the public minio endpoint
83
- # from ~/.steve/credentials.json. This bypasses Lakekeeper's access control for S3
84
- # writes — credentials live on disk rather than being vended by the cluster.
85
- #
86
- # To revert: configure Lakekeeper with its public S3 endpoint so presigned URIs
87
- # contain a resolvable hostname, then remove the merged.pop() calls below and
88
- # restore `s3.signer.uri: lakekeeper_base` in s3_overrides.
89
- tier = self._tier.upper()
90
- lakekeeper_base = (self._lakekeeper_endpoint or "http://lakekeeper:8181/catalog").rstrip("/")
91
120
  from steve_cli.auth import get_s3_endpoint as _get_s3_ep
121
+ _tier = tier.upper()
92
122
  _s3_ep = _get_s3_ep(self._workspace_id) or os.getenv("S3_ENDPOINT", "http://minio:9000")
93
- s3_overrides = {
123
+ _s3_overrides = {
94
124
  "s3.endpoint": _s3_ep,
95
- "s3.access-key-id": os.getenv(f"{tier}_ACCESS_KEY") or os.getenv("AWS_ACCESS_KEY_ID", ""),
96
- "s3.secret-access-key": os.getenv(f"{tier}_SECRET_KEY") or os.getenv("AWS_SECRET_ACCESS_KEY", ""),
125
+ "s3.access-key-id": os.getenv(f"{_tier}_ACCESS_KEY") or os.getenv("AWS_ACCESS_KEY_ID", ""),
126
+ "s3.secret-access-key": os.getenv(f"{_tier}_SECRET_KEY") or os.getenv("AWS_SECRET_ACCESS_KEY", ""),
97
127
  "s3.path-style-access": "true",
98
128
  } if self._lakekeeper_endpoint else {}
129
+ _base = self._lakekeeper_base
99
130
 
100
131
  class _PatchedRestCatalog(RestCatalog):
101
132
  def _response_to_table(self_, identifier_tuple, table_response):
102
- merged = {**table_response.metadata.properties, **table_response.config, **s3_overrides, "uri": lakekeeper_base}
103
- # Strip remote-signer config — see WORKAROUND comment above.
133
+ merged = {**table_response.metadata.properties, **table_response.config, **_s3_overrides, "uri": _base}
104
134
  merged.pop("s3.signer", None)
105
135
  merged.pop("s3.signer.uri", None)
106
136
  merged.pop("s3.signer.endpoint", None)
@@ -113,27 +143,40 @@ class TrinoStorage:
113
143
  config=table_response.config,
114
144
  )
115
145
 
146
+ self._catalog_cls = _PatchedRestCatalog
147
+ self._extra_catalog_kwargs: dict = {**_s3_overrides}
148
+ else:
149
+ self._catalog_cls = None
150
+ self._extra_catalog_kwargs = {}
151
+
152
+ self.__iceberg_catalog = None
153
+
154
+ @property
155
+ def _iceberg_catalog(self):
156
+ if self.__iceberg_catalog is None:
157
+ from pyiceberg.catalog.rest import RestCatalog
116
158
  from steve_cli.auth import get_token
117
159
  stp_token = get_token(self._workspace_id)
118
160
  token_kwargs = {"token": stp_token} if stp_token else {}
161
+ catalog_cls = self._catalog_cls or RestCatalog
119
162
  try:
120
- catalog = _PatchedRestCatalog(
163
+ catalog = catalog_cls(
121
164
  "lakekeeper",
122
- uri=lakekeeper_base,
165
+ uri=self._lakekeeper_base,
123
166
  warehouse=self._lakekeeper_warehouse,
124
- **s3_overrides,
167
+ **self._extra_catalog_kwargs,
125
168
  **token_kwargs,
126
169
  )
127
170
  except Exception as exc:
128
171
  msg = str(exc)
129
172
  if "NoSuchWarehouseException" in msg or "does not exist" in msg:
130
173
  raise RuntimeError(
131
- f"Lakekeeper warehouse '{self._lakekeeper_warehouse}' not found at {lakekeeper_base}.\n"
174
+ f"Lakekeeper warehouse '{self._lakekeeper_warehouse}' not found at {self._lakekeeper_base}.\n"
132
175
  f"The workspace may not have been provisioned yet — ensure the workspace was created in the data mesh before running this job."
133
176
  ) from None
134
177
  raise
135
- if self._lakekeeper_endpoint:
136
- catalog.uri = lakekeeper_base
178
+ if self._catalog_cls and self._lakekeeper_endpoint:
179
+ catalog.uri = self._lakekeeper_base
137
180
  self.__iceberg_catalog = catalog
138
181
  return self.__iceberg_catalog
139
182
 
@@ -277,8 +320,7 @@ class TrinoStorage:
277
320
  import datetime
278
321
 
279
322
  from pyiceberg.exceptions import NamespaceAlreadyExistsError
280
- lakekeeper_base = (self._lakekeeper_endpoint or "http://lakekeeper:8181/catalog").rstrip("/")
281
- namespace_props = {"location": self._iceberg_location_prefix}
323
+ namespace_props = {"location": self._iceberg_location_prefix} if self._iceberg_location_prefix else {}
282
324
  created = False
283
325
  try:
284
326
  self._iceberg_catalog.create_namespace(self.schema, properties=namespace_props)
@@ -291,12 +333,12 @@ class TrinoStorage:
291
333
  if "403" in msg or "Access Denied" in msg or "Forbidden" in msg:
292
334
  raise OSError(
293
335
  f"TrinoStorage: namespace '{self.schema}' creation denied at Lakekeeper "
294
- f"[{lakekeeper_base}] — check that the token has write access to warehouse "
336
+ f"[{self._lakekeeper_base}] — check that the token has write access to warehouse "
295
337
  f"'{self._lakekeeper_warehouse}'"
296
338
  ) from exc
297
339
  raise OSError(
298
340
  f"TrinoStorage: namespace '{self.schema}' creation failed at Lakekeeper "
299
- f"[{lakekeeper_base}] — {type(exc).__name__}: {exc}"
341
+ f"[{self._lakekeeper_base}] — {type(exc).__name__}: {exc}"
300
342
  ) from exc
301
343
 
302
344
  if created:
@@ -316,7 +358,7 @@ class TrinoStorage:
316
358
  "branch": [self._branch],
317
359
  "tier": [self._tier],
318
360
  "schema_name": [self.schema],
319
- "iceberg_location": [self._iceberg_location_prefix],
361
+ "iceberg_location": [self._iceberg_location_prefix or ""],
320
362
  "created_at": [datetime.datetime.now(datetime.timezone.utc).isoformat()],
321
363
  }))
322
364
  logger.info("Created _meta table in namespace %s", self.schema)
@@ -324,39 +366,101 @@ class TrinoStorage:
324
366
  logger.warning("Could not create _meta table in %s: %s", self.schema, exc)
325
367
 
326
368
  try:
369
+ _location_clause = f" WITH (location = '{self._iceberg_location_prefix}')" if self._iceberg_location_prefix else ""
327
370
  self._execute(
328
- f'CREATE SCHEMA IF NOT EXISTS "{self.catalog}".{self.schema} '
329
- f"WITH (location = '{self._iceberg_location_prefix}')"
371
+ f'CREATE SCHEMA IF NOT EXISTS "{self.catalog}".{self.schema}{_location_clause}'
330
372
  )
331
373
  logger.debug("Ensured Trino schema %s.%s", self.catalog, self.schema)
332
374
  except Exception as exc:
333
375
  logger.warning("Could not ensure Trino schema %s.%s: %s", self.catalog, self.schema, exc)
334
376
 
335
377
  def put_bytes(self, data: bytes, path: str) -> None:
336
- from pyiceberg.exceptions import NoSuchTableError
378
+ from pyiceberg.exceptions import NoSuchTableError, SignError
337
379
 
338
380
  arrow_table = pq.read_table(io.BytesIO(data))
339
381
  table_id = f"{self.schema}.{self._table_name(path)}"
340
382
  catalog = self._iceberg_catalog
341
- lakekeeper_base = self._lakekeeper_endpoint or "http://lakekeeper:8181/catalog"
383
+ lakekeeper_base = self._lakekeeper_base
342
384
 
343
385
  self._ensure_namespace()
344
386
 
345
387
  def _append(iceberg_table, arrow_table, table_id):
388
+ if os.getenv("STEVE_TRINO_TRACE") == "1":
389
+ io_props = getattr(iceberg_table.io, "properties", {})
390
+ logger.warning(
391
+ "[trino] append: table=%s, location=%s, s3.endpoint=%s, signer.uri=%s",
392
+ table_id,
393
+ iceberg_table.location(),
394
+ io_props.get("s3.endpoint", "n/a"),
395
+ io_props.get("s3.signer.uri", "n/a"),
396
+ )
397
+
398
+ s3_client = None
399
+ if os.getenv("STEVE_TRINO_TRACE") == "1":
400
+ try:
401
+ import threading as _threading
402
+ s3_fs = iceberg_table.io._get_fs("s3")
403
+ s3_client = s3_fs.s3
404
+
405
+ def _trace_sign(request, **kwargs):
406
+ logger.warning("[trino] s3 SIGN %s req_id=%x (thread=%s)",
407
+ request.method, id(request),
408
+ _threading.current_thread().name)
409
+
410
+ def _trace_send(request, **kwargs):
411
+ hdrs = dict(getattr(request, "headers", {}))
412
+ auth = next((v for k, v in hdrs.items() if k.lower() == "authorization"), "<MISSING>")
413
+ logger.warning("[trino] s3.%s req_id=%x auth=%s (thread=%s)",
414
+ request.method, id(request),
415
+ auth[:24] if auth != "<MISSING>" else "<MISSING>",
416
+ _threading.current_thread().name)
417
+
418
+ def _trace_retry(response=None, operation=None, request_dict=None, **kwargs):
419
+ if not response:
420
+ return
421
+ status = getattr(response[0], "status_code", "?")
422
+ if status >= 300:
423
+ url = (request_dict or {}).get("url", "?")
424
+ logger.warning("[trino] s3 <- %s %s %s", operation or "?", status, url)
425
+
426
+ s3_client.meta.events.register("before-sign.s3", _trace_sign, unique_id="steve-trino-s3-sign")
427
+ s3_client.meta.events.register("before-send.s3", _trace_send, unique_id="steve-trino-s3-trace")
428
+ s3_client.meta.events.register("needs-retry.s3", _trace_retry, unique_id="steve-trino-s3-retry")
429
+ except Exception as hook_exc:
430
+ logger.warning("[trino] TRACE hook failed: %s", hook_exc)
431
+ s3_client = None
432
+
346
433
  try:
347
434
  iceberg_table.append(arrow_table)
435
+ except SignError as exc:
436
+ raise OSError(
437
+ f"TrinoStorage: Lakekeeper refused to sign write for '{table_id}' "
438
+ f"[signer={lakekeeper_base}, warehouse={self._lakekeeper_warehouse}] — {exc}\n"
439
+ f" Hint: the token may lack write permissions on this warehouse."
440
+ ) from exc
348
441
  except Exception as exc:
349
442
  msg = str(exc)
350
443
  if "403" in msg or "Access Denied" in msg or "Forbidden" in msg:
351
444
  raise OSError(
352
- f"TrinoStorage: write to '{table_id}' denied via Lakekeeper [{lakekeeper_base}] — "
353
- f"Access Denied on warehouse '{self._lakekeeper_warehouse}' "
354
- f"(check S3 bucket policy and token permissions)"
445
+ f"TrinoStorage: write to '{table_id}' denied by MinIO "
446
+ f"[location={iceberg_table.location()}, warehouse={self._lakekeeper_warehouse}] — "
447
+ f"{type(exc).__name__}: {exc}"
355
448
  ) from exc
356
449
  raise OSError(
357
450
  f"TrinoStorage: write to '{table_id}' failed via Lakekeeper [{lakekeeper_base}] — "
358
451
  f"{type(exc).__name__}: {exc}"
359
452
  ) from exc
453
+ finally:
454
+ if s3_client is not None:
455
+ for event, uid in (
456
+ ("before-sign.s3", "steve-trino-s3-sign"),
457
+ ("before-send.s3", "steve-trino-s3-trace"),
458
+ ("needs-retry.s3", "steve-trino-s3-retry"),
459
+ ):
460
+ try:
461
+ s3_client.meta.events.unregister(event, unique_id=uid)
462
+ except Exception:
463
+ pass
360
464
 
361
465
  try:
362
466
  iceberg_table = catalog.load_table(table_id)
@@ -399,13 +503,19 @@ class TrinoStorage:
399
503
  iceberg_schema = Schema(*fields)
400
504
  try:
401
505
  iceberg_table = catalog.create_table(table_id, schema=iceberg_schema)
506
+ except SignError as exc:
507
+ raise OSError(
508
+ f"TrinoStorage: Lakekeeper refused to sign table creation for '{table_id}' "
509
+ f"[signer={lakekeeper_base}, warehouse={self._lakekeeper_warehouse}] — {exc}\n"
510
+ f" Hint: the token may lack write permissions on this warehouse."
511
+ ) from exc
402
512
  except Exception as exc:
403
513
  msg = str(exc)
404
514
  if "403" in msg or "Access Denied" in msg or "Forbidden" in msg:
405
515
  raise OSError(
406
- f"TrinoStorage: create table '{table_id}' denied via Lakekeeper [{lakekeeper_base}] — "
407
- f"Access Denied on warehouse '{self._lakekeeper_warehouse}' "
408
- f"(check S3 bucket policy and token permissions)"
516
+ f"TrinoStorage: create table '{table_id}' denied by MinIO "
517
+ f"[location={self._iceberg_location_prefix or '<lakekeeper-managed>'}, warehouse={self._lakekeeper_warehouse}] — "
518
+ f"{type(exc).__name__}: {exc}"
409
519
  ) from exc
410
520
  raise OSError(
411
521
  f"TrinoStorage: create table '{table_id}' failed via Lakekeeper [{lakekeeper_base}] — "
@@ -422,7 +532,7 @@ class TrinoStorage:
422
532
  logger.warning(
423
533
  "[trino] put_bytes: wrote %d rows → %s.%s (warehouse=%s, location=%s%s)",
424
534
  len(arrow_table), self.catalog, table_id,
425
- self._lakekeeper_warehouse, self._iceberg_location_prefix,
535
+ self._lakekeeper_warehouse, self._iceberg_location_prefix or "<lakekeeper-managed>",
426
536
  f", snapshot={snapshot.snapshot_id}" if snapshot else "",
427
537
  )
428
538
  logger.info("Written %d rows to %s", len(arrow_table), table_id)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: steve-cli
3
- Version: 1.1.1
3
+ Version: 1.1.2
4
4
  Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
5
5
  Author: Frank
6
6
  License: MIT
@@ -49,7 +49,7 @@ Provides-Extra: trino
49
49
  Requires-Dist: trino>=0.330; extra == "trino"
50
50
  Requires-Dist: pyarrow>=17.0.0; extra == "trino"
51
51
  Requires-Dist: requests>=2.31.0; extra == "trino"
52
- Requires-Dist: pyiceberg>=0.7.0; extra == "trino"
52
+ Requires-Dist: pyiceberg>=0.12.0; extra == "trino"
53
53
  Requires-Dist: s3fs>=2024.1.0; extra == "trino"
54
54
  Provides-Extra: visidata
55
55
  Requires-Dist: visidata>=3.0; extra == "visidata"
@@ -46,7 +46,7 @@ pyarrow>=17.0.0
46
46
  trino>=0.330
47
47
  pyarrow>=17.0.0
48
48
  requests>=2.31.0
49
- pyiceberg>=0.7.0
49
+ pyiceberg>=0.12.0
50
50
  s3fs>=2024.1.0
51
51
 
52
52
  [validoopsie]
File without changes
File without changes
File without changes
File without changes