steve-cli 1.1.1__tar.gz → 1.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {steve_cli-1.1.1 → steve_cli-1.1.3}/PKG-INFO +4 -2
- {steve_cli-1.1.1 → steve_cli-1.1.3}/pyproject.toml +5 -2
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/cli.py +5 -1
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/decorators/lineage_job.py +8 -1
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/storage/__init__.py +2 -0
- steve_cli-1.1.3/steve_cli/storage/lakekeeper.py +235 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/storage/trino.py +171 -61
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli.egg-info/PKG-INFO +4 -2
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli.egg-info/SOURCES.txt +1 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli.egg-info/requires.txt +4 -1
- {steve_cli-1.1.1 → steve_cli-1.1.3}/README.md +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/setup.cfg +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/__init__.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/auth.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/decorators/__init__.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/lineage/__init__.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/lineage/adapters/__init__.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/lineage/adapters/composite.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/lineage/adapters/dataproductregistry.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/lineage/adapters/logging.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/lineage/adapters/null.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/lineage/adapters/openlineage.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/lineage/collector.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/lineage/port.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/lineage/registry.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/lineage/storage.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/ontology.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/policies/__init__.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/policies/client.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/storage/branch.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/storage/metadata/__init__.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/storage/metadata/extractors/__init__.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/storage/metadata/extractors/csv.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/storage/metadata/extractors/excel.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/storage/metadata/extractors/generic.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/storage/metadata/extractors/json.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/storage/metadata/extractors/parquet.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/storage/metadata/port.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/storage/metadata/registry.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/storage/parquet.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/storage/protocol.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/storage/s3.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/storage.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/validation/__init__.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/validation/adapters/__init__.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/validation/adapters/great_expectations.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/validation/adapters/null.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/validation/adapters/validoopsie.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/validation/port.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/validation/registry.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli/vkg.py +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli.egg-info/dependency_links.txt +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli.egg-info/entry_points.txt +0 -0
- {steve_cli-1.1.1 → steve_cli-1.1.3}/steve_cli.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: steve-cli
|
|
3
|
-
Version: 1.1.
|
|
3
|
+
Version: 1.1.3
|
|
4
4
|
Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
|
|
5
5
|
Author: Frank
|
|
6
6
|
License: MIT
|
|
@@ -37,6 +37,8 @@ Requires-Dist: pandas>=2.0.3; extra == "pandas"
|
|
|
37
37
|
Requires-Dist: pyarrow>=17.0.0; extra == "pandas"
|
|
38
38
|
Provides-Extra: lineage
|
|
39
39
|
Requires-Dist: openlineage-python>=1.0.0; extra == "lineage"
|
|
40
|
+
Provides-Extra: lakekeeper
|
|
41
|
+
Requires-Dist: pylakekeeper>=0.4.0; extra == "lakekeeper"
|
|
40
42
|
Provides-Extra: validoopsie
|
|
41
43
|
Requires-Dist: validoopsie>=0.1.0; extra == "validoopsie"
|
|
42
44
|
Requires-Dist: polars>=1.8.2; extra == "validoopsie"
|
|
@@ -49,7 +51,7 @@ Provides-Extra: trino
|
|
|
49
51
|
Requires-Dist: trino>=0.330; extra == "trino"
|
|
50
52
|
Requires-Dist: pyarrow>=17.0.0; extra == "trino"
|
|
51
53
|
Requires-Dist: requests>=2.31.0; extra == "trino"
|
|
52
|
-
Requires-Dist: pyiceberg>=0.
|
|
54
|
+
Requires-Dist: pyiceberg>=0.12.0; extra == "trino"
|
|
53
55
|
Requires-Dist: s3fs>=2024.1.0; extra == "trino"
|
|
54
56
|
Provides-Extra: visidata
|
|
55
57
|
Requires-Dist: visidata>=3.0; extra == "visidata"
|
|
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
|
|
|
5
5
|
|
|
6
6
|
[project]
|
|
7
7
|
name = "steve-cli"
|
|
8
|
-
version = "1.1.
|
|
8
|
+
version = "1.1.3"
|
|
9
9
|
description = "A simple CLI tool to run jobs from jobs.yaml with proper environment setup"
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
license = {text = "MIT"}
|
|
@@ -50,6 +50,9 @@ pandas = [
|
|
|
50
50
|
lineage = [
|
|
51
51
|
"openlineage-python>=1.0.0",
|
|
52
52
|
]
|
|
53
|
+
lakekeeper = [
|
|
54
|
+
"pylakekeeper>=0.4.0",
|
|
55
|
+
]
|
|
53
56
|
validoopsie = [
|
|
54
57
|
"validoopsie>=0.1.0",
|
|
55
58
|
"polars>=1.8.2",
|
|
@@ -65,7 +68,7 @@ trino = [
|
|
|
65
68
|
"trino>=0.330",
|
|
66
69
|
"pyarrow>=17.0.0",
|
|
67
70
|
"requests>=2.31.0",
|
|
68
|
-
"pyiceberg>=0.
|
|
71
|
+
"pyiceberg>=0.12.0",
|
|
69
72
|
"s3fs>=2024.1.0",
|
|
70
73
|
]
|
|
71
74
|
visidata = [
|
|
@@ -1548,7 +1548,11 @@ def upgrade():
|
|
|
1548
1548
|
env=os.environ.copy(),
|
|
1549
1549
|
check=False,
|
|
1550
1550
|
)
|
|
1551
|
-
|
|
1551
|
+
if result.returncode != 0:
|
|
1552
|
+
sys.exit(result.returncode)
|
|
1553
|
+
click.echo("🔒 Syncing lockfile...")
|
|
1554
|
+
sync = subprocess.run(["uv", "sync"], env=os.environ.copy(), check=False)
|
|
1555
|
+
sys.exit(sync.returncode)
|
|
1552
1556
|
|
|
1553
1557
|
|
|
1554
1558
|
from steve_cli.ontology import ontology
|
|
@@ -55,8 +55,15 @@ def lineage_job(
|
|
|
55
55
|
session=session,
|
|
56
56
|
)
|
|
57
57
|
|
|
58
|
+
def get_files(tier: str = "bronze", workspace: str | None = None) -> LineageStorage:
|
|
59
|
+
from steve_cli.storage.lakekeeper import LakekeeperStorage
|
|
60
|
+
return LineageStorage(
|
|
61
|
+
storage=lambda: LakekeeperStorage(tier=tier, workspace=workspace),
|
|
62
|
+
session=session,
|
|
63
|
+
)
|
|
64
|
+
|
|
58
65
|
sig = inspect.signature(fn)
|
|
59
|
-
available = {"get_storage": get_storage, "get_tables": get_tables}
|
|
66
|
+
available = {"get_storage": get_storage, "get_tables": get_tables, "get_files": get_files}
|
|
60
67
|
injectable = {k: v for k, v in available.items() if k in sig.parameters}
|
|
61
68
|
|
|
62
69
|
try:
|
|
@@ -2,6 +2,7 @@ from .branch import get_branch_prefix
|
|
|
2
2
|
from .protocol import Storage
|
|
3
3
|
from .s3 import S3Storage
|
|
4
4
|
from .trino import TrinoStorage
|
|
5
|
+
from .lakekeeper import LakekeeperStorage
|
|
5
6
|
from .parquet import ParquetMetadata, extract_parquet_metadata
|
|
6
7
|
from .metadata import FileMetadata, ColumnMetadata, MetadataExtractorPort, MetadataRegistry
|
|
7
8
|
|
|
@@ -10,6 +11,7 @@ __all__ = [
|
|
|
10
11
|
"Storage",
|
|
11
12
|
"S3Storage",
|
|
12
13
|
"TrinoStorage",
|
|
14
|
+
"LakekeeperStorage",
|
|
13
15
|
"ParquetMetadata",
|
|
14
16
|
"extract_parquet_metadata",
|
|
15
17
|
"FileMetadata",
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path, PurePosixPath
|
|
7
|
+
|
|
8
|
+
import requests
|
|
9
|
+
|
|
10
|
+
from .branch import get_branch_prefix
|
|
11
|
+
from steve_cli.auth import get_service_url, get_token
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
_UUID_RE = re.compile(
|
|
16
|
+
r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
_SIGNER_SSL_PATCHED = False
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _patch_s3v4_signer_no_ssl_verify() -> None:
|
|
23
|
+
"""Replace SIGNERS['S3V4RestSigner'] with a subclass whose requests.Session has verify=False.
|
|
24
|
+
|
|
25
|
+
Needed when the Lakekeeper signing endpoint uses a cert with a hostname mismatch.
|
|
26
|
+
Mirrors the pyiceberg get_fs monkey-patch in trino.py — applied once, process-wide.
|
|
27
|
+
"""
|
|
28
|
+
global _SIGNER_SSL_PATCHED
|
|
29
|
+
if _SIGNER_SSL_PATCHED:
|
|
30
|
+
return
|
|
31
|
+
try:
|
|
32
|
+
from pyiceberg.io.fsspec import SIGNERS, S3V4RestSigner # type: ignore[attr-defined]
|
|
33
|
+
orig = SIGNERS.get("S3V4RestSigner", S3V4RestSigner)
|
|
34
|
+
|
|
35
|
+
class _NoVerify(orig): # type: ignore[valid-type]
|
|
36
|
+
def __init__(self, props): # type: ignore[override]
|
|
37
|
+
super().__init__(props)
|
|
38
|
+
self._session.verify = False
|
|
39
|
+
|
|
40
|
+
SIGNERS["S3V4RestSigner"] = _NoVerify
|
|
41
|
+
_SIGNER_SSL_PATCHED = True
|
|
42
|
+
logger.warning(
|
|
43
|
+
"Applied S3V4RestSigner SSL-verify=False patch (LAKEKEEPER_SSL_VERIFY=0). "
|
|
44
|
+
"Remove once the Lakekeeper TLS certificate covers the full hostname."
|
|
45
|
+
)
|
|
46
|
+
except Exception as exc:
|
|
47
|
+
logger.warning("Could not patch S3V4RestSigner SSL verify: %s", exc)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class LakekeeperStorage:
|
|
51
|
+
"""
|
|
52
|
+
Storage backend for unstructured/binary files via Lakekeeper Generic Tables (DATASET format).
|
|
53
|
+
|
|
54
|
+
This warehouse uses remote signing (s3.signer: S3V4RestSigner) — Lakekeeper signs every
|
|
55
|
+
S3 request rather than vending short-lived credentials. We therefore use pyiceberg's
|
|
56
|
+
FsspecFileIO with S3V4RestSigner, the same mechanism TrinoStorage uses for Iceberg tables.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
def __init__(self, tier: str = "bronze", workspace: str | None = None):
|
|
60
|
+
# aiobotocore 3.9+ defaults to STREAMING-UNSIGNED-PAYLOAD-TRAILER (flexible checksums)
|
|
61
|
+
# which Lakekeeper's remote signing proxy and MinIO don't support.
|
|
62
|
+
os.environ.setdefault("AWS_REQUEST_CHECKSUM_CALCULATION", "when_required")
|
|
63
|
+
import aiobotocore.httpsession
|
|
64
|
+
aiobotocore.httpsession.DEFAULT_KEEPALIVE_TIMEOUT = 3
|
|
65
|
+
|
|
66
|
+
self._tier = tier.lower()
|
|
67
|
+
self._branch = get_branch_prefix()
|
|
68
|
+
self._workspace_id = os.getenv("WORKSPACE_ID")
|
|
69
|
+
|
|
70
|
+
_lk_env = os.getenv("LAKEKEEPER_ENDPOINT")
|
|
71
|
+
_lk_svc = get_service_url("lakekeeper", self._workspace_id)
|
|
72
|
+
self._lakekeeper_base_url = (
|
|
73
|
+
_lk_env.rstrip("/") if _lk_env
|
|
74
|
+
else (_lk_svc.rstrip("/") if _lk_svc else "http://lakekeeper:8181")
|
|
75
|
+
)
|
|
76
|
+
self._token = get_token(self._workspace_id)
|
|
77
|
+
|
|
78
|
+
resolved_workspace = workspace or os.getenv("WORKSPACE_NAME")
|
|
79
|
+
default_warehouse = f"{resolved_workspace}-{tier}" if resolved_workspace else "minio"
|
|
80
|
+
self._warehouse = os.getenv("LAKEKEEPER_WAREHOUSE", default_warehouse)
|
|
81
|
+
|
|
82
|
+
self._namespace = self._branch
|
|
83
|
+
self._table = f"{self._tier}_files"
|
|
84
|
+
self._table_ensured = False
|
|
85
|
+
self.__lk_client = None
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def _ssl_verify(self) -> bool:
|
|
89
|
+
return os.getenv("LAKEKEEPER_SSL_VERIFY", "1") != "0"
|
|
90
|
+
|
|
91
|
+
def _resolve_warehouse_uuid(self) -> str:
|
|
92
|
+
"""Resolve warehouse name → UUID (generic-tables path requires a UUID, not a name)."""
|
|
93
|
+
if _UUID_RE.match(self._warehouse):
|
|
94
|
+
return self._warehouse
|
|
95
|
+
headers = {"Authorization": f"Bearer {self._token}"} if self._token else {}
|
|
96
|
+
try:
|
|
97
|
+
resp = requests.get(
|
|
98
|
+
f"{self._lakekeeper_base_url}/management/v1/warehouse",
|
|
99
|
+
headers=headers,
|
|
100
|
+
verify=self._ssl_verify,
|
|
101
|
+
timeout=10.0,
|
|
102
|
+
)
|
|
103
|
+
resp.raise_for_status()
|
|
104
|
+
except Exception as exc:
|
|
105
|
+
raise OSError(
|
|
106
|
+
f"LakekeeperStorage: could not list warehouses at {self._lakekeeper_base_url} "
|
|
107
|
+
f"to resolve '{self._warehouse}' → UUID — {type(exc).__name__}: {exc}"
|
|
108
|
+
) from exc
|
|
109
|
+
warehouses = resp.json().get("warehouses", [])
|
|
110
|
+
for w in warehouses:
|
|
111
|
+
if w.get("name") == self._warehouse:
|
|
112
|
+
return w["id"]
|
|
113
|
+
names = [w.get("name") for w in warehouses]
|
|
114
|
+
raise OSError(
|
|
115
|
+
f"LakekeeperStorage: warehouse '{self._warehouse}' not found at "
|
|
116
|
+
f"{self._lakekeeper_base_url} — available: {names}"
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
def _get_client(self):
|
|
120
|
+
if self.__lk_client is None:
|
|
121
|
+
import httpx
|
|
122
|
+
from pylakekeeper import Client, StaticToken
|
|
123
|
+
auth = StaticToken(self._token) if self._token else StaticToken("dev")
|
|
124
|
+
warehouse_uuid = self._resolve_warehouse_uuid()
|
|
125
|
+
client = Client(
|
|
126
|
+
base_url=self._lakekeeper_base_url,
|
|
127
|
+
warehouse=warehouse_uuid,
|
|
128
|
+
auth=auth,
|
|
129
|
+
)
|
|
130
|
+
if not self._ssl_verify:
|
|
131
|
+
client._transport._client = httpx.Client(
|
|
132
|
+
base_url=self._lakekeeper_base_url.rstrip("/"),
|
|
133
|
+
verify=False,
|
|
134
|
+
timeout=30.0,
|
|
135
|
+
)
|
|
136
|
+
self.__lk_client = client
|
|
137
|
+
return self.__lk_client
|
|
138
|
+
|
|
139
|
+
def _ensure_table(self) -> None:
|
|
140
|
+
if self._table_ensured:
|
|
141
|
+
return
|
|
142
|
+
from pylakekeeper import ConflictError, GenericTableFormat
|
|
143
|
+
c = self._get_client()
|
|
144
|
+
try:
|
|
145
|
+
c.generic_tables.create(
|
|
146
|
+
self._namespace, self._table,
|
|
147
|
+
format=GenericTableFormat.DATASET,
|
|
148
|
+
doc=f"{self._tier} unstructured files ({self._branch})",
|
|
149
|
+
)
|
|
150
|
+
logger.info("Created Lakekeeper generic table %s.%s", self._namespace, self._table)
|
|
151
|
+
except ConflictError:
|
|
152
|
+
pass
|
|
153
|
+
except Exception as exc:
|
|
154
|
+
raise OSError(
|
|
155
|
+
f"LakekeeperStorage: could not create generic table "
|
|
156
|
+
f"'{self._namespace}.{self._table}' at {self._lakekeeper_base_url} "
|
|
157
|
+
f"[warehouse={self._warehouse}] — {type(exc).__name__}: {exc}"
|
|
158
|
+
) from exc
|
|
159
|
+
self._table_ensured = True
|
|
160
|
+
|
|
161
|
+
def _build_io(self, credentials: dict):
|
|
162
|
+
"""Build a pyiceberg FsspecFileIO from Lakekeeper remote-signing credentials."""
|
|
163
|
+
if not self._ssl_verify:
|
|
164
|
+
_patch_s3v4_signer_no_ssl_verify()
|
|
165
|
+
from pyiceberg.io import load_file_io
|
|
166
|
+
props = {
|
|
167
|
+
**credentials,
|
|
168
|
+
"token": self._token or "",
|
|
169
|
+
# S3V4RestSigner falls back to properties["uri"] when s3.signer.uri is absent.
|
|
170
|
+
# Python evaluates default arguments eagerly, so "uri" must always be present
|
|
171
|
+
# even when "s3.signer.uri" is set — otherwise dict.get() raises KeyError.
|
|
172
|
+
"uri": (
|
|
173
|
+
credentials.get("s3.signer.uri")
|
|
174
|
+
or credentials.get("signer.uri")
|
|
175
|
+
or self._lakekeeper_base_url.rstrip("/") + "/catalog"
|
|
176
|
+
),
|
|
177
|
+
}
|
|
178
|
+
return load_file_io(props, "")
|
|
179
|
+
|
|
180
|
+
def _vend(self):
|
|
181
|
+
"""Return (pyiceberg FileIO, location_prefix) with fresh vended table metadata."""
|
|
182
|
+
self._ensure_table()
|
|
183
|
+
resp = self._get_client().generic_tables.load(
|
|
184
|
+
self._namespace, self._table, vended=True
|
|
185
|
+
)
|
|
186
|
+
return self._build_io(resp.credentials), resp.location.rstrip("/")
|
|
187
|
+
|
|
188
|
+
def _full_path(self, location: str, path: str) -> str:
|
|
189
|
+
clean = str(PurePosixPath(path).as_posix().lstrip("/"))
|
|
190
|
+
return f"{location}/{clean}"
|
|
191
|
+
|
|
192
|
+
def put_bytes(self, data: bytes, path: str) -> None:
|
|
193
|
+
io, loc = self._vend()
|
|
194
|
+
with io.new_output(self._full_path(loc, path)).create(overwrite=True) as f:
|
|
195
|
+
f.write(data)
|
|
196
|
+
logger.info("Stored %d bytes → %s/%s", len(data), loc, path)
|
|
197
|
+
|
|
198
|
+
def get_bytes(self, path: str) -> bytes:
|
|
199
|
+
io, loc = self._vend()
|
|
200
|
+
with io.new_input(self._full_path(loc, path)).open() as f:
|
|
201
|
+
return f.read()
|
|
202
|
+
|
|
203
|
+
def put_file(self, local_path: str, path: str) -> None:
|
|
204
|
+
io, loc = self._vend()
|
|
205
|
+
data = Path(local_path).read_bytes()
|
|
206
|
+
with io.new_output(self._full_path(loc, path)).create(overwrite=True) as f:
|
|
207
|
+
f.write(data)
|
|
208
|
+
logger.info("Uploaded %s → %s/%s", local_path, loc, path)
|
|
209
|
+
|
|
210
|
+
def get_file(self, path: str, local_path: str) -> None:
|
|
211
|
+
Path(local_path).parent.mkdir(parents=True, exist_ok=True)
|
|
212
|
+
io, loc = self._vend()
|
|
213
|
+
with io.new_input(self._full_path(loc, path)).open() as f:
|
|
214
|
+
Path(local_path).write_bytes(f.read())
|
|
215
|
+
|
|
216
|
+
def list(self, prefix: str = "") -> list[str]:
|
|
217
|
+
io, loc = self._vend()
|
|
218
|
+
search = f"{loc}/{prefix.lstrip('/')}" if prefix else loc
|
|
219
|
+
fs = io.get_fs("s3")
|
|
220
|
+
try:
|
|
221
|
+
paths = fs.find(search, detail=False)
|
|
222
|
+
except FileNotFoundError:
|
|
223
|
+
return []
|
|
224
|
+
loc_stripped = loc.removeprefix("s3://")
|
|
225
|
+
result = []
|
|
226
|
+
for p in paths:
|
|
227
|
+
p_stripped = p.removeprefix("s3://")
|
|
228
|
+
if p_stripped.startswith(loc_stripped + "/"):
|
|
229
|
+
result.append(p_stripped[len(loc_stripped) + 1:])
|
|
230
|
+
else:
|
|
231
|
+
result.append(p_stripped)
|
|
232
|
+
return result
|
|
233
|
+
|
|
234
|
+
def list_all(self) -> list[str]:
|
|
235
|
+
return self.list()
|
|
@@ -19,8 +19,60 @@ def _derive_schema(_tier: str, _workspace: str | None, branch: str) -> str:
|
|
|
19
19
|
return branch
|
|
20
20
|
|
|
21
21
|
|
|
22
|
+
def _patch_pyiceberg_get_fs_once() -> None:
|
|
23
|
+
"""Fix pyiceberg race: per-thread get_fs() causes repeated _s3() unregister+register_last
|
|
24
|
+
cycles, leaving a window where S3V4RestSigner is absent from before-sign.s3 handlers.
|
|
25
|
+
Replace get_fs() with an instance-level shared cache so _s3() runs only once per
|
|
26
|
+
FsspecFileIO instance (from the calling thread), eliminating the re-registration race."""
|
|
27
|
+
logger.warning(
|
|
28
|
+
"Applied pyiceberg get_fs monkey-patch (instance-level cache) to work around "
|
|
29
|
+
"S3V4RestSigner race condition. Remove once upstream fixes are released: "
|
|
30
|
+
"https://github.com/apache/iceberg-python/issues/3625 "
|
|
31
|
+
"https://github.com/apache/iceberg-python/issues/3896"
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
try:
|
|
35
|
+
import threading as _t
|
|
36
|
+
from pyiceberg.io.fsspec import FsspecFileIO
|
|
37
|
+
|
|
38
|
+
_caches: dict[int, dict] = {}
|
|
39
|
+
_lock = _t.Lock()
|
|
40
|
+
_orig = FsspecFileIO.get_fs
|
|
41
|
+
|
|
42
|
+
def _get_fs_shared(self, scheme: str, hostname=None):
|
|
43
|
+
fio_id = id(self)
|
|
44
|
+
key = (scheme, hostname)
|
|
45
|
+
cache = _caches.get(fio_id)
|
|
46
|
+
if cache is not None and key in cache:
|
|
47
|
+
return cache[key]
|
|
48
|
+
with _lock:
|
|
49
|
+
if fio_id not in _caches:
|
|
50
|
+
_caches[fio_id] = {}
|
|
51
|
+
cache = _caches[fio_id]
|
|
52
|
+
if key not in cache:
|
|
53
|
+
cache[key] = _orig(self, scheme, hostname)
|
|
54
|
+
return cache[key]
|
|
55
|
+
|
|
56
|
+
FsspecFileIO.get_fs = _get_fs_shared
|
|
57
|
+
|
|
58
|
+
except Exception as exc:
|
|
59
|
+
logger.warning("Could not patch pyiceberg FsspecFileIO.get_fs: %s", exc)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
_patch_pyiceberg_get_fs_once()
|
|
63
|
+
|
|
64
|
+
|
|
22
65
|
class TrinoStorage:
|
|
23
66
|
def __init__(self, tier: str = "bronze", workspace: str | None = None):
|
|
67
|
+
# aiobotocore 3.9+ defaults to STREAMING-UNSIGNED-PAYLOAD-TRAILER (flexible checksums)
|
|
68
|
+
# which Lakekeeper's remote signing proxy and MinIO don't support.
|
|
69
|
+
os.environ.setdefault('AWS_REQUEST_CHECKSUM_CALCULATION', 'when_required')
|
|
70
|
+
# Reduce keepalive from 12s to 3s: MinIO closes server-side connections faster than
|
|
71
|
+
# aiobotocore's default, causing stale pooled connections to return bare HTTP 403
|
|
72
|
+
# ("Forbidden", no S3 XML) on reuse. 3s ensures idle connections are evicted before
|
|
73
|
+
# the next put_bytes call reaches the S3 write phase.
|
|
74
|
+
import aiobotocore.httpsession
|
|
75
|
+
aiobotocore.httpsession.DEFAULT_KEEPALIVE_TIMEOUT = 3
|
|
24
76
|
from steve_cli.auth import get_service_url
|
|
25
77
|
self._branch = get_branch_prefix()
|
|
26
78
|
self._tier = tier.lower()
|
|
@@ -46,61 +98,39 @@ class TrinoStorage:
|
|
|
46
98
|
_lk_env = os.getenv("LAKEKEEPER_ENDPOINT")
|
|
47
99
|
_lk_svc = get_service_url("lakekeeper", self._workspace_id)
|
|
48
100
|
self._lakekeeper_endpoint = _lk_env.rstrip("/") if _lk_env else (_lk_svc.rstrip("/") + "/catalog" if _lk_svc else None)
|
|
101
|
+
self._lakekeeper_base = (self._lakekeeper_endpoint or "http://lakekeeper:8181/catalog").rstrip("/")
|
|
49
102
|
self._lakekeeper_warehouse = os.getenv("LAKEKEEPER_WAREHOUSE", default_catalog)
|
|
50
103
|
_iceberg_bucket = os.getenv(f"{tier.upper()}_BUCKET", "")
|
|
51
104
|
if _iceberg_bucket:
|
|
52
|
-
self._iceberg_location_prefix = f"s3://{_iceberg_bucket}/_iceberg/{self._branch}"
|
|
105
|
+
self._iceberg_location_prefix: str | None = f"s3://{_iceberg_bucket}/_iceberg/{self._branch}"
|
|
53
106
|
else:
|
|
54
|
-
|
|
55
|
-
f"TrinoStorage: missing bucket configuration for tier '{tier}' — "
|
|
56
|
-
f"{tier.upper()}_BUCKET is not set.\n"
|
|
57
|
-
f" Required env vars: {tier.upper()}_BUCKET, {tier.upper()}_ACCESS_KEY, {tier.upper()}_SECRET_KEY\n"
|
|
58
|
-
f" This workspace may only have read access to the '{tier}' tier. "
|
|
59
|
-
f"Write access requires provisioning credentials (run 'uv run steve setup env' or check workspace provisioning)."
|
|
60
|
-
)
|
|
61
|
-
self.__iceberg_catalog = None
|
|
107
|
+
self._iceberg_location_prefix = None
|
|
62
108
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
109
|
+
if os.getenv("STEVE_DIRECT_S3") == "1":
|
|
110
|
+
if not self._iceberg_location_prefix:
|
|
111
|
+
raise OSError(
|
|
112
|
+
f"TrinoStorage (STEVE_DIRECT_S3=1): missing bucket configuration for tier '{tier}' — "
|
|
113
|
+
f"{tier.upper()}_BUCKET is not set.\n"
|
|
114
|
+
f" Direct S3 mode requires explicit bucket credentials "
|
|
115
|
+
f"(run 'uv run steve setup env' or check workspace provisioning)."
|
|
116
|
+
)
|
|
66
117
|
from pyiceberg.catalog.rest import RestCatalog
|
|
67
118
|
from pyiceberg.io import load_file_io
|
|
68
119
|
from pyiceberg.table import Table
|
|
69
|
-
|
|
70
|
-
# WORKAROUND: Lakekeeper's remote S3 presigner is bypassed for external access.
|
|
71
|
-
#
|
|
72
|
-
# Lakekeeper returns `s3.signer` / `s3.signer.uri` in the table config, which
|
|
73
|
-
# pyiceberg uses to sign S3 requests via Lakekeeper's signing endpoint. This works
|
|
74
|
-
# inside the cluster but fails from a laptop because:
|
|
75
|
-
# 1. The signed URI Lakekeeper produces contains the cluster-internal hostname
|
|
76
|
-
# (e.g. http://minio:9000) which is unreachable externally.
|
|
77
|
-
# 2. Even after overriding the signer URI to the public lakekeeper subdomain,
|
|
78
|
-
# Lakekeeper's signer returns HTTP 500 for URIs with the public minio hostname
|
|
79
|
-
# because it only knows the internal minio endpoint.
|
|
80
|
-
#
|
|
81
|
-
# Current workaround: strip all signer config from the merged table properties and
|
|
82
|
-
# use direct S3 credentials (access-key + secret-key) with the public minio endpoint
|
|
83
|
-
# from ~/.steve/credentials.json. This bypasses Lakekeeper's access control for S3
|
|
84
|
-
# writes — credentials live on disk rather than being vended by the cluster.
|
|
85
|
-
#
|
|
86
|
-
# To revert: configure Lakekeeper with its public S3 endpoint so presigned URIs
|
|
87
|
-
# contain a resolvable hostname, then remove the merged.pop() calls below and
|
|
88
|
-
# restore `s3.signer.uri: lakekeeper_base` in s3_overrides.
|
|
89
|
-
tier = self._tier.upper()
|
|
90
|
-
lakekeeper_base = (self._lakekeeper_endpoint or "http://lakekeeper:8181/catalog").rstrip("/")
|
|
91
120
|
from steve_cli.auth import get_s3_endpoint as _get_s3_ep
|
|
121
|
+
_tier = tier.upper()
|
|
92
122
|
_s3_ep = _get_s3_ep(self._workspace_id) or os.getenv("S3_ENDPOINT", "http://minio:9000")
|
|
93
|
-
|
|
123
|
+
_s3_overrides = {
|
|
94
124
|
"s3.endpoint": _s3_ep,
|
|
95
|
-
"s3.access-key-id": os.getenv(f"{
|
|
96
|
-
"s3.secret-access-key": os.getenv(f"{
|
|
125
|
+
"s3.access-key-id": os.getenv(f"{_tier}_ACCESS_KEY") or os.getenv("AWS_ACCESS_KEY_ID", ""),
|
|
126
|
+
"s3.secret-access-key": os.getenv(f"{_tier}_SECRET_KEY") or os.getenv("AWS_SECRET_ACCESS_KEY", ""),
|
|
97
127
|
"s3.path-style-access": "true",
|
|
98
128
|
} if self._lakekeeper_endpoint else {}
|
|
129
|
+
_base = self._lakekeeper_base
|
|
99
130
|
|
|
100
131
|
class _PatchedRestCatalog(RestCatalog):
|
|
101
132
|
def _response_to_table(self_, identifier_tuple, table_response):
|
|
102
|
-
merged = {**table_response.metadata.properties, **table_response.config, **
|
|
103
|
-
# Strip remote-signer config — see WORKAROUND comment above.
|
|
133
|
+
merged = {**table_response.metadata.properties, **table_response.config, **_s3_overrides, "uri": _base}
|
|
104
134
|
merged.pop("s3.signer", None)
|
|
105
135
|
merged.pop("s3.signer.uri", None)
|
|
106
136
|
merged.pop("s3.signer.endpoint", None)
|
|
@@ -113,27 +143,40 @@ class TrinoStorage:
|
|
|
113
143
|
config=table_response.config,
|
|
114
144
|
)
|
|
115
145
|
|
|
146
|
+
self._catalog_cls = _PatchedRestCatalog
|
|
147
|
+
self._extra_catalog_kwargs: dict = {**_s3_overrides}
|
|
148
|
+
else:
|
|
149
|
+
self._catalog_cls = None
|
|
150
|
+
self._extra_catalog_kwargs = {}
|
|
151
|
+
|
|
152
|
+
self.__iceberg_catalog = None
|
|
153
|
+
|
|
154
|
+
@property
|
|
155
|
+
def _iceberg_catalog(self):
|
|
156
|
+
if self.__iceberg_catalog is None:
|
|
157
|
+
from pyiceberg.catalog.rest import RestCatalog
|
|
116
158
|
from steve_cli.auth import get_token
|
|
117
159
|
stp_token = get_token(self._workspace_id)
|
|
118
160
|
token_kwargs = {"token": stp_token} if stp_token else {}
|
|
161
|
+
catalog_cls = self._catalog_cls or RestCatalog
|
|
119
162
|
try:
|
|
120
|
-
catalog =
|
|
163
|
+
catalog = catalog_cls(
|
|
121
164
|
"lakekeeper",
|
|
122
|
-
uri=
|
|
165
|
+
uri=self._lakekeeper_base,
|
|
123
166
|
warehouse=self._lakekeeper_warehouse,
|
|
124
|
-
**
|
|
167
|
+
**self._extra_catalog_kwargs,
|
|
125
168
|
**token_kwargs,
|
|
126
169
|
)
|
|
127
170
|
except Exception as exc:
|
|
128
171
|
msg = str(exc)
|
|
129
172
|
if "NoSuchWarehouseException" in msg or "does not exist" in msg:
|
|
130
173
|
raise RuntimeError(
|
|
131
|
-
f"Lakekeeper warehouse '{self._lakekeeper_warehouse}' not found at {
|
|
174
|
+
f"Lakekeeper warehouse '{self._lakekeeper_warehouse}' not found at {self._lakekeeper_base}.\n"
|
|
132
175
|
f"The workspace may not have been provisioned yet — ensure the workspace was created in the data mesh before running this job."
|
|
133
176
|
) from None
|
|
134
177
|
raise
|
|
135
|
-
if self._lakekeeper_endpoint:
|
|
136
|
-
catalog.uri =
|
|
178
|
+
if self._catalog_cls and self._lakekeeper_endpoint:
|
|
179
|
+
catalog.uri = self._lakekeeper_base
|
|
137
180
|
self.__iceberg_catalog = catalog
|
|
138
181
|
return self.__iceberg_catalog
|
|
139
182
|
|
|
@@ -277,8 +320,7 @@ class TrinoStorage:
|
|
|
277
320
|
import datetime
|
|
278
321
|
|
|
279
322
|
from pyiceberg.exceptions import NamespaceAlreadyExistsError
|
|
280
|
-
|
|
281
|
-
namespace_props = {"location": self._iceberg_location_prefix}
|
|
323
|
+
namespace_props = {"location": self._iceberg_location_prefix} if self._iceberg_location_prefix else {}
|
|
282
324
|
created = False
|
|
283
325
|
try:
|
|
284
326
|
self._iceberg_catalog.create_namespace(self.schema, properties=namespace_props)
|
|
@@ -291,12 +333,12 @@ class TrinoStorage:
|
|
|
291
333
|
if "403" in msg or "Access Denied" in msg or "Forbidden" in msg:
|
|
292
334
|
raise OSError(
|
|
293
335
|
f"TrinoStorage: namespace '{self.schema}' creation denied at Lakekeeper "
|
|
294
|
-
f"[{
|
|
336
|
+
f"[{self._lakekeeper_base}] — check that the token has write access to warehouse "
|
|
295
337
|
f"'{self._lakekeeper_warehouse}'"
|
|
296
338
|
) from exc
|
|
297
339
|
raise OSError(
|
|
298
340
|
f"TrinoStorage: namespace '{self.schema}' creation failed at Lakekeeper "
|
|
299
|
-
f"[{
|
|
341
|
+
f"[{self._lakekeeper_base}] — {type(exc).__name__}: {exc}"
|
|
300
342
|
) from exc
|
|
301
343
|
|
|
302
344
|
if created:
|
|
@@ -316,7 +358,7 @@ class TrinoStorage:
|
|
|
316
358
|
"branch": [self._branch],
|
|
317
359
|
"tier": [self._tier],
|
|
318
360
|
"schema_name": [self.schema],
|
|
319
|
-
"iceberg_location": [self._iceberg_location_prefix],
|
|
361
|
+
"iceberg_location": [self._iceberg_location_prefix or ""],
|
|
320
362
|
"created_at": [datetime.datetime.now(datetime.timezone.utc).isoformat()],
|
|
321
363
|
}))
|
|
322
364
|
logger.info("Created _meta table in namespace %s", self.schema)
|
|
@@ -324,39 +366,101 @@ class TrinoStorage:
|
|
|
324
366
|
logger.warning("Could not create _meta table in %s: %s", self.schema, exc)
|
|
325
367
|
|
|
326
368
|
try:
|
|
369
|
+
_location_clause = f" WITH (location = '{self._iceberg_location_prefix}')" if self._iceberg_location_prefix else ""
|
|
327
370
|
self._execute(
|
|
328
|
-
f'CREATE SCHEMA IF NOT EXISTS "{self.catalog}".{self.schema}
|
|
329
|
-
f"WITH (location = '{self._iceberg_location_prefix}')"
|
|
371
|
+
f'CREATE SCHEMA IF NOT EXISTS "{self.catalog}".{self.schema}{_location_clause}'
|
|
330
372
|
)
|
|
331
373
|
logger.debug("Ensured Trino schema %s.%s", self.catalog, self.schema)
|
|
332
374
|
except Exception as exc:
|
|
333
375
|
logger.warning("Could not ensure Trino schema %s.%s: %s", self.catalog, self.schema, exc)
|
|
334
376
|
|
|
335
377
|
def put_bytes(self, data: bytes, path: str) -> None:
|
|
336
|
-
from pyiceberg.exceptions import NoSuchTableError
|
|
378
|
+
from pyiceberg.exceptions import NoSuchTableError, SignError
|
|
337
379
|
|
|
338
380
|
arrow_table = pq.read_table(io.BytesIO(data))
|
|
339
381
|
table_id = f"{self.schema}.{self._table_name(path)}"
|
|
340
382
|
catalog = self._iceberg_catalog
|
|
341
|
-
lakekeeper_base = self.
|
|
383
|
+
lakekeeper_base = self._lakekeeper_base
|
|
342
384
|
|
|
343
385
|
self._ensure_namespace()
|
|
344
386
|
|
|
345
387
|
def _append(iceberg_table, arrow_table, table_id):
|
|
388
|
+
if os.getenv("STEVE_TRINO_TRACE") == "1":
|
|
389
|
+
io_props = getattr(iceberg_table.io, "properties", {})
|
|
390
|
+
logger.warning(
|
|
391
|
+
"[trino] append: table=%s, location=%s, s3.endpoint=%s, signer.uri=%s",
|
|
392
|
+
table_id,
|
|
393
|
+
iceberg_table.location(),
|
|
394
|
+
io_props.get("s3.endpoint", "n/a"),
|
|
395
|
+
io_props.get("s3.signer.uri", "n/a"),
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
s3_client = None
|
|
399
|
+
if os.getenv("STEVE_TRINO_TRACE") == "1":
|
|
400
|
+
try:
|
|
401
|
+
import threading as _threading
|
|
402
|
+
s3_fs = iceberg_table.io._get_fs("s3")
|
|
403
|
+
s3_client = s3_fs.s3
|
|
404
|
+
|
|
405
|
+
def _trace_sign(request, **kwargs):
|
|
406
|
+
logger.warning("[trino] s3 SIGN %s req_id=%x (thread=%s)",
|
|
407
|
+
request.method, id(request),
|
|
408
|
+
_threading.current_thread().name)
|
|
409
|
+
|
|
410
|
+
def _trace_send(request, **kwargs):
|
|
411
|
+
hdrs = dict(getattr(request, "headers", {}))
|
|
412
|
+
auth = next((v for k, v in hdrs.items() if k.lower() == "authorization"), "<MISSING>")
|
|
413
|
+
logger.warning("[trino] s3.%s req_id=%x auth=%s (thread=%s)",
|
|
414
|
+
request.method, id(request),
|
|
415
|
+
auth[:24] if auth != "<MISSING>" else "<MISSING>",
|
|
416
|
+
_threading.current_thread().name)
|
|
417
|
+
|
|
418
|
+
def _trace_retry(response=None, operation=None, request_dict=None, **kwargs):
|
|
419
|
+
if not response:
|
|
420
|
+
return
|
|
421
|
+
status = getattr(response[0], "status_code", "?")
|
|
422
|
+
if status >= 300:
|
|
423
|
+
url = (request_dict or {}).get("url", "?")
|
|
424
|
+
logger.warning("[trino] s3 <- %s %s %s", operation or "?", status, url)
|
|
425
|
+
|
|
426
|
+
s3_client.meta.events.register("before-sign.s3", _trace_sign, unique_id="steve-trino-s3-sign")
|
|
427
|
+
s3_client.meta.events.register("before-send.s3", _trace_send, unique_id="steve-trino-s3-trace")
|
|
428
|
+
s3_client.meta.events.register("needs-retry.s3", _trace_retry, unique_id="steve-trino-s3-retry")
|
|
429
|
+
except Exception as hook_exc:
|
|
430
|
+
logger.warning("[trino] TRACE hook failed: %s", hook_exc)
|
|
431
|
+
s3_client = None
|
|
432
|
+
|
|
346
433
|
try:
|
|
347
434
|
iceberg_table.append(arrow_table)
|
|
435
|
+
except SignError as exc:
|
|
436
|
+
raise OSError(
|
|
437
|
+
f"TrinoStorage: Lakekeeper refused to sign write for '{table_id}' "
|
|
438
|
+
f"[signer={lakekeeper_base}, warehouse={self._lakekeeper_warehouse}] — {exc}\n"
|
|
439
|
+
f" Hint: the token may lack write permissions on this warehouse."
|
|
440
|
+
) from exc
|
|
348
441
|
except Exception as exc:
|
|
349
442
|
msg = str(exc)
|
|
350
443
|
if "403" in msg or "Access Denied" in msg or "Forbidden" in msg:
|
|
351
444
|
raise OSError(
|
|
352
|
-
f"TrinoStorage: write to '{table_id}' denied
|
|
353
|
-
f"
|
|
354
|
-
f"(
|
|
445
|
+
f"TrinoStorage: write to '{table_id}' denied by MinIO "
|
|
446
|
+
f"[location={iceberg_table.location()}, warehouse={self._lakekeeper_warehouse}] — "
|
|
447
|
+
f"{type(exc).__name__}: {exc}"
|
|
355
448
|
) from exc
|
|
356
449
|
raise OSError(
|
|
357
450
|
f"TrinoStorage: write to '{table_id}' failed via Lakekeeper [{lakekeeper_base}] — "
|
|
358
451
|
f"{type(exc).__name__}: {exc}"
|
|
359
452
|
) from exc
|
|
453
|
+
finally:
|
|
454
|
+
if s3_client is not None:
|
|
455
|
+
for event, uid in (
|
|
456
|
+
("before-sign.s3", "steve-trino-s3-sign"),
|
|
457
|
+
("before-send.s3", "steve-trino-s3-trace"),
|
|
458
|
+
("needs-retry.s3", "steve-trino-s3-retry"),
|
|
459
|
+
):
|
|
460
|
+
try:
|
|
461
|
+
s3_client.meta.events.unregister(event, unique_id=uid)
|
|
462
|
+
except Exception:
|
|
463
|
+
pass
|
|
360
464
|
|
|
361
465
|
try:
|
|
362
466
|
iceberg_table = catalog.load_table(table_id)
|
|
@@ -399,13 +503,19 @@ class TrinoStorage:
|
|
|
399
503
|
iceberg_schema = Schema(*fields)
|
|
400
504
|
try:
|
|
401
505
|
iceberg_table = catalog.create_table(table_id, schema=iceberg_schema)
|
|
506
|
+
except SignError as exc:
|
|
507
|
+
raise OSError(
|
|
508
|
+
f"TrinoStorage: Lakekeeper refused to sign table creation for '{table_id}' "
|
|
509
|
+
f"[signer={lakekeeper_base}, warehouse={self._lakekeeper_warehouse}] — {exc}\n"
|
|
510
|
+
f" Hint: the token may lack write permissions on this warehouse."
|
|
511
|
+
) from exc
|
|
402
512
|
except Exception as exc:
|
|
403
513
|
msg = str(exc)
|
|
404
514
|
if "403" in msg or "Access Denied" in msg or "Forbidden" in msg:
|
|
405
515
|
raise OSError(
|
|
406
|
-
f"TrinoStorage: create table '{table_id}' denied
|
|
407
|
-
f"
|
|
408
|
-
f"(
|
|
516
|
+
f"TrinoStorage: create table '{table_id}' denied by MinIO "
|
|
517
|
+
f"[location={self._iceberg_location_prefix or '<lakekeeper-managed>'}, warehouse={self._lakekeeper_warehouse}] — "
|
|
518
|
+
f"{type(exc).__name__}: {exc}"
|
|
409
519
|
) from exc
|
|
410
520
|
raise OSError(
|
|
411
521
|
f"TrinoStorage: create table '{table_id}' failed via Lakekeeper [{lakekeeper_base}] — "
|
|
@@ -422,7 +532,7 @@ class TrinoStorage:
|
|
|
422
532
|
logger.warning(
|
|
423
533
|
"[trino] put_bytes: wrote %d rows → %s.%s (warehouse=%s, location=%s%s)",
|
|
424
534
|
len(arrow_table), self.catalog, table_id,
|
|
425
|
-
self._lakekeeper_warehouse, self._iceberg_location_prefix,
|
|
535
|
+
self._lakekeeper_warehouse, self._iceberg_location_prefix or "<lakekeeper-managed>",
|
|
426
536
|
f", snapshot={snapshot.snapshot_id}" if snapshot else "",
|
|
427
537
|
)
|
|
428
538
|
logger.info("Written %d rows to %s", len(arrow_table), table_id)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: steve-cli
|
|
3
|
-
Version: 1.1.
|
|
3
|
+
Version: 1.1.3
|
|
4
4
|
Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
|
|
5
5
|
Author: Frank
|
|
6
6
|
License: MIT
|
|
@@ -37,6 +37,8 @@ Requires-Dist: pandas>=2.0.3; extra == "pandas"
|
|
|
37
37
|
Requires-Dist: pyarrow>=17.0.0; extra == "pandas"
|
|
38
38
|
Provides-Extra: lineage
|
|
39
39
|
Requires-Dist: openlineage-python>=1.0.0; extra == "lineage"
|
|
40
|
+
Provides-Extra: lakekeeper
|
|
41
|
+
Requires-Dist: pylakekeeper>=0.4.0; extra == "lakekeeper"
|
|
40
42
|
Provides-Extra: validoopsie
|
|
41
43
|
Requires-Dist: validoopsie>=0.1.0; extra == "validoopsie"
|
|
42
44
|
Requires-Dist: polars>=1.8.2; extra == "validoopsie"
|
|
@@ -49,7 +51,7 @@ Provides-Extra: trino
|
|
|
49
51
|
Requires-Dist: trino>=0.330; extra == "trino"
|
|
50
52
|
Requires-Dist: pyarrow>=17.0.0; extra == "trino"
|
|
51
53
|
Requires-Dist: requests>=2.31.0; extra == "trino"
|
|
52
|
-
Requires-Dist: pyiceberg>=0.
|
|
54
|
+
Requires-Dist: pyiceberg>=0.12.0; extra == "trino"
|
|
53
55
|
Requires-Dist: s3fs>=2024.1.0; extra == "trino"
|
|
54
56
|
Provides-Extra: visidata
|
|
55
57
|
Requires-Dist: visidata>=3.0; extra == "visidata"
|
|
@@ -31,6 +31,9 @@ openpyxl>=3.1.0
|
|
|
31
31
|
great-expectations>=0.18.0
|
|
32
32
|
pandas>=2.0.3
|
|
33
33
|
|
|
34
|
+
[lakekeeper]
|
|
35
|
+
pylakekeeper>=0.4.0
|
|
36
|
+
|
|
34
37
|
[lineage]
|
|
35
38
|
openlineage-python>=1.0.0
|
|
36
39
|
|
|
@@ -46,7 +49,7 @@ pyarrow>=17.0.0
|
|
|
46
49
|
trino>=0.330
|
|
47
50
|
pyarrow>=17.0.0
|
|
48
51
|
requests>=2.31.0
|
|
49
|
-
pyiceberg>=0.
|
|
52
|
+
pyiceberg>=0.12.0
|
|
50
53
|
s3fs>=2024.1.0
|
|
51
54
|
|
|
52
55
|
[validoopsie]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|