steve-cli 1.1.4__tar.gz → 1.1.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {steve_cli-1.1.4 → steve_cli-1.1.6}/PKG-INFO +1 -1
- {steve_cli-1.1.4 → steve_cli-1.1.6}/pyproject.toml +1 -1
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/storage/__init__.py +5 -0
- steve_cli-1.1.6/steve_cli/storage/auth.py +65 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/storage/trino.py +40 -9
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli.egg-info/PKG-INFO +1 -1
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli.egg-info/SOURCES.txt +1 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/README.md +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/setup.cfg +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/__init__.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/auth.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/cli.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/decorators/__init__.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/decorators/lineage_job.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/lineage/__init__.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/lineage/adapters/__init__.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/lineage/adapters/composite.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/lineage/adapters/dataproductregistry.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/lineage/adapters/logging.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/lineage/adapters/null.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/lineage/adapters/openlineage.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/lineage/collector.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/lineage/port.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/lineage/registry.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/lineage/storage.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/ontology.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/policies/__init__.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/policies/client.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/storage/branch.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/storage/lakekeeper.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/storage/metadata/__init__.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/storage/metadata/extractors/__init__.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/storage/metadata/extractors/csv.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/storage/metadata/extractors/excel.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/storage/metadata/extractors/generic.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/storage/metadata/extractors/json.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/storage/metadata/extractors/parquet.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/storage/metadata/port.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/storage/metadata/registry.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/storage/parquet.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/storage/protocol.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/storage/s3.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/storage.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/validation/__init__.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/validation/adapters/__init__.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/validation/adapters/great_expectations.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/validation/adapters/null.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/validation/adapters/validoopsie.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/validation/port.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/validation/registry.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli/vkg.py +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli.egg-info/dependency_links.txt +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli.egg-info/entry_points.txt +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli.egg-info/requires.txt +0 -0
- {steve_cli-1.1.4 → steve_cli-1.1.6}/steve_cli.egg-info/top_level.txt +0 -0
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from .auth import BearerTokenStrategy, CredentialsFileStrategy, EnvVarStrategy, TrinoAuthStrategy
|
|
1
2
|
from .branch import get_branch_prefix
|
|
2
3
|
from .protocol import Storage
|
|
3
4
|
from .s3 import S3Storage
|
|
@@ -7,6 +8,10 @@ from .parquet import ParquetMetadata, extract_parquet_metadata
|
|
|
7
8
|
from .metadata import FileMetadata, ColumnMetadata, MetadataExtractorPort, MetadataRegistry
|
|
8
9
|
|
|
9
10
|
__all__ = [
|
|
11
|
+
"BearerTokenStrategy",
|
|
12
|
+
"CredentialsFileStrategy",
|
|
13
|
+
"EnvVarStrategy",
|
|
14
|
+
"TrinoAuthStrategy",
|
|
10
15
|
"get_branch_prefix",
|
|
11
16
|
"Storage",
|
|
12
17
|
"S3Storage",
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from typing import Protocol, runtime_checkable
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@runtime_checkable
|
|
8
|
+
class TrinoAuthStrategy(Protocol):
|
|
9
|
+
def get_token(self) -> str | None: ...
|
|
10
|
+
def get_service_url(self, service: str) -> str | None: ...
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class CredentialsFileStrategy:
|
|
14
|
+
"""Default: reads token and service URLs from ~/.steve/credentials.json."""
|
|
15
|
+
|
|
16
|
+
def __init__(self, workspace_id: str | None = None) -> None:
|
|
17
|
+
self._workspace_id = workspace_id
|
|
18
|
+
|
|
19
|
+
def get_token(self) -> str | None:
|
|
20
|
+
from steve_cli.auth import get_token
|
|
21
|
+
return get_token(self._workspace_id)
|
|
22
|
+
|
|
23
|
+
def get_service_url(self, service: str) -> str | None:
|
|
24
|
+
from steve_cli.auth import get_service_url
|
|
25
|
+
return get_service_url(service, self._workspace_id)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class BearerTokenStrategy:
|
|
29
|
+
"""Explicit token + base URL for injected credentials (e.g. X-AK-Token header).
|
|
30
|
+
|
|
31
|
+
Service URLs are constructed as https://<service>-<workspace_id>.<base_url>.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def __init__(self, token: str, workspace_id: str, base_url: str) -> None:
|
|
35
|
+
self._token = token
|
|
36
|
+
self._workspace_id = workspace_id
|
|
37
|
+
self._base_url = base_url.rstrip("/")
|
|
38
|
+
|
|
39
|
+
def get_token(self) -> str | None:
|
|
40
|
+
return self._token or None
|
|
41
|
+
|
|
42
|
+
def get_service_url(self, service: str) -> str | None:
|
|
43
|
+
if not self._workspace_id or not self._base_url:
|
|
44
|
+
return None
|
|
45
|
+
return f"https://{service}-{self._workspace_id}.{self._base_url}"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class EnvVarStrategy:
|
|
49
|
+
"""Reads TRINO_TOKEN + TRINO_URL (or STEVE_BASE_URL + WORKSPACE_ID) from env.
|
|
50
|
+
|
|
51
|
+
Useful in containers where credentials are injected as env vars rather than a file.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
def get_token(self) -> str | None:
|
|
55
|
+
return os.getenv("TRINO_TOKEN") or None
|
|
56
|
+
|
|
57
|
+
def get_service_url(self, service: str) -> str | None:
|
|
58
|
+
explicit = os.getenv(f"{service.upper()}_URL")
|
|
59
|
+
if explicit:
|
|
60
|
+
return explicit
|
|
61
|
+
workspace_id = os.getenv("WORKSPACE_ID")
|
|
62
|
+
base_url = os.getenv("STEVE_BASE_URL", "").rstrip("/")
|
|
63
|
+
if workspace_id and base_url:
|
|
64
|
+
return f"https://{service}-{workspace_id}.{base_url}"
|
|
65
|
+
return None
|
|
@@ -10,6 +10,8 @@ import pyarrow as pa
|
|
|
10
10
|
import pyarrow.parquet as pq
|
|
11
11
|
import requests
|
|
12
12
|
|
|
13
|
+
from steve_cli.storage.auth import CredentialsFileStrategy, TrinoAuthStrategy
|
|
14
|
+
|
|
13
15
|
from .branch import get_branch_prefix
|
|
14
16
|
|
|
15
17
|
logger = logging.getLogger(__name__)
|
|
@@ -33,6 +35,7 @@ def _patch_pyiceberg_get_fs_once() -> None:
|
|
|
33
35
|
|
|
34
36
|
try:
|
|
35
37
|
import threading as _t
|
|
38
|
+
|
|
36
39
|
from pyiceberg.io.fsspec import FsspecFileIO
|
|
37
40
|
|
|
38
41
|
_caches: dict[int, dict] = {}
|
|
@@ -63,7 +66,12 @@ _patch_pyiceberg_get_fs_once()
|
|
|
63
66
|
|
|
64
67
|
|
|
65
68
|
class TrinoStorage:
|
|
66
|
-
def __init__(
|
|
69
|
+
def __init__(
|
|
70
|
+
self,
|
|
71
|
+
tier: str = "bronze",
|
|
72
|
+
workspace: str | None = None,
|
|
73
|
+
auth: TrinoAuthStrategy | None = None,
|
|
74
|
+
):
|
|
67
75
|
# aiobotocore 3.9+ defaults to STREAMING-UNSIGNED-PAYLOAD-TRAILER (flexible checksums)
|
|
68
76
|
# which Lakekeeper's remote signing proxy and MinIO don't support.
|
|
69
77
|
os.environ.setdefault('AWS_REQUEST_CHECKSUM_CALCULATION', 'when_required')
|
|
@@ -73,11 +81,12 @@ class TrinoStorage:
|
|
|
73
81
|
# the next put_bytes call reaches the S3 write phase.
|
|
74
82
|
import aiobotocore.httpsession
|
|
75
83
|
aiobotocore.httpsession.DEFAULT_KEEPALIVE_TIMEOUT = 3
|
|
76
|
-
|
|
84
|
+
|
|
77
85
|
self._branch = get_branch_prefix()
|
|
78
86
|
self._tier = tier.lower()
|
|
79
87
|
self._workspace_id = os.getenv("WORKSPACE_ID")
|
|
80
|
-
|
|
88
|
+
self._auth: TrinoAuthStrategy = auth if auth is not None else CredentialsFileStrategy(self._workspace_id)
|
|
89
|
+
trino_endpoint = self._auth.get_service_url("trino")
|
|
81
90
|
if not trino_endpoint:
|
|
82
91
|
from steve_cli.auth import _get_workspace_entry
|
|
83
92
|
entry = _get_workspace_entry(self._workspace_id)
|
|
@@ -96,7 +105,7 @@ class TrinoStorage:
|
|
|
96
105
|
self.user = os.getenv("TRINO_USER") or resolved_workspace or "admin"
|
|
97
106
|
self._base = trino_endpoint.rstrip("/")
|
|
98
107
|
_lk_env = os.getenv("LAKEKEEPER_ENDPOINT")
|
|
99
|
-
_lk_svc = get_service_url("lakekeeper"
|
|
108
|
+
_lk_svc = self._auth.get_service_url("lakekeeper")
|
|
100
109
|
self._lakekeeper_endpoint = _lk_env.rstrip("/") if _lk_env else (_lk_svc.rstrip("/") + "/catalog" if _lk_svc else None)
|
|
101
110
|
self._lakekeeper_base = (self._lakekeeper_endpoint or "http://lakekeeper:8181/catalog").rstrip("/")
|
|
102
111
|
self._lakekeeper_warehouse = os.getenv("LAKEKEEPER_WAREHOUSE", default_catalog)
|
|
@@ -117,7 +126,9 @@ class TrinoStorage:
|
|
|
117
126
|
from pyiceberg.catalog.rest import RestCatalog
|
|
118
127
|
from pyiceberg.io import load_file_io
|
|
119
128
|
from pyiceberg.table import Table
|
|
129
|
+
|
|
120
130
|
from steve_cli.auth import get_s3_endpoint as _get_s3_ep
|
|
131
|
+
|
|
121
132
|
_tier = tier.upper()
|
|
122
133
|
_s3_ep = _get_s3_ep(self._workspace_id) or os.getenv("S3_ENDPOINT", "http://minio:9000")
|
|
123
134
|
_s3_overrides = {
|
|
@@ -155,8 +166,7 @@ class TrinoStorage:
|
|
|
155
166
|
def _iceberg_catalog(self):
|
|
156
167
|
if self.__iceberg_catalog is None:
|
|
157
168
|
from pyiceberg.catalog.rest import RestCatalog
|
|
158
|
-
|
|
159
|
-
stp_token = get_token(self._workspace_id)
|
|
169
|
+
stp_token = self._auth.get_token()
|
|
160
170
|
token_kwargs = {"token": stp_token} if stp_token else {}
|
|
161
171
|
catalog_cls = self._catalog_cls or RestCatalog
|
|
162
172
|
try:
|
|
@@ -181,17 +191,19 @@ class TrinoStorage:
|
|
|
181
191
|
return self.__iceberg_catalog
|
|
182
192
|
|
|
183
193
|
def _execute(self, sql: str) -> list[dict]:
|
|
184
|
-
from steve_cli.auth import get_token
|
|
185
194
|
trace = os.getenv("STEVE_TRINO_TRACE") == "1"
|
|
186
195
|
headers = {
|
|
187
196
|
"X-Trino-User": self.user,
|
|
188
197
|
"X-Trino-Catalog": self.catalog,
|
|
189
198
|
"X-Trino-Schema": self.schema,
|
|
190
199
|
}
|
|
191
|
-
token = get_token(
|
|
200
|
+
token = self._auth.get_token()
|
|
192
201
|
if token:
|
|
193
202
|
headers["Authorization"] = f"Bearer {token}"
|
|
194
203
|
|
|
204
|
+
if trace:
|
|
205
|
+
logger.warning(f"[trino] {self._base}" )
|
|
206
|
+
|
|
195
207
|
t0 = time.perf_counter()
|
|
196
208
|
try:
|
|
197
209
|
resp = requests.post(f"{self._base}/v1/statement", data=sql, headers=headers)
|
|
@@ -393,12 +405,31 @@ class TrinoStorage:
|
|
|
393
405
|
def _append(iceberg_table, arrow_table, table_id):
|
|
394
406
|
if os.getenv("STEVE_TRINO_TRACE") == "1":
|
|
395
407
|
io_props = getattr(iceberg_table.io, "properties", {})
|
|
408
|
+
access_key = io_props.get("s3.access-key-id", "n/a")
|
|
409
|
+
session_tok = io_props.get("s3.session-token", None)
|
|
410
|
+
# Decode JWT payload (no key needed) to see embedded session policy
|
|
411
|
+
session_policy_decoded = "none"
|
|
412
|
+
if session_tok:
|
|
413
|
+
try:
|
|
414
|
+
import base64 as _b64, json as _json
|
|
415
|
+
parts = session_tok.split(".")
|
|
416
|
+
if len(parts) >= 2:
|
|
417
|
+
payload_b64 = parts[1] + "=" * (-len(parts[1]) % 4)
|
|
418
|
+
payload = _json.loads(_b64.urlsafe_b64decode(payload_b64))
|
|
419
|
+
sp = payload.get("sessionPolicy", payload.get("Policy", payload))
|
|
420
|
+
session_policy_decoded = _json.dumps(sp)
|
|
421
|
+
except Exception as _e:
|
|
422
|
+
session_policy_decoded = f"decode-error:{_e}"
|
|
396
423
|
logger.warning(
|
|
397
|
-
"[trino] append: table=%s, location=%s, s3.endpoint=%s, signer.uri=%s"
|
|
424
|
+
"[trino] append: table=%s, location=%s, s3.endpoint=%s, signer.uri=%s, "
|
|
425
|
+
"s3.access-key-id=%s, session_policy=%s, auth=%s",
|
|
398
426
|
table_id,
|
|
399
427
|
iceberg_table.location(),
|
|
400
428
|
io_props.get("s3.endpoint", "n/a"),
|
|
401
429
|
io_props.get("s3.signer.uri", "n/a"),
|
|
430
|
+
access_key,
|
|
431
|
+
session_policy_decoded,
|
|
432
|
+
type(self._auth).__name__,
|
|
402
433
|
)
|
|
403
434
|
|
|
404
435
|
s3_client = None
|
|
@@ -28,6 +28,7 @@ steve_cli/lineage/adapters/openlineage.py
|
|
|
28
28
|
steve_cli/policies/__init__.py
|
|
29
29
|
steve_cli/policies/client.py
|
|
30
30
|
steve_cli/storage/__init__.py
|
|
31
|
+
steve_cli/storage/auth.py
|
|
31
32
|
steve_cli/storage/branch.py
|
|
32
33
|
steve_cli/storage/lakekeeper.py
|
|
33
34
|
steve_cli/storage/parquet.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|