steve-cli 1.1.2__tar.gz → 1.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {steve_cli-1.1.2 → steve_cli-1.1.4}/PKG-INFO +3 -1
- {steve_cli-1.1.2 → steve_cli-1.1.4}/pyproject.toml +4 -1
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/cli.py +23 -8
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/decorators/lineage_job.py +8 -1
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/policies/client.py +10 -7
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/__init__.py +2 -0
- steve_cli-1.1.4/steve_cli/storage/lakekeeper.py +235 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/trino.py +6 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli.egg-info/PKG-INFO +3 -1
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli.egg-info/SOURCES.txt +1 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli.egg-info/requires.txt +3 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/README.md +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/setup.cfg +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/__init__.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/auth.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/decorators/__init__.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/__init__.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/adapters/__init__.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/adapters/composite.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/adapters/dataproductregistry.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/adapters/logging.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/adapters/null.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/adapters/openlineage.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/collector.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/port.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/registry.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/storage.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/ontology.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/policies/__init__.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/branch.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/__init__.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/extractors/__init__.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/extractors/csv.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/extractors/excel.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/extractors/generic.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/extractors/json.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/extractors/parquet.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/port.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/registry.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/parquet.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/protocol.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/s3.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/validation/__init__.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/validation/adapters/__init__.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/validation/adapters/great_expectations.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/validation/adapters/null.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/validation/adapters/validoopsie.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/validation/port.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/validation/registry.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/vkg.py +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli.egg-info/dependency_links.txt +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli.egg-info/entry_points.txt +0 -0
- {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: steve-cli
|
|
3
|
-
Version: 1.1.
|
|
3
|
+
Version: 1.1.4
|
|
4
4
|
Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
|
|
5
5
|
Author: Frank
|
|
6
6
|
License: MIT
|
|
@@ -37,6 +37,8 @@ Requires-Dist: pandas>=2.0.3; extra == "pandas"
|
|
|
37
37
|
Requires-Dist: pyarrow>=17.0.0; extra == "pandas"
|
|
38
38
|
Provides-Extra: lineage
|
|
39
39
|
Requires-Dist: openlineage-python>=1.0.0; extra == "lineage"
|
|
40
|
+
Provides-Extra: lakekeeper
|
|
41
|
+
Requires-Dist: pylakekeeper>=0.4.0; extra == "lakekeeper"
|
|
40
42
|
Provides-Extra: validoopsie
|
|
41
43
|
Requires-Dist: validoopsie>=0.1.0; extra == "validoopsie"
|
|
42
44
|
Requires-Dist: polars>=1.8.2; extra == "validoopsie"
|
|
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
|
|
|
5
5
|
|
|
6
6
|
[project]
|
|
7
7
|
name = "steve-cli"
|
|
8
|
-
version = "1.1.
|
|
8
|
+
version = "1.1.4"
|
|
9
9
|
description = "A simple CLI tool to run jobs from jobs.yaml with proper environment setup"
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
license = {text = "MIT"}
|
|
@@ -50,6 +50,9 @@ pandas = [
|
|
|
50
50
|
lineage = [
|
|
51
51
|
"openlineage-python>=1.0.0",
|
|
52
52
|
]
|
|
53
|
+
lakekeeper = [
|
|
54
|
+
"pylakekeeper>=0.4.0",
|
|
55
|
+
]
|
|
53
56
|
validoopsie = [
|
|
54
57
|
"validoopsie>=0.1.0",
|
|
55
58
|
"polars>=1.8.2",
|
|
@@ -59,8 +59,24 @@ class JobsConfig:
|
|
|
59
59
|
return job
|
|
60
60
|
return None
|
|
61
61
|
|
|
62
|
-
def
|
|
63
|
-
|
|
62
|
+
def _job_depths(self) -> dict[str, int]:
|
|
63
|
+
depths: dict[str, int] = {}
|
|
64
|
+
for job in self.jobs:
|
|
65
|
+
name = job.get("name", "")
|
|
66
|
+
parents = job.get("dependsOn", [])
|
|
67
|
+
depths[name] = max((depths.get(p, 0) for p in parents), default=-1) + 1
|
|
68
|
+
return depths
|
|
69
|
+
|
|
70
|
+
def list_jobs(self) -> List[questionary.Choice]:
|
|
71
|
+
depths = self._job_depths()
|
|
72
|
+
choices = []
|
|
73
|
+
for job in self.jobs:
|
|
74
|
+
name = job.get("name", "unnamed")
|
|
75
|
+
depth = depths.get(name, 0)
|
|
76
|
+
indent = " " * depth
|
|
77
|
+
prefix = "└─ " if depth > 0 else ""
|
|
78
|
+
choices.append(questionary.Choice(title=f"{indent}{prefix}{name}", value=name))
|
|
79
|
+
return choices
|
|
64
80
|
|
|
65
81
|
|
|
66
82
|
def run_job_command(job: Dict[str, Any]) -> int:
|
|
@@ -120,13 +136,13 @@ def jobs(ctx: click.Context, jobs_file: Optional[Path]):
|
|
|
120
136
|
if ctx.invoked_subcommand is not None:
|
|
121
137
|
return
|
|
122
138
|
config = JobsConfig(jobs_file)
|
|
123
|
-
|
|
124
|
-
if not
|
|
139
|
+
choices = config.list_jobs()
|
|
140
|
+
if not choices:
|
|
125
141
|
click.echo("❌ No jobs found in jobs.yaml")
|
|
126
142
|
return
|
|
127
143
|
choice = questionary.select(
|
|
128
144
|
"Select a job to run:",
|
|
129
|
-
choices=
|
|
145
|
+
choices=choices,
|
|
130
146
|
).ask()
|
|
131
147
|
if choice is None:
|
|
132
148
|
sys.exit(0)
|
|
@@ -140,9 +156,8 @@ def jobs(ctx: click.Context, jobs_file: Optional[Path]):
|
|
|
140
156
|
def jobs_list(jobs_file: Optional[Path]):
|
|
141
157
|
"""List all available jobs."""
|
|
142
158
|
config = JobsConfig(jobs_file)
|
|
143
|
-
job_names = config.list_jobs()
|
|
144
159
|
|
|
145
|
-
if not
|
|
160
|
+
if not config.jobs:
|
|
146
161
|
click.echo("❌ No jobs found in jobs.yaml")
|
|
147
162
|
return
|
|
148
163
|
|
|
@@ -951,7 +966,7 @@ def policies_apply(file: Path | None, validate: str | None):
|
|
|
951
966
|
policies_file = Path(file) if file else None
|
|
952
967
|
try:
|
|
953
968
|
branch, sanitized = PolicyClient().apply_from_file(policies_file, validate=validate)
|
|
954
|
-
click.secho(f"Branch: {branch} → schema
|
|
969
|
+
click.secho(f"Branch: {branch} → schema: {sanitized}", fg="cyan")
|
|
955
970
|
click.secho("Policies applied successfully.", fg="green")
|
|
956
971
|
except FileNotFoundError as e:
|
|
957
972
|
click.secho(str(e), fg="red", err=True)
|
|
@@ -55,8 +55,15 @@ def lineage_job(
|
|
|
55
55
|
session=session,
|
|
56
56
|
)
|
|
57
57
|
|
|
58
|
+
def get_files(tier: str = "bronze", workspace: str | None = None) -> LineageStorage:
|
|
59
|
+
from steve_cli.storage.lakekeeper import LakekeeperStorage
|
|
60
|
+
return LineageStorage(
|
|
61
|
+
storage=lambda: LakekeeperStorage(tier=tier, workspace=workspace),
|
|
62
|
+
session=session,
|
|
63
|
+
)
|
|
64
|
+
|
|
58
65
|
sig = inspect.signature(fn)
|
|
59
|
-
available = {"get_storage": get_storage, "get_tables": get_tables}
|
|
66
|
+
available = {"get_storage": get_storage, "get_tables": get_tables, "get_files": get_files}
|
|
60
67
|
injectable = {k: v for k, v in available.items() if k in sig.parameters}
|
|
61
68
|
|
|
62
69
|
try:
|
|
@@ -33,6 +33,9 @@ def _expand_rules(rules: list[dict[str, Any]], sanitized_branch: str, workspace:
|
|
|
33
33
|
for entry in rule.get("columns", []):
|
|
34
34
|
if "schema" in entry:
|
|
35
35
|
entry["schema"] = _expand_schema(entry["schema"], sanitized_branch)
|
|
36
|
+
if workspace and "catalogs" not in entry:
|
|
37
|
+
tier = entry.get("tier", "bronze")
|
|
38
|
+
entry["catalogs"] = [f"{workspace}-{tier}"]
|
|
36
39
|
expanded.append(rule)
|
|
37
40
|
return expanded
|
|
38
41
|
|
|
@@ -122,20 +125,20 @@ class PolicyClient:
|
|
|
122
125
|
workspace_id = os.environ.get("WORKSPACE_ID")
|
|
123
126
|
workspace_name = os.environ.get("WORKSPACE_NAME", "")
|
|
124
127
|
|
|
125
|
-
seen: dict[str, list[str]] = {}
|
|
128
|
+
seen: dict[tuple[str, str], list[str]] = {} # (schema, tier) → tables
|
|
126
129
|
for rule in rules:
|
|
127
130
|
for entry in list(rule.get("tables", [])) + list(rule.get("columns", [])):
|
|
128
131
|
schema = entry.get("schema", "")
|
|
129
132
|
table = entry.get("table", "")
|
|
133
|
+
tier = entry.get("tier", "bronze")
|
|
130
134
|
if schema and table:
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
135
|
+
key = (schema, tier)
|
|
136
|
+
seen.setdefault(key, [])
|
|
137
|
+
if table not in seen[key]:
|
|
138
|
+
seen[key].append(table)
|
|
134
139
|
|
|
135
140
|
results: list[str] = []
|
|
136
|
-
for schema, tables in seen.items():
|
|
137
|
-
tier = "bronze"
|
|
138
|
-
ws_clean = workspace_name.lower().replace("-", "_") if workspace_name else ""
|
|
141
|
+
for (schema, tier), tables in seen.items():
|
|
139
142
|
catalog = f"{workspace_name}-{tier}" if workspace_name else f"minio-{tier}"
|
|
140
143
|
|
|
141
144
|
try:
|
|
@@ -2,6 +2,7 @@ from .branch import get_branch_prefix
|
|
|
2
2
|
from .protocol import Storage
|
|
3
3
|
from .s3 import S3Storage
|
|
4
4
|
from .trino import TrinoStorage
|
|
5
|
+
from .lakekeeper import LakekeeperStorage
|
|
5
6
|
from .parquet import ParquetMetadata, extract_parquet_metadata
|
|
6
7
|
from .metadata import FileMetadata, ColumnMetadata, MetadataExtractorPort, MetadataRegistry
|
|
7
8
|
|
|
@@ -10,6 +11,7 @@ __all__ = [
|
|
|
10
11
|
"Storage",
|
|
11
12
|
"S3Storage",
|
|
12
13
|
"TrinoStorage",
|
|
14
|
+
"LakekeeperStorage",
|
|
13
15
|
"ParquetMetadata",
|
|
14
16
|
"extract_parquet_metadata",
|
|
15
17
|
"FileMetadata",
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path, PurePosixPath
|
|
7
|
+
|
|
8
|
+
import requests
|
|
9
|
+
|
|
10
|
+
from .branch import get_branch_prefix
|
|
11
|
+
from steve_cli.auth import get_service_url, get_token
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
_UUID_RE = re.compile(
|
|
16
|
+
r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
_SIGNER_SSL_PATCHED = False
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _patch_s3v4_signer_no_ssl_verify() -> None:
|
|
23
|
+
"""Replace SIGNERS['S3V4RestSigner'] with a subclass whose requests.Session has verify=False.
|
|
24
|
+
|
|
25
|
+
Needed when the Lakekeeper signing endpoint uses a cert with a hostname mismatch.
|
|
26
|
+
Mirrors the pyiceberg get_fs monkey-patch in trino.py — applied once, process-wide.
|
|
27
|
+
"""
|
|
28
|
+
global _SIGNER_SSL_PATCHED
|
|
29
|
+
if _SIGNER_SSL_PATCHED:
|
|
30
|
+
return
|
|
31
|
+
try:
|
|
32
|
+
from pyiceberg.io.fsspec import SIGNERS, S3V4RestSigner # type: ignore[attr-defined]
|
|
33
|
+
orig = SIGNERS.get("S3V4RestSigner", S3V4RestSigner)
|
|
34
|
+
|
|
35
|
+
class _NoVerify(orig): # type: ignore[valid-type]
|
|
36
|
+
def __init__(self, props): # type: ignore[override]
|
|
37
|
+
super().__init__(props)
|
|
38
|
+
self._session.verify = False
|
|
39
|
+
|
|
40
|
+
SIGNERS["S3V4RestSigner"] = _NoVerify
|
|
41
|
+
_SIGNER_SSL_PATCHED = True
|
|
42
|
+
logger.warning(
|
|
43
|
+
"Applied S3V4RestSigner SSL-verify=False patch (LAKEKEEPER_SSL_VERIFY=0). "
|
|
44
|
+
"Remove once the Lakekeeper TLS certificate covers the full hostname."
|
|
45
|
+
)
|
|
46
|
+
except Exception as exc:
|
|
47
|
+
logger.warning("Could not patch S3V4RestSigner SSL verify: %s", exc)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class LakekeeperStorage:
|
|
51
|
+
"""
|
|
52
|
+
Storage backend for unstructured/binary files via Lakekeeper Generic Tables (DATASET format).
|
|
53
|
+
|
|
54
|
+
This warehouse uses remote signing (s3.signer: S3V4RestSigner) — Lakekeeper signs every
|
|
55
|
+
S3 request rather than vending short-lived credentials. We therefore use pyiceberg's
|
|
56
|
+
FsspecFileIO with S3V4RestSigner, the same mechanism TrinoStorage uses for Iceberg tables.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
def __init__(self, tier: str = "bronze", workspace: str | None = None):
|
|
60
|
+
# aiobotocore 3.9+ defaults to STREAMING-UNSIGNED-PAYLOAD-TRAILER (flexible checksums)
|
|
61
|
+
# which Lakekeeper's remote signing proxy and MinIO don't support.
|
|
62
|
+
os.environ.setdefault("AWS_REQUEST_CHECKSUM_CALCULATION", "when_required")
|
|
63
|
+
import aiobotocore.httpsession
|
|
64
|
+
aiobotocore.httpsession.DEFAULT_KEEPALIVE_TIMEOUT = 3
|
|
65
|
+
|
|
66
|
+
self._tier = tier.lower()
|
|
67
|
+
self._branch = get_branch_prefix()
|
|
68
|
+
self._workspace_id = os.getenv("WORKSPACE_ID")
|
|
69
|
+
|
|
70
|
+
_lk_env = os.getenv("LAKEKEEPER_ENDPOINT")
|
|
71
|
+
_lk_svc = get_service_url("lakekeeper", self._workspace_id)
|
|
72
|
+
self._lakekeeper_base_url = (
|
|
73
|
+
_lk_env.rstrip("/") if _lk_env
|
|
74
|
+
else (_lk_svc.rstrip("/") if _lk_svc else "http://lakekeeper:8181")
|
|
75
|
+
)
|
|
76
|
+
self._token = get_token(self._workspace_id)
|
|
77
|
+
|
|
78
|
+
resolved_workspace = workspace or os.getenv("WORKSPACE_NAME")
|
|
79
|
+
default_warehouse = f"{resolved_workspace}-{tier}" if resolved_workspace else "minio"
|
|
80
|
+
self._warehouse = os.getenv("LAKEKEEPER_WAREHOUSE", default_warehouse)
|
|
81
|
+
|
|
82
|
+
self._namespace = self._branch
|
|
83
|
+
self._table = f"{self._tier}_files"
|
|
84
|
+
self._table_ensured = False
|
|
85
|
+
self.__lk_client = None
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def _ssl_verify(self) -> bool:
|
|
89
|
+
return os.getenv("LAKEKEEPER_SSL_VERIFY", "1") != "0"
|
|
90
|
+
|
|
91
|
+
def _resolve_warehouse_uuid(self) -> str:
|
|
92
|
+
"""Resolve warehouse name → UUID (generic-tables path requires a UUID, not a name)."""
|
|
93
|
+
if _UUID_RE.match(self._warehouse):
|
|
94
|
+
return self._warehouse
|
|
95
|
+
headers = {"Authorization": f"Bearer {self._token}"} if self._token else {}
|
|
96
|
+
try:
|
|
97
|
+
resp = requests.get(
|
|
98
|
+
f"{self._lakekeeper_base_url}/management/v1/warehouse",
|
|
99
|
+
headers=headers,
|
|
100
|
+
verify=self._ssl_verify,
|
|
101
|
+
timeout=10.0,
|
|
102
|
+
)
|
|
103
|
+
resp.raise_for_status()
|
|
104
|
+
except Exception as exc:
|
|
105
|
+
raise OSError(
|
|
106
|
+
f"LakekeeperStorage: could not list warehouses at {self._lakekeeper_base_url} "
|
|
107
|
+
f"to resolve '{self._warehouse}' → UUID — {type(exc).__name__}: {exc}"
|
|
108
|
+
) from exc
|
|
109
|
+
warehouses = resp.json().get("warehouses", [])
|
|
110
|
+
for w in warehouses:
|
|
111
|
+
if w.get("name") == self._warehouse:
|
|
112
|
+
return w["id"]
|
|
113
|
+
names = [w.get("name") for w in warehouses]
|
|
114
|
+
raise OSError(
|
|
115
|
+
f"LakekeeperStorage: warehouse '{self._warehouse}' not found at "
|
|
116
|
+
f"{self._lakekeeper_base_url} — available: {names}"
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
def _get_client(self):
|
|
120
|
+
if self.__lk_client is None:
|
|
121
|
+
import httpx
|
|
122
|
+
from pylakekeeper import Client, StaticToken
|
|
123
|
+
auth = StaticToken(self._token) if self._token else StaticToken("dev")
|
|
124
|
+
warehouse_uuid = self._resolve_warehouse_uuid()
|
|
125
|
+
client = Client(
|
|
126
|
+
base_url=self._lakekeeper_base_url,
|
|
127
|
+
warehouse=warehouse_uuid,
|
|
128
|
+
auth=auth,
|
|
129
|
+
)
|
|
130
|
+
if not self._ssl_verify:
|
|
131
|
+
client._transport._client = httpx.Client(
|
|
132
|
+
base_url=self._lakekeeper_base_url.rstrip("/"),
|
|
133
|
+
verify=False,
|
|
134
|
+
timeout=30.0,
|
|
135
|
+
)
|
|
136
|
+
self.__lk_client = client
|
|
137
|
+
return self.__lk_client
|
|
138
|
+
|
|
139
|
+
def _ensure_table(self) -> None:
|
|
140
|
+
if self._table_ensured:
|
|
141
|
+
return
|
|
142
|
+
from pylakekeeper import ConflictError, GenericTableFormat
|
|
143
|
+
c = self._get_client()
|
|
144
|
+
try:
|
|
145
|
+
c.generic_tables.create(
|
|
146
|
+
self._namespace, self._table,
|
|
147
|
+
format=GenericTableFormat.DATASET,
|
|
148
|
+
doc=f"{self._tier} unstructured files ({self._branch})",
|
|
149
|
+
)
|
|
150
|
+
logger.info("Created Lakekeeper generic table %s.%s", self._namespace, self._table)
|
|
151
|
+
except ConflictError:
|
|
152
|
+
pass
|
|
153
|
+
except Exception as exc:
|
|
154
|
+
raise OSError(
|
|
155
|
+
f"LakekeeperStorage: could not create generic table "
|
|
156
|
+
f"'{self._namespace}.{self._table}' at {self._lakekeeper_base_url} "
|
|
157
|
+
f"[warehouse={self._warehouse}] — {type(exc).__name__}: {exc}"
|
|
158
|
+
) from exc
|
|
159
|
+
self._table_ensured = True
|
|
160
|
+
|
|
161
|
+
def _build_io(self, credentials: dict):
|
|
162
|
+
"""Build a pyiceberg FsspecFileIO from Lakekeeper remote-signing credentials."""
|
|
163
|
+
if not self._ssl_verify:
|
|
164
|
+
_patch_s3v4_signer_no_ssl_verify()
|
|
165
|
+
from pyiceberg.io import load_file_io
|
|
166
|
+
props = {
|
|
167
|
+
**credentials,
|
|
168
|
+
"token": self._token or "",
|
|
169
|
+
# S3V4RestSigner falls back to properties["uri"] when s3.signer.uri is absent.
|
|
170
|
+
# Python evaluates default arguments eagerly, so "uri" must always be present
|
|
171
|
+
# even when "s3.signer.uri" is set — otherwise dict.get() raises KeyError.
|
|
172
|
+
"uri": (
|
|
173
|
+
credentials.get("s3.signer.uri")
|
|
174
|
+
or credentials.get("signer.uri")
|
|
175
|
+
or self._lakekeeper_base_url.rstrip("/") + "/catalog"
|
|
176
|
+
),
|
|
177
|
+
}
|
|
178
|
+
return load_file_io(props, "")
|
|
179
|
+
|
|
180
|
+
def _vend(self):
|
|
181
|
+
"""Return (pyiceberg FileIO, location_prefix) with fresh vended table metadata."""
|
|
182
|
+
self._ensure_table()
|
|
183
|
+
resp = self._get_client().generic_tables.load(
|
|
184
|
+
self._namespace, self._table, vended=True
|
|
185
|
+
)
|
|
186
|
+
return self._build_io(resp.credentials), resp.location.rstrip("/")
|
|
187
|
+
|
|
188
|
+
def _full_path(self, location: str, path: str) -> str:
|
|
189
|
+
clean = str(PurePosixPath(path).as_posix().lstrip("/"))
|
|
190
|
+
return f"{location}/{clean}"
|
|
191
|
+
|
|
192
|
+
def put_bytes(self, data: bytes, path: str) -> None:
|
|
193
|
+
io, loc = self._vend()
|
|
194
|
+
with io.new_output(self._full_path(loc, path)).create(overwrite=True) as f:
|
|
195
|
+
f.write(data)
|
|
196
|
+
logger.info("Stored %d bytes → %s/%s", len(data), loc, path)
|
|
197
|
+
|
|
198
|
+
def get_bytes(self, path: str) -> bytes:
|
|
199
|
+
io, loc = self._vend()
|
|
200
|
+
with io.new_input(self._full_path(loc, path)).open() as f:
|
|
201
|
+
return f.read()
|
|
202
|
+
|
|
203
|
+
def put_file(self, local_path: str, path: str) -> None:
|
|
204
|
+
io, loc = self._vend()
|
|
205
|
+
data = Path(local_path).read_bytes()
|
|
206
|
+
with io.new_output(self._full_path(loc, path)).create(overwrite=True) as f:
|
|
207
|
+
f.write(data)
|
|
208
|
+
logger.info("Uploaded %s → %s/%s", local_path, loc, path)
|
|
209
|
+
|
|
210
|
+
def get_file(self, path: str, local_path: str) -> None:
|
|
211
|
+
Path(local_path).parent.mkdir(parents=True, exist_ok=True)
|
|
212
|
+
io, loc = self._vend()
|
|
213
|
+
with io.new_input(self._full_path(loc, path)).open() as f:
|
|
214
|
+
Path(local_path).write_bytes(f.read())
|
|
215
|
+
|
|
216
|
+
def list(self, prefix: str = "") -> list[str]:
|
|
217
|
+
io, loc = self._vend()
|
|
218
|
+
search = f"{loc}/{prefix.lstrip('/')}" if prefix else loc
|
|
219
|
+
fs = io.get_fs("s3")
|
|
220
|
+
try:
|
|
221
|
+
paths = fs.find(search, detail=False)
|
|
222
|
+
except FileNotFoundError:
|
|
223
|
+
return []
|
|
224
|
+
loc_stripped = loc.removeprefix("s3://")
|
|
225
|
+
result = []
|
|
226
|
+
for p in paths:
|
|
227
|
+
p_stripped = p.removeprefix("s3://")
|
|
228
|
+
if p_stripped.startswith(loc_stripped + "/"):
|
|
229
|
+
result.append(p_stripped[len(loc_stripped) + 1:])
|
|
230
|
+
else:
|
|
231
|
+
result.append(p_stripped)
|
|
232
|
+
return result
|
|
233
|
+
|
|
234
|
+
def list_all(self) -> list[str]:
|
|
235
|
+
return self.list()
|
|
@@ -213,6 +213,12 @@ class TrinoStorage:
|
|
|
213
213
|
rows: list[dict] = []
|
|
214
214
|
poll = 0
|
|
215
215
|
while True:
|
|
216
|
+
if data.get("error"):
|
|
217
|
+
err = data["error"]
|
|
218
|
+
raise OSError(
|
|
219
|
+
f"TrinoStorage: Trino query failed — {err.get('message', err)} "
|
|
220
|
+
f"(type={err.get('errorType')}, code={err.get('errorCode')}, sql: {sql[:120]!r})"
|
|
221
|
+
)
|
|
216
222
|
if "data" in data and "columns" in data:
|
|
217
223
|
col_names = [c["name"] for c in data["columns"]]
|
|
218
224
|
for row in data["data"]:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: steve-cli
|
|
3
|
-
Version: 1.1.
|
|
3
|
+
Version: 1.1.4
|
|
4
4
|
Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
|
|
5
5
|
Author: Frank
|
|
6
6
|
License: MIT
|
|
@@ -37,6 +37,8 @@ Requires-Dist: pandas>=2.0.3; extra == "pandas"
|
|
|
37
37
|
Requires-Dist: pyarrow>=17.0.0; extra == "pandas"
|
|
38
38
|
Provides-Extra: lineage
|
|
39
39
|
Requires-Dist: openlineage-python>=1.0.0; extra == "lineage"
|
|
40
|
+
Provides-Extra: lakekeeper
|
|
41
|
+
Requires-Dist: pylakekeeper>=0.4.0; extra == "lakekeeper"
|
|
40
42
|
Provides-Extra: validoopsie
|
|
41
43
|
Requires-Dist: validoopsie>=0.1.0; extra == "validoopsie"
|
|
42
44
|
Requires-Dist: polars>=1.8.2; extra == "validoopsie"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|