steve-cli 1.1.2__tar.gz → 1.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. {steve_cli-1.1.2 → steve_cli-1.1.4}/PKG-INFO +3 -1
  2. {steve_cli-1.1.2 → steve_cli-1.1.4}/pyproject.toml +4 -1
  3. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/cli.py +23 -8
  4. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/decorators/lineage_job.py +8 -1
  5. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/policies/client.py +10 -7
  6. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/__init__.py +2 -0
  7. steve_cli-1.1.4/steve_cli/storage/lakekeeper.py +235 -0
  8. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/trino.py +6 -0
  9. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli.egg-info/PKG-INFO +3 -1
  10. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli.egg-info/SOURCES.txt +1 -0
  11. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli.egg-info/requires.txt +3 -0
  12. {steve_cli-1.1.2 → steve_cli-1.1.4}/README.md +0 -0
  13. {steve_cli-1.1.2 → steve_cli-1.1.4}/setup.cfg +0 -0
  14. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/__init__.py +0 -0
  15. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/auth.py +0 -0
  16. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/decorators/__init__.py +0 -0
  17. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/__init__.py +0 -0
  18. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/adapters/__init__.py +0 -0
  19. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/adapters/composite.py +0 -0
  20. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/adapters/dataproductregistry.py +0 -0
  21. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/adapters/logging.py +0 -0
  22. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/adapters/null.py +0 -0
  23. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/adapters/openlineage.py +0 -0
  24. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/collector.py +0 -0
  25. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/port.py +0 -0
  26. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/registry.py +0 -0
  27. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/lineage/storage.py +0 -0
  28. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/ontology.py +0 -0
  29. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/policies/__init__.py +0 -0
  30. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/branch.py +0 -0
  31. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/__init__.py +0 -0
  32. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/extractors/__init__.py +0 -0
  33. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/extractors/csv.py +0 -0
  34. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/extractors/excel.py +0 -0
  35. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/extractors/generic.py +0 -0
  36. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/extractors/json.py +0 -0
  37. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/extractors/parquet.py +0 -0
  38. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/port.py +0 -0
  39. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/metadata/registry.py +0 -0
  40. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/parquet.py +0 -0
  41. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/protocol.py +0 -0
  42. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage/s3.py +0 -0
  43. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/storage.py +0 -0
  44. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/validation/__init__.py +0 -0
  45. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/validation/adapters/__init__.py +0 -0
  46. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/validation/adapters/great_expectations.py +0 -0
  47. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/validation/adapters/null.py +0 -0
  48. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/validation/adapters/validoopsie.py +0 -0
  49. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/validation/port.py +0 -0
  50. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/validation/registry.py +0 -0
  51. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli/vkg.py +0 -0
  52. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli.egg-info/dependency_links.txt +0 -0
  53. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli.egg-info/entry_points.txt +0 -0
  54. {steve_cli-1.1.2 → steve_cli-1.1.4}/steve_cli.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: steve-cli
3
- Version: 1.1.2
3
+ Version: 1.1.4
4
4
  Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
5
5
  Author: Frank
6
6
  License: MIT
@@ -37,6 +37,8 @@ Requires-Dist: pandas>=2.0.3; extra == "pandas"
37
37
  Requires-Dist: pyarrow>=17.0.0; extra == "pandas"
38
38
  Provides-Extra: lineage
39
39
  Requires-Dist: openlineage-python>=1.0.0; extra == "lineage"
40
+ Provides-Extra: lakekeeper
41
+ Requires-Dist: pylakekeeper>=0.4.0; extra == "lakekeeper"
40
42
  Provides-Extra: validoopsie
41
43
  Requires-Dist: validoopsie>=0.1.0; extra == "validoopsie"
42
44
  Requires-Dist: polars>=1.8.2; extra == "validoopsie"
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
5
5
 
6
6
  [project]
7
7
  name = "steve-cli"
8
- version = "1.1.2"
8
+ version = "1.1.4"
9
9
  description = "A simple CLI tool to run jobs from jobs.yaml with proper environment setup"
10
10
  readme = "README.md"
11
11
  license = {text = "MIT"}
@@ -50,6 +50,9 @@ pandas = [
50
50
  lineage = [
51
51
  "openlineage-python>=1.0.0",
52
52
  ]
53
+ lakekeeper = [
54
+ "pylakekeeper>=0.4.0",
55
+ ]
53
56
  validoopsie = [
54
57
  "validoopsie>=0.1.0",
55
58
  "polars>=1.8.2",
@@ -59,8 +59,24 @@ class JobsConfig:
59
59
  return job
60
60
  return None
61
61
 
62
- def list_jobs(self) -> List[str]:
63
- return [job.get('name', 'unnamed') for job in self.jobs]
62
+ def _job_depths(self) -> dict[str, int]:
63
+ depths: dict[str, int] = {}
64
+ for job in self.jobs:
65
+ name = job.get("name", "")
66
+ parents = job.get("dependsOn", [])
67
+ depths[name] = max((depths.get(p, 0) for p in parents), default=-1) + 1
68
+ return depths
69
+
70
+ def list_jobs(self) -> List[questionary.Choice]:
71
+ depths = self._job_depths()
72
+ choices = []
73
+ for job in self.jobs:
74
+ name = job.get("name", "unnamed")
75
+ depth = depths.get(name, 0)
76
+ indent = " " * depth
77
+ prefix = "└─ " if depth > 0 else ""
78
+ choices.append(questionary.Choice(title=f"{indent}{prefix}{name}", value=name))
79
+ return choices
64
80
 
65
81
 
66
82
  def run_job_command(job: Dict[str, Any]) -> int:
@@ -120,13 +136,13 @@ def jobs(ctx: click.Context, jobs_file: Optional[Path]):
120
136
  if ctx.invoked_subcommand is not None:
121
137
  return
122
138
  config = JobsConfig(jobs_file)
123
- job_names = config.list_jobs()
124
- if not job_names:
139
+ choices = config.list_jobs()
140
+ if not choices:
125
141
  click.echo("❌ No jobs found in jobs.yaml")
126
142
  return
127
143
  choice = questionary.select(
128
144
  "Select a job to run:",
129
- choices=job_names,
145
+ choices=choices,
130
146
  ).ask()
131
147
  if choice is None:
132
148
  sys.exit(0)
@@ -140,9 +156,8 @@ def jobs(ctx: click.Context, jobs_file: Optional[Path]):
140
156
  def jobs_list(jobs_file: Optional[Path]):
141
157
  """List all available jobs."""
142
158
  config = JobsConfig(jobs_file)
143
- job_names = config.list_jobs()
144
159
 
145
- if not job_names:
160
+ if not config.jobs:
146
161
  click.echo("❌ No jobs found in jobs.yaml")
147
162
  return
148
163
 
@@ -951,7 +966,7 @@ def policies_apply(file: Path | None, validate: str | None):
951
966
  policies_file = Path(file) if file else None
952
967
  try:
953
968
  branch, sanitized = PolicyClient().apply_from_file(policies_file, validate=validate)
954
- click.secho(f"Branch: {branch} → schema suffix __{sanitized}", fg="cyan")
969
+ click.secho(f"Branch: {branch} → schema: {sanitized}", fg="cyan")
955
970
  click.secho("Policies applied successfully.", fg="green")
956
971
  except FileNotFoundError as e:
957
972
  click.secho(str(e), fg="red", err=True)
@@ -55,8 +55,15 @@ def lineage_job(
55
55
  session=session,
56
56
  )
57
57
 
58
+ def get_files(tier: str = "bronze", workspace: str | None = None) -> LineageStorage:
59
+ from steve_cli.storage.lakekeeper import LakekeeperStorage
60
+ return LineageStorage(
61
+ storage=lambda: LakekeeperStorage(tier=tier, workspace=workspace),
62
+ session=session,
63
+ )
64
+
58
65
  sig = inspect.signature(fn)
59
- available = {"get_storage": get_storage, "get_tables": get_tables}
66
+ available = {"get_storage": get_storage, "get_tables": get_tables, "get_files": get_files}
60
67
  injectable = {k: v for k, v in available.items() if k in sig.parameters}
61
68
 
62
69
  try:
@@ -33,6 +33,9 @@ def _expand_rules(rules: list[dict[str, Any]], sanitized_branch: str, workspace:
33
33
  for entry in rule.get("columns", []):
34
34
  if "schema" in entry:
35
35
  entry["schema"] = _expand_schema(entry["schema"], sanitized_branch)
36
+ if workspace and "catalogs" not in entry:
37
+ tier = entry.get("tier", "bronze")
38
+ entry["catalogs"] = [f"{workspace}-{tier}"]
36
39
  expanded.append(rule)
37
40
  return expanded
38
41
 
@@ -122,20 +125,20 @@ class PolicyClient:
122
125
  workspace_id = os.environ.get("WORKSPACE_ID")
123
126
  workspace_name = os.environ.get("WORKSPACE_NAME", "")
124
127
 
125
- seen: dict[str, list[str]] = {}
128
+ seen: dict[tuple[str, str], list[str]] = {} # (schema, tier) → tables
126
129
  for rule in rules:
127
130
  for entry in list(rule.get("tables", [])) + list(rule.get("columns", [])):
128
131
  schema = entry.get("schema", "")
129
132
  table = entry.get("table", "")
133
+ tier = entry.get("tier", "bronze")
130
134
  if schema and table:
131
- seen.setdefault(schema, [])
132
- if table not in seen[schema]:
133
- seen[schema].append(table)
135
+ key = (schema, tier)
136
+ seen.setdefault(key, [])
137
+ if table not in seen[key]:
138
+ seen[key].append(table)
134
139
 
135
140
  results: list[str] = []
136
- for schema, tables in seen.items():
137
- tier = "bronze"
138
- ws_clean = workspace_name.lower().replace("-", "_") if workspace_name else ""
141
+ for (schema, tier), tables in seen.items():
139
142
  catalog = f"{workspace_name}-{tier}" if workspace_name else f"minio-{tier}"
140
143
 
141
144
  try:
@@ -2,6 +2,7 @@ from .branch import get_branch_prefix
2
2
  from .protocol import Storage
3
3
  from .s3 import S3Storage
4
4
  from .trino import TrinoStorage
5
+ from .lakekeeper import LakekeeperStorage
5
6
  from .parquet import ParquetMetadata, extract_parquet_metadata
6
7
  from .metadata import FileMetadata, ColumnMetadata, MetadataExtractorPort, MetadataRegistry
7
8
 
@@ -10,6 +11,7 @@ __all__ = [
10
11
  "Storage",
11
12
  "S3Storage",
12
13
  "TrinoStorage",
14
+ "LakekeeperStorage",
13
15
  "ParquetMetadata",
14
16
  "extract_parquet_metadata",
15
17
  "FileMetadata",
@@ -0,0 +1,235 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ import os
5
+ import re
6
+ from pathlib import Path, PurePosixPath
7
+
8
+ import requests
9
+
10
+ from .branch import get_branch_prefix
11
+ from steve_cli.auth import get_service_url, get_token
12
+
13
+ logger = logging.getLogger(__name__)
14
+
15
+ _UUID_RE = re.compile(
16
+ r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I
17
+ )
18
+
19
+ _SIGNER_SSL_PATCHED = False
20
+
21
+
22
+ def _patch_s3v4_signer_no_ssl_verify() -> None:
23
+ """Replace SIGNERS['S3V4RestSigner'] with a subclass whose requests.Session has verify=False.
24
+
25
+ Needed when the Lakekeeper signing endpoint uses a cert with a hostname mismatch.
26
+ Mirrors the pyiceberg get_fs monkey-patch in trino.py — applied once, process-wide.
27
+ """
28
+ global _SIGNER_SSL_PATCHED
29
+ if _SIGNER_SSL_PATCHED:
30
+ return
31
+ try:
32
+ from pyiceberg.io.fsspec import SIGNERS, S3V4RestSigner # type: ignore[attr-defined]
33
+ orig = SIGNERS.get("S3V4RestSigner", S3V4RestSigner)
34
+
35
+ class _NoVerify(orig): # type: ignore[valid-type]
36
+ def __init__(self, props): # type: ignore[override]
37
+ super().__init__(props)
38
+ self._session.verify = False
39
+
40
+ SIGNERS["S3V4RestSigner"] = _NoVerify
41
+ _SIGNER_SSL_PATCHED = True
42
+ logger.warning(
43
+ "Applied S3V4RestSigner SSL-verify=False patch (LAKEKEEPER_SSL_VERIFY=0). "
44
+ "Remove once the Lakekeeper TLS certificate covers the full hostname."
45
+ )
46
+ except Exception as exc:
47
+ logger.warning("Could not patch S3V4RestSigner SSL verify: %s", exc)
48
+
49
+
50
+ class LakekeeperStorage:
51
+ """
52
+ Storage backend for unstructured/binary files via Lakekeeper Generic Tables (DATASET format).
53
+
54
+ This warehouse uses remote signing (s3.signer: S3V4RestSigner) — Lakekeeper signs every
55
+ S3 request rather than vending short-lived credentials. We therefore use pyiceberg's
56
+ FsspecFileIO with S3V4RestSigner, the same mechanism TrinoStorage uses for Iceberg tables.
57
+ """
58
+
59
+ def __init__(self, tier: str = "bronze", workspace: str | None = None):
60
+ # aiobotocore 3.9+ defaults to STREAMING-UNSIGNED-PAYLOAD-TRAILER (flexible checksums)
61
+ # which Lakekeeper's remote signing proxy and MinIO don't support.
62
+ os.environ.setdefault("AWS_REQUEST_CHECKSUM_CALCULATION", "when_required")
63
+ import aiobotocore.httpsession
64
+ aiobotocore.httpsession.DEFAULT_KEEPALIVE_TIMEOUT = 3
65
+
66
+ self._tier = tier.lower()
67
+ self._branch = get_branch_prefix()
68
+ self._workspace_id = os.getenv("WORKSPACE_ID")
69
+
70
+ _lk_env = os.getenv("LAKEKEEPER_ENDPOINT")
71
+ _lk_svc = get_service_url("lakekeeper", self._workspace_id)
72
+ self._lakekeeper_base_url = (
73
+ _lk_env.rstrip("/") if _lk_env
74
+ else (_lk_svc.rstrip("/") if _lk_svc else "http://lakekeeper:8181")
75
+ )
76
+ self._token = get_token(self._workspace_id)
77
+
78
+ resolved_workspace = workspace or os.getenv("WORKSPACE_NAME")
79
+ default_warehouse = f"{resolved_workspace}-{tier}" if resolved_workspace else "minio"
80
+ self._warehouse = os.getenv("LAKEKEEPER_WAREHOUSE", default_warehouse)
81
+
82
+ self._namespace = self._branch
83
+ self._table = f"{self._tier}_files"
84
+ self._table_ensured = False
85
+ self.__lk_client = None
86
+
87
+ @property
88
+ def _ssl_verify(self) -> bool:
89
+ return os.getenv("LAKEKEEPER_SSL_VERIFY", "1") != "0"
90
+
91
+ def _resolve_warehouse_uuid(self) -> str:
92
+ """Resolve warehouse name → UUID (generic-tables path requires a UUID, not a name)."""
93
+ if _UUID_RE.match(self._warehouse):
94
+ return self._warehouse
95
+ headers = {"Authorization": f"Bearer {self._token}"} if self._token else {}
96
+ try:
97
+ resp = requests.get(
98
+ f"{self._lakekeeper_base_url}/management/v1/warehouse",
99
+ headers=headers,
100
+ verify=self._ssl_verify,
101
+ timeout=10.0,
102
+ )
103
+ resp.raise_for_status()
104
+ except Exception as exc:
105
+ raise OSError(
106
+ f"LakekeeperStorage: could not list warehouses at {self._lakekeeper_base_url} "
107
+ f"to resolve '{self._warehouse}' → UUID — {type(exc).__name__}: {exc}"
108
+ ) from exc
109
+ warehouses = resp.json().get("warehouses", [])
110
+ for w in warehouses:
111
+ if w.get("name") == self._warehouse:
112
+ return w["id"]
113
+ names = [w.get("name") for w in warehouses]
114
+ raise OSError(
115
+ f"LakekeeperStorage: warehouse '{self._warehouse}' not found at "
116
+ f"{self._lakekeeper_base_url} — available: {names}"
117
+ )
118
+
119
+ def _get_client(self):
120
+ if self.__lk_client is None:
121
+ import httpx
122
+ from pylakekeeper import Client, StaticToken
123
+ auth = StaticToken(self._token) if self._token else StaticToken("dev")
124
+ warehouse_uuid = self._resolve_warehouse_uuid()
125
+ client = Client(
126
+ base_url=self._lakekeeper_base_url,
127
+ warehouse=warehouse_uuid,
128
+ auth=auth,
129
+ )
130
+ if not self._ssl_verify:
131
+ client._transport._client = httpx.Client(
132
+ base_url=self._lakekeeper_base_url.rstrip("/"),
133
+ verify=False,
134
+ timeout=30.0,
135
+ )
136
+ self.__lk_client = client
137
+ return self.__lk_client
138
+
139
+ def _ensure_table(self) -> None:
140
+ if self._table_ensured:
141
+ return
142
+ from pylakekeeper import ConflictError, GenericTableFormat
143
+ c = self._get_client()
144
+ try:
145
+ c.generic_tables.create(
146
+ self._namespace, self._table,
147
+ format=GenericTableFormat.DATASET,
148
+ doc=f"{self._tier} unstructured files ({self._branch})",
149
+ )
150
+ logger.info("Created Lakekeeper generic table %s.%s", self._namespace, self._table)
151
+ except ConflictError:
152
+ pass
153
+ except Exception as exc:
154
+ raise OSError(
155
+ f"LakekeeperStorage: could not create generic table "
156
+ f"'{self._namespace}.{self._table}' at {self._lakekeeper_base_url} "
157
+ f"[warehouse={self._warehouse}] — {type(exc).__name__}: {exc}"
158
+ ) from exc
159
+ self._table_ensured = True
160
+
161
+ def _build_io(self, credentials: dict):
162
+ """Build a pyiceberg FsspecFileIO from Lakekeeper remote-signing credentials."""
163
+ if not self._ssl_verify:
164
+ _patch_s3v4_signer_no_ssl_verify()
165
+ from pyiceberg.io import load_file_io
166
+ props = {
167
+ **credentials,
168
+ "token": self._token or "",
169
+ # S3V4RestSigner falls back to properties["uri"] when s3.signer.uri is absent.
170
+ # Python evaluates default arguments eagerly, so "uri" must always be present
171
+ # even when "s3.signer.uri" is set — otherwise dict.get() raises KeyError.
172
+ "uri": (
173
+ credentials.get("s3.signer.uri")
174
+ or credentials.get("signer.uri")
175
+ or self._lakekeeper_base_url.rstrip("/") + "/catalog"
176
+ ),
177
+ }
178
+ return load_file_io(props, "")
179
+
180
+ def _vend(self):
181
+ """Return (pyiceberg FileIO, location_prefix) with fresh vended table metadata."""
182
+ self._ensure_table()
183
+ resp = self._get_client().generic_tables.load(
184
+ self._namespace, self._table, vended=True
185
+ )
186
+ return self._build_io(resp.credentials), resp.location.rstrip("/")
187
+
188
+ def _full_path(self, location: str, path: str) -> str:
189
+ clean = str(PurePosixPath(path).as_posix().lstrip("/"))
190
+ return f"{location}/{clean}"
191
+
192
+ def put_bytes(self, data: bytes, path: str) -> None:
193
+ io, loc = self._vend()
194
+ with io.new_output(self._full_path(loc, path)).create(overwrite=True) as f:
195
+ f.write(data)
196
+ logger.info("Stored %d bytes → %s/%s", len(data), loc, path)
197
+
198
+ def get_bytes(self, path: str) -> bytes:
199
+ io, loc = self._vend()
200
+ with io.new_input(self._full_path(loc, path)).open() as f:
201
+ return f.read()
202
+
203
+ def put_file(self, local_path: str, path: str) -> None:
204
+ io, loc = self._vend()
205
+ data = Path(local_path).read_bytes()
206
+ with io.new_output(self._full_path(loc, path)).create(overwrite=True) as f:
207
+ f.write(data)
208
+ logger.info("Uploaded %s → %s/%s", local_path, loc, path)
209
+
210
+ def get_file(self, path: str, local_path: str) -> None:
211
+ Path(local_path).parent.mkdir(parents=True, exist_ok=True)
212
+ io, loc = self._vend()
213
+ with io.new_input(self._full_path(loc, path)).open() as f:
214
+ Path(local_path).write_bytes(f.read())
215
+
216
+ def list(self, prefix: str = "") -> list[str]:
217
+ io, loc = self._vend()
218
+ search = f"{loc}/{prefix.lstrip('/')}" if prefix else loc
219
+ fs = io.get_fs("s3")
220
+ try:
221
+ paths = fs.find(search, detail=False)
222
+ except FileNotFoundError:
223
+ return []
224
+ loc_stripped = loc.removeprefix("s3://")
225
+ result = []
226
+ for p in paths:
227
+ p_stripped = p.removeprefix("s3://")
228
+ if p_stripped.startswith(loc_stripped + "/"):
229
+ result.append(p_stripped[len(loc_stripped) + 1:])
230
+ else:
231
+ result.append(p_stripped)
232
+ return result
233
+
234
+ def list_all(self) -> list[str]:
235
+ return self.list()
@@ -213,6 +213,12 @@ class TrinoStorage:
213
213
  rows: list[dict] = []
214
214
  poll = 0
215
215
  while True:
216
+ if data.get("error"):
217
+ err = data["error"]
218
+ raise OSError(
219
+ f"TrinoStorage: Trino query failed — {err.get('message', err)} "
220
+ f"(type={err.get('errorType')}, code={err.get('errorCode')}, sql: {sql[:120]!r})"
221
+ )
216
222
  if "data" in data and "columns" in data:
217
223
  col_names = [c["name"] for c in data["columns"]]
218
224
  for row in data["data"]:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: steve-cli
3
- Version: 1.1.2
3
+ Version: 1.1.4
4
4
  Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
5
5
  Author: Frank
6
6
  License: MIT
@@ -37,6 +37,8 @@ Requires-Dist: pandas>=2.0.3; extra == "pandas"
37
37
  Requires-Dist: pyarrow>=17.0.0; extra == "pandas"
38
38
  Provides-Extra: lineage
39
39
  Requires-Dist: openlineage-python>=1.0.0; extra == "lineage"
40
+ Provides-Extra: lakekeeper
41
+ Requires-Dist: pylakekeeper>=0.4.0; extra == "lakekeeper"
40
42
  Provides-Extra: validoopsie
41
43
  Requires-Dist: validoopsie>=0.1.0; extra == "validoopsie"
42
44
  Requires-Dist: polars>=1.8.2; extra == "validoopsie"
@@ -29,6 +29,7 @@ steve_cli/policies/__init__.py
29
29
  steve_cli/policies/client.py
30
30
  steve_cli/storage/__init__.py
31
31
  steve_cli/storage/branch.py
32
+ steve_cli/storage/lakekeeper.py
32
33
  steve_cli/storage/parquet.py
33
34
  steve_cli/storage/protocol.py
34
35
  steve_cli/storage/s3.py
@@ -31,6 +31,9 @@ openpyxl>=3.1.0
31
31
  great-expectations>=0.18.0
32
32
  pandas>=2.0.3
33
33
 
34
+ [lakekeeper]
35
+ pylakekeeper>=0.4.0
36
+
34
37
  [lineage]
35
38
  openlineage-python>=1.0.0
36
39
 
File without changes
File without changes
File without changes
File without changes