steve-cli 1.1.3__tar.gz → 1.1.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {steve_cli-1.1.3 → steve_cli-1.1.5}/PKG-INFO +1 -1
  2. {steve_cli-1.1.3 → steve_cli-1.1.5}/pyproject.toml +1 -1
  3. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/cli.py +23 -8
  4. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/policies/client.py +10 -7
  5. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/storage/__init__.py +5 -0
  6. steve_cli-1.1.5/steve_cli/storage/auth.py +65 -0
  7. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/storage/trino.py +26 -8
  8. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli.egg-info/PKG-INFO +1 -1
  9. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli.egg-info/SOURCES.txt +1 -0
  10. {steve_cli-1.1.3 → steve_cli-1.1.5}/README.md +0 -0
  11. {steve_cli-1.1.3 → steve_cli-1.1.5}/setup.cfg +0 -0
  12. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/__init__.py +0 -0
  13. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/auth.py +0 -0
  14. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/decorators/__init__.py +0 -0
  15. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/decorators/lineage_job.py +0 -0
  16. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/lineage/__init__.py +0 -0
  17. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/lineage/adapters/__init__.py +0 -0
  18. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/lineage/adapters/composite.py +0 -0
  19. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/lineage/adapters/dataproductregistry.py +0 -0
  20. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/lineage/adapters/logging.py +0 -0
  21. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/lineage/adapters/null.py +0 -0
  22. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/lineage/adapters/openlineage.py +0 -0
  23. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/lineage/collector.py +0 -0
  24. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/lineage/port.py +0 -0
  25. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/lineage/registry.py +0 -0
  26. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/lineage/storage.py +0 -0
  27. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/ontology.py +0 -0
  28. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/policies/__init__.py +0 -0
  29. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/storage/branch.py +0 -0
  30. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/storage/lakekeeper.py +0 -0
  31. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/storage/metadata/__init__.py +0 -0
  32. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/storage/metadata/extractors/__init__.py +0 -0
  33. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/storage/metadata/extractors/csv.py +0 -0
  34. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/storage/metadata/extractors/excel.py +0 -0
  35. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/storage/metadata/extractors/generic.py +0 -0
  36. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/storage/metadata/extractors/json.py +0 -0
  37. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/storage/metadata/extractors/parquet.py +0 -0
  38. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/storage/metadata/port.py +0 -0
  39. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/storage/metadata/registry.py +0 -0
  40. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/storage/parquet.py +0 -0
  41. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/storage/protocol.py +0 -0
  42. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/storage/s3.py +0 -0
  43. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/storage.py +0 -0
  44. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/validation/__init__.py +0 -0
  45. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/validation/adapters/__init__.py +0 -0
  46. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/validation/adapters/great_expectations.py +0 -0
  47. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/validation/adapters/null.py +0 -0
  48. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/validation/adapters/validoopsie.py +0 -0
  49. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/validation/port.py +0 -0
  50. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/validation/registry.py +0 -0
  51. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli/vkg.py +0 -0
  52. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli.egg-info/dependency_links.txt +0 -0
  53. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli.egg-info/entry_points.txt +0 -0
  54. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli.egg-info/requires.txt +0 -0
  55. {steve_cli-1.1.3 → steve_cli-1.1.5}/steve_cli.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: steve-cli
3
- Version: 1.1.3
3
+ Version: 1.1.5
4
4
  Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
5
5
  Author: Frank
6
6
  License: MIT
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
5
5
 
6
6
  [project]
7
7
  name = "steve-cli"
8
- version = "1.1.3"
8
+ version = "1.1.5"
9
9
  description = "A simple CLI tool to run jobs from jobs.yaml with proper environment setup"
10
10
  readme = "README.md"
11
11
  license = {text = "MIT"}
@@ -59,8 +59,24 @@ class JobsConfig:
59
59
  return job
60
60
  return None
61
61
 
62
- def list_jobs(self) -> List[str]:
63
- return [job.get('name', 'unnamed') for job in self.jobs]
62
+ def _job_depths(self) -> dict[str, int]:
63
+ depths: dict[str, int] = {}
64
+ for job in self.jobs:
65
+ name = job.get("name", "")
66
+ parents = job.get("dependsOn", [])
67
+ depths[name] = max((depths.get(p, 0) for p in parents), default=-1) + 1
68
+ return depths
69
+
70
+ def list_jobs(self) -> List[questionary.Choice]:
71
+ depths = self._job_depths()
72
+ choices = []
73
+ for job in self.jobs:
74
+ name = job.get("name", "unnamed")
75
+ depth = depths.get(name, 0)
76
+ indent = " " * depth
77
+ prefix = "└─ " if depth > 0 else ""
78
+ choices.append(questionary.Choice(title=f"{indent}{prefix}{name}", value=name))
79
+ return choices
64
80
 
65
81
 
66
82
  def run_job_command(job: Dict[str, Any]) -> int:
@@ -120,13 +136,13 @@ def jobs(ctx: click.Context, jobs_file: Optional[Path]):
120
136
  if ctx.invoked_subcommand is not None:
121
137
  return
122
138
  config = JobsConfig(jobs_file)
123
- job_names = config.list_jobs()
124
- if not job_names:
139
+ choices = config.list_jobs()
140
+ if not choices:
125
141
  click.echo("❌ No jobs found in jobs.yaml")
126
142
  return
127
143
  choice = questionary.select(
128
144
  "Select a job to run:",
129
- choices=job_names,
145
+ choices=choices,
130
146
  ).ask()
131
147
  if choice is None:
132
148
  sys.exit(0)
@@ -140,9 +156,8 @@ def jobs(ctx: click.Context, jobs_file: Optional[Path]):
140
156
  def jobs_list(jobs_file: Optional[Path]):
141
157
  """List all available jobs."""
142
158
  config = JobsConfig(jobs_file)
143
- job_names = config.list_jobs()
144
159
 
145
- if not job_names:
160
+ if not config.jobs:
146
161
  click.echo("❌ No jobs found in jobs.yaml")
147
162
  return
148
163
 
@@ -951,7 +966,7 @@ def policies_apply(file: Path | None, validate: str | None):
951
966
  policies_file = Path(file) if file else None
952
967
  try:
953
968
  branch, sanitized = PolicyClient().apply_from_file(policies_file, validate=validate)
954
- click.secho(f"Branch: {branch} → schema suffix __{sanitized}", fg="cyan")
969
+ click.secho(f"Branch: {branch} → schema: {sanitized}", fg="cyan")
955
970
  click.secho("Policies applied successfully.", fg="green")
956
971
  except FileNotFoundError as e:
957
972
  click.secho(str(e), fg="red", err=True)
@@ -33,6 +33,9 @@ def _expand_rules(rules: list[dict[str, Any]], sanitized_branch: str, workspace:
33
33
  for entry in rule.get("columns", []):
34
34
  if "schema" in entry:
35
35
  entry["schema"] = _expand_schema(entry["schema"], sanitized_branch)
36
+ if workspace and "catalogs" not in entry:
37
+ tier = entry.get("tier", "bronze")
38
+ entry["catalogs"] = [f"{workspace}-{tier}"]
36
39
  expanded.append(rule)
37
40
  return expanded
38
41
 
@@ -122,20 +125,20 @@ class PolicyClient:
122
125
  workspace_id = os.environ.get("WORKSPACE_ID")
123
126
  workspace_name = os.environ.get("WORKSPACE_NAME", "")
124
127
 
125
- seen: dict[str, list[str]] = {}
128
+ seen: dict[tuple[str, str], list[str]] = {} # (schema, tier) → tables
126
129
  for rule in rules:
127
130
  for entry in list(rule.get("tables", [])) + list(rule.get("columns", [])):
128
131
  schema = entry.get("schema", "")
129
132
  table = entry.get("table", "")
133
+ tier = entry.get("tier", "bronze")
130
134
  if schema and table:
131
- seen.setdefault(schema, [])
132
- if table not in seen[schema]:
133
- seen[schema].append(table)
135
+ key = (schema, tier)
136
+ seen.setdefault(key, [])
137
+ if table not in seen[key]:
138
+ seen[key].append(table)
134
139
 
135
140
  results: list[str] = []
136
- for schema, tables in seen.items():
137
- tier = "bronze"
138
- ws_clean = workspace_name.lower().replace("-", "_") if workspace_name else ""
141
+ for (schema, tier), tables in seen.items():
139
142
  catalog = f"{workspace_name}-{tier}" if workspace_name else f"minio-{tier}"
140
143
 
141
144
  try:
@@ -1,3 +1,4 @@
1
+ from .auth import BearerTokenStrategy, CredentialsFileStrategy, EnvVarStrategy, TrinoAuthStrategy
1
2
  from .branch import get_branch_prefix
2
3
  from .protocol import Storage
3
4
  from .s3 import S3Storage
@@ -7,6 +8,10 @@ from .parquet import ParquetMetadata, extract_parquet_metadata
7
8
  from .metadata import FileMetadata, ColumnMetadata, MetadataExtractorPort, MetadataRegistry
8
9
 
9
10
  __all__ = [
11
+ "BearerTokenStrategy",
12
+ "CredentialsFileStrategy",
13
+ "EnvVarStrategy",
14
+ "TrinoAuthStrategy",
10
15
  "get_branch_prefix",
11
16
  "Storage",
12
17
  "S3Storage",
@@ -0,0 +1,65 @@
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ from typing import Protocol, runtime_checkable
5
+
6
+
7
+ @runtime_checkable
8
+ class TrinoAuthStrategy(Protocol):
9
+ def get_token(self) -> str | None: ...
10
+ def get_service_url(self, service: str) -> str | None: ...
11
+
12
+
13
+ class CredentialsFileStrategy:
14
+ """Default: reads token and service URLs from ~/.steve/credentials.json."""
15
+
16
+ def __init__(self, workspace_id: str | None = None) -> None:
17
+ self._workspace_id = workspace_id
18
+
19
+ def get_token(self) -> str | None:
20
+ from steve_cli.auth import get_token
21
+ return get_token(self._workspace_id)
22
+
23
+ def get_service_url(self, service: str) -> str | None:
24
+ from steve_cli.auth import get_service_url
25
+ return get_service_url(service, self._workspace_id)
26
+
27
+
28
+ class BearerTokenStrategy:
29
+ """Explicit token + base URL for injected credentials (e.g. X-AK-Token header).
30
+
31
+ Service URLs are constructed as https://<service>-<workspace_id>.<base_url>.
32
+ """
33
+
34
+ def __init__(self, token: str, workspace_id: str, base_url: str) -> None:
35
+ self._token = token
36
+ self._workspace_id = workspace_id
37
+ self._base_url = base_url.rstrip("/")
38
+
39
+ def get_token(self) -> str | None:
40
+ return self._token or None
41
+
42
+ def get_service_url(self, service: str) -> str | None:
43
+ if not self._workspace_id or not self._base_url:
44
+ return None
45
+ return f"https://{service}-{self._workspace_id}.{self._base_url}"
46
+
47
+
48
+ class EnvVarStrategy:
49
+ """Reads TRINO_TOKEN + TRINO_URL (or STEVE_BASE_URL + WORKSPACE_ID) from env.
50
+
51
+ Useful in containers where credentials are injected as env vars rather than a file.
52
+ """
53
+
54
+ def get_token(self) -> str | None:
55
+ return os.getenv("TRINO_TOKEN") or None
56
+
57
+ def get_service_url(self, service: str) -> str | None:
58
+ explicit = os.getenv(f"{service.upper()}_URL")
59
+ if explicit:
60
+ return explicit
61
+ workspace_id = os.getenv("WORKSPACE_ID")
62
+ base_url = os.getenv("STEVE_BASE_URL", "").rstrip("/")
63
+ if workspace_id and base_url:
64
+ return f"https://{service}-{workspace_id}.{base_url}"
65
+ return None
@@ -10,6 +10,8 @@ import pyarrow as pa
10
10
  import pyarrow.parquet as pq
11
11
  import requests
12
12
 
13
+ from steve_cli.storage.auth import CredentialsFileStrategy, TrinoAuthStrategy
14
+
13
15
  from .branch import get_branch_prefix
14
16
 
15
17
  logger = logging.getLogger(__name__)
@@ -33,6 +35,7 @@ def _patch_pyiceberg_get_fs_once() -> None:
33
35
 
34
36
  try:
35
37
  import threading as _t
38
+
36
39
  from pyiceberg.io.fsspec import FsspecFileIO
37
40
 
38
41
  _caches: dict[int, dict] = {}
@@ -63,7 +66,12 @@ _patch_pyiceberg_get_fs_once()
63
66
 
64
67
 
65
68
  class TrinoStorage:
66
- def __init__(self, tier: str = "bronze", workspace: str | None = None):
69
+ def __init__(
70
+ self,
71
+ tier: str = "bronze",
72
+ workspace: str | None = None,
73
+ auth: TrinoAuthStrategy | None = None,
74
+ ):
67
75
  # aiobotocore 3.9+ defaults to STREAMING-UNSIGNED-PAYLOAD-TRAILER (flexible checksums)
68
76
  # which Lakekeeper's remote signing proxy and MinIO don't support.
69
77
  os.environ.setdefault('AWS_REQUEST_CHECKSUM_CALCULATION', 'when_required')
@@ -73,11 +81,12 @@ class TrinoStorage:
73
81
  # the next put_bytes call reaches the S3 write phase.
74
82
  import aiobotocore.httpsession
75
83
  aiobotocore.httpsession.DEFAULT_KEEPALIVE_TIMEOUT = 3
76
- from steve_cli.auth import get_service_url
84
+
77
85
  self._branch = get_branch_prefix()
78
86
  self._tier = tier.lower()
79
87
  self._workspace_id = os.getenv("WORKSPACE_ID")
80
- trino_endpoint = get_service_url("trino", self._workspace_id)
88
+ self._auth: TrinoAuthStrategy = auth if auth is not None else CredentialsFileStrategy(self._workspace_id)
89
+ trino_endpoint = self._auth.get_service_url("trino")
81
90
  if not trino_endpoint:
82
91
  from steve_cli.auth import _get_workspace_entry
83
92
  entry = _get_workspace_entry(self._workspace_id)
@@ -96,7 +105,7 @@ class TrinoStorage:
96
105
  self.user = os.getenv("TRINO_USER") or resolved_workspace or "admin"
97
106
  self._base = trino_endpoint.rstrip("/")
98
107
  _lk_env = os.getenv("LAKEKEEPER_ENDPOINT")
99
- _lk_svc = get_service_url("lakekeeper", self._workspace_id)
108
+ _lk_svc = self._auth.get_service_url("lakekeeper")
100
109
  self._lakekeeper_endpoint = _lk_env.rstrip("/") if _lk_env else (_lk_svc.rstrip("/") + "/catalog" if _lk_svc else None)
101
110
  self._lakekeeper_base = (self._lakekeeper_endpoint or "http://lakekeeper:8181/catalog").rstrip("/")
102
111
  self._lakekeeper_warehouse = os.getenv("LAKEKEEPER_WAREHOUSE", default_catalog)
@@ -117,7 +126,9 @@ class TrinoStorage:
117
126
  from pyiceberg.catalog.rest import RestCatalog
118
127
  from pyiceberg.io import load_file_io
119
128
  from pyiceberg.table import Table
129
+
120
130
  from steve_cli.auth import get_s3_endpoint as _get_s3_ep
131
+
121
132
  _tier = tier.upper()
122
133
  _s3_ep = _get_s3_ep(self._workspace_id) or os.getenv("S3_ENDPOINT", "http://minio:9000")
123
134
  _s3_overrides = {
@@ -155,8 +166,7 @@ class TrinoStorage:
155
166
  def _iceberg_catalog(self):
156
167
  if self.__iceberg_catalog is None:
157
168
  from pyiceberg.catalog.rest import RestCatalog
158
- from steve_cli.auth import get_token
159
- stp_token = get_token(self._workspace_id)
169
+ stp_token = self._auth.get_token()
160
170
  token_kwargs = {"token": stp_token} if stp_token else {}
161
171
  catalog_cls = self._catalog_cls or RestCatalog
162
172
  try:
@@ -181,17 +191,19 @@ class TrinoStorage:
181
191
  return self.__iceberg_catalog
182
192
 
183
193
  def _execute(self, sql: str) -> list[dict]:
184
- from steve_cli.auth import get_token
185
194
  trace = os.getenv("STEVE_TRINO_TRACE") == "1"
186
195
  headers = {
187
196
  "X-Trino-User": self.user,
188
197
  "X-Trino-Catalog": self.catalog,
189
198
  "X-Trino-Schema": self.schema,
190
199
  }
191
- token = get_token(self._workspace_id)
200
+ token = self._auth.get_token()
192
201
  if token:
193
202
  headers["Authorization"] = f"Bearer {token}"
194
203
 
204
+ if trace:
205
+ logger.warning(f"[trino] {self._base}" )
206
+
195
207
  t0 = time.perf_counter()
196
208
  try:
197
209
  resp = requests.post(f"{self._base}/v1/statement", data=sql, headers=headers)
@@ -213,6 +225,12 @@ class TrinoStorage:
213
225
  rows: list[dict] = []
214
226
  poll = 0
215
227
  while True:
228
+ if data.get("error"):
229
+ err = data["error"]
230
+ raise OSError(
231
+ f"TrinoStorage: Trino query failed — {err.get('message', err)} "
232
+ f"(type={err.get('errorType')}, code={err.get('errorCode')}, sql: {sql[:120]!r})"
233
+ )
216
234
  if "data" in data and "columns" in data:
217
235
  col_names = [c["name"] for c in data["columns"]]
218
236
  for row in data["data"]:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: steve-cli
3
- Version: 1.1.3
3
+ Version: 1.1.5
4
4
  Summary: A simple CLI tool to run jobs from jobs.yaml with proper environment setup
5
5
  Author: Frank
6
6
  License: MIT
@@ -28,6 +28,7 @@ steve_cli/lineage/adapters/openlineage.py
28
28
  steve_cli/policies/__init__.py
29
29
  steve_cli/policies/client.py
30
30
  steve_cli/storage/__init__.py
31
+ steve_cli/storage/auth.py
31
32
  steve_cli/storage/branch.py
32
33
  steve_cli/storage/lakekeeper.py
33
34
  steve_cli/storage/parquet.py
File without changes
File without changes
File without changes
File without changes