steve-cli 1.1.7__tar.gz → 1.1.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {steve_cli-1.1.7 → steve_cli-1.1.8}/PKG-INFO +1 -1
- {steve_cli-1.1.7 → steve_cli-1.1.8}/pyproject.toml +1 -1
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/cli.py +28 -2
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/lineage/storage.py +18 -12
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage/lakekeeper.py +8 -2
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage/protocol.py +4 -2
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage/s3.py +8 -2
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage/trino.py +31 -6
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage.py +12 -4
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli.egg-info/PKG-INFO +1 -1
- {steve_cli-1.1.7 → steve_cli-1.1.8}/README.md +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/setup.cfg +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/__init__.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/auth.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/decorators/__init__.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/decorators/lineage_job.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/lineage/__init__.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/lineage/adapters/__init__.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/lineage/adapters/composite.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/lineage/adapters/dataproductregistry.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/lineage/adapters/logging.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/lineage/adapters/null.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/lineage/adapters/openlineage.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/lineage/collector.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/lineage/port.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/lineage/registry.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/ontology.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/policies/__init__.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/policies/client.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage/__init__.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage/auth.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage/branch.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage/metadata/__init__.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage/metadata/extractors/__init__.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage/metadata/extractors/csv.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage/metadata/extractors/excel.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage/metadata/extractors/generic.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage/metadata/extractors/json.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage/metadata/extractors/parquet.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage/metadata/port.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage/metadata/registry.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/storage/parquet.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/validation/__init__.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/validation/adapters/__init__.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/validation/adapters/great_expectations.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/validation/adapters/null.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/validation/adapters/validoopsie.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/validation/port.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/validation/registry.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli/vkg.py +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli.egg-info/SOURCES.txt +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli.egg-info/dependency_links.txt +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli.egg-info/entry_points.txt +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli.egg-info/requires.txt +0 -0
- {steve_cli-1.1.7 → steve_cli-1.1.8}/steve_cli.egg-info/top_level.txt +0 -0
|
@@ -1233,21 +1233,47 @@ def policies_policy_rm(pdp_name: str | None, yes: bool):
|
|
|
1233
1233
|
|
|
1234
1234
|
|
|
1235
1235
|
@main.command("login")
|
|
1236
|
+
@click.argument("workspace_name_arg", required=False, default=None, metavar="WORKSPACE_NAME")
|
|
1236
1237
|
@click.option("--token", default=None, help="Service principal token (stp_...) to save")
|
|
1237
1238
|
@click.option("--workspace", "workspace_id", default=None, help="Workspace ID (defaults to WORKSPACE_ID from .env)")
|
|
1238
1239
|
@click.option("--url", "base_url", default=None, help="Platform root host, e.g. jds-dev.internal.jambit.io")
|
|
1239
1240
|
@click.option("--workspace-name", "workspace_name_opt", default=None, help="Workspace name")
|
|
1240
1241
|
@click.option("--expires", "expires_at", default=None, help="Token expiry date/time (e.g. 2026-12-31 or ISO 8601). Informational only.")
|
|
1242
|
+
@click.option("--token-data", "token_data", default=None, help="Base64-encoded JSON blob with token, url, expires, and workspaces map. Generated by the Admin UI. Supersedes all other flags.")
|
|
1241
1243
|
@click.option('--env-file', '-e', type=click.Path(path_type=Path), multiple=True,
|
|
1242
1244
|
help='Path to .env file(s). Defaults to .env and .workspaces.env')
|
|
1243
|
-
def login(token: str | None, workspace_id: str | None, base_url: str | None, workspace_name_opt: str | None, expires_at: str | None, env_file: tuple):
|
|
1245
|
+
def login(workspace_name_arg: str | None, token: str | None, workspace_id: str | None, base_url: str | None, workspace_name_opt: str | None, expires_at: str | None, token_data: str | None, env_file: tuple):
|
|
1244
1246
|
"""Print CLI access URLs or save a service principal token."""
|
|
1247
|
+
if token_data:
|
|
1248
|
+
import base64 as _b64, json as _json
|
|
1249
|
+
from steve_cli.auth import save_credentials as _save
|
|
1250
|
+
try:
|
|
1251
|
+
payload = _json.loads(_b64.b64decode(token_data).decode())
|
|
1252
|
+
except Exception as e:
|
|
1253
|
+
click.secho(f"ERROR: --token-data is not valid base64 JSON: {e}", fg="red", err=True)
|
|
1254
|
+
raise SystemExit(1)
|
|
1255
|
+
_token = payload.get("token", "")
|
|
1256
|
+
_url = payload.get("url")
|
|
1257
|
+
_expires = payload.get("expires")
|
|
1258
|
+
_workspaces: dict = payload.get("workspaces", {})
|
|
1259
|
+
if not _token.startswith("stp_"):
|
|
1260
|
+
click.secho("ERROR: token in --token-data must start with 'stp_'.", fg="red", err=True)
|
|
1261
|
+
raise SystemExit(1)
|
|
1262
|
+
if not _workspaces:
|
|
1263
|
+
click.secho("ERROR: --token-data contains no workspaces.", fg="red", err=True)
|
|
1264
|
+
raise SystemExit(1)
|
|
1265
|
+
for ws_name, ws_id in _workspaces.items():
|
|
1266
|
+
_save(ws_id, _token, base_url=_url, workspace_name=ws_name, expires_at=_expires)
|
|
1267
|
+
click.secho(f" ✓ {ws_name} ({ws_id})", fg="green")
|
|
1268
|
+
click.secho(f"\nLogged in to {len(_workspaces)} workspace(s). Credentials saved to ~/.steve/credentials.json", fg="green")
|
|
1269
|
+
return
|
|
1270
|
+
|
|
1245
1271
|
cwd = Path.cwd()
|
|
1246
1272
|
env_files = [Path(f) for f in env_file] if env_file else [cwd / ".env", cwd / ".workspaces.env"]
|
|
1247
1273
|
for ef in env_files:
|
|
1248
1274
|
load_dotenv(ef)
|
|
1249
1275
|
|
|
1250
|
-
resolved_workspace = workspace_id or os.getenv("WORKSPACE_ID")
|
|
1276
|
+
resolved_workspace = workspace_id or os.getenv("WORKSPACE_ID") or workspace_name_arg
|
|
1251
1277
|
workspace_name = workspace_name_opt or os.getenv("WORKSPACE_NAME") or resolved_workspace
|
|
1252
1278
|
|
|
1253
1279
|
if token:
|
|
@@ -71,16 +71,18 @@ class LineageStorage:
|
|
|
71
71
|
return f"{s.endpoint.rstrip('/')}/{name}"
|
|
72
72
|
return None
|
|
73
73
|
|
|
74
|
-
def put_file(self, local_path: str, path: str
|
|
75
|
-
|
|
74
|
+
def put_file(self, local_path: str, path: str, mode: str = "append",
|
|
75
|
+
key: str | list[str] | None = None) -> None:
|
|
76
|
+
self._storage.put_file(local_path, path, mode=mode, key=key)
|
|
76
77
|
self._session.record_write(path, namespace=self._dataset_namespace)
|
|
77
78
|
|
|
78
79
|
def get_file(self, path: str, local_path: str) -> None:
|
|
79
80
|
self._storage.get_file(path, local_path)
|
|
80
81
|
self._session.record_read(path, namespace=self._dataset_namespace)
|
|
81
82
|
|
|
82
|
-
def put_bytes(self, data: bytes, path: str
|
|
83
|
-
|
|
83
|
+
def put_bytes(self, data: bytes, path: str, mode: str = "append",
|
|
84
|
+
key: str | list[str] | None = None) -> None:
|
|
85
|
+
self._storage.put_bytes(data, path, mode=mode, key=key)
|
|
84
86
|
facets = _extract_facets(data, path)
|
|
85
87
|
self._session.record_write(path, facets or None, namespace=self._dataset_namespace)
|
|
86
88
|
|
|
@@ -93,8 +95,9 @@ class LineageStorage:
|
|
|
93
95
|
def read(self, path: str) -> DatasetHandle:
|
|
94
96
|
return DatasetHandle(path, self.get_bytes(path), self._session, is_read=True)
|
|
95
97
|
|
|
96
|
-
def write(self, path: str, data: bytes
|
|
97
|
-
|
|
98
|
+
def write(self, path: str, data: bytes, mode: str = "append",
|
|
99
|
+
key: str | list[str] | None = None) -> DatasetHandle:
|
|
100
|
+
self.put_bytes(data, path, mode=mode, key=key)
|
|
98
101
|
return DatasetHandle(path, data, self._session)
|
|
99
102
|
|
|
100
103
|
def read_df(self, path: str, parser: Callable, validator: Any = None) -> Any:
|
|
@@ -118,20 +121,23 @@ class LineageStorage:
|
|
|
118
121
|
import polars as pl
|
|
119
122
|
return self.read_df(path, lambda b: pl.read_excel(b, **kwargs), validator)
|
|
120
123
|
|
|
121
|
-
def write_df(self, path: str, df: Any, serializer: Callable
|
|
124
|
+
def write_df(self, path: str, df: Any, serializer: Callable, mode: str = "append",
|
|
125
|
+
key: str | list[str] | None = None) -> DatasetHandle:
|
|
122
126
|
data = serializer(df)
|
|
123
|
-
self.put_bytes(data, path)
|
|
127
|
+
self.put_bytes(data, path, mode=mode, key=key)
|
|
124
128
|
return DatasetHandle(path, data, self._session)
|
|
125
129
|
|
|
126
|
-
def write_csv(self, path: str, df: Any,
|
|
127
|
-
|
|
130
|
+
def write_csv(self, path: str, df: Any, mode: str = "append",
|
|
131
|
+
key: str | list[str] | None = None, **kwargs: Any) -> DatasetHandle:
|
|
132
|
+
return self.write_df(path, df, lambda d: d.write_csv(**kwargs).encode(), mode=mode, key=key)
|
|
128
133
|
|
|
129
|
-
def write_parquet(self, path: str, df: Any,
|
|
134
|
+
def write_parquet(self, path: str, df: Any, mode: str = "append",
|
|
135
|
+
key: str | list[str] | None = None, **kwargs: Any) -> DatasetHandle:
|
|
130
136
|
def _to_bytes(d: Any) -> bytes:
|
|
131
137
|
buf = BytesIO()
|
|
132
138
|
d.write_parquet(buf, **kwargs)
|
|
133
139
|
return buf.getvalue()
|
|
134
|
-
return self.write_df(path, df, _to_bytes)
|
|
140
|
+
return self.write_df(path, df, _to_bytes, mode=mode, key=key)
|
|
135
141
|
|
|
136
142
|
def attach_facets(self, path: str, facets: dict) -> None:
|
|
137
143
|
self._session.attach_facets(path, facets)
|
|
@@ -189,7 +189,10 @@ class LakekeeperStorage:
|
|
|
189
189
|
clean = str(PurePosixPath(path).as_posix().lstrip("/"))
|
|
190
190
|
return f"{location}/{clean}"
|
|
191
191
|
|
|
192
|
-
def put_bytes(self, data: bytes, path: str
|
|
192
|
+
def put_bytes(self, data: bytes, path: str, mode: str = "append",
|
|
193
|
+
key: str | list[str] | None = None) -> None:
|
|
194
|
+
if mode == "upsert":
|
|
195
|
+
raise NotImplementedError("LakekeeperStorage does not support upsert — use TrinoStorage for Iceberg upserts")
|
|
193
196
|
io, loc = self._vend()
|
|
194
197
|
with io.new_output(self._full_path(loc, path)).create(overwrite=True) as f:
|
|
195
198
|
f.write(data)
|
|
@@ -200,7 +203,10 @@ class LakekeeperStorage:
|
|
|
200
203
|
with io.new_input(self._full_path(loc, path)).open() as f:
|
|
201
204
|
return f.read()
|
|
202
205
|
|
|
203
|
-
def put_file(self, local_path: str, path: str
|
|
206
|
+
def put_file(self, local_path: str, path: str, mode: str = "append",
|
|
207
|
+
key: str | list[str] | None = None) -> None:
|
|
208
|
+
if mode == "upsert":
|
|
209
|
+
raise NotImplementedError("LakekeeperStorage does not support upsert — use TrinoStorage for Iceberg upserts")
|
|
204
210
|
io, loc = self._vend()
|
|
205
211
|
data = Path(local_path).read_bytes()
|
|
206
212
|
with io.new_output(self._full_path(loc, path)).create(overwrite=True) as f:
|
|
@@ -5,9 +5,11 @@ from typing import List, Protocol, runtime_checkable
|
|
|
5
5
|
|
|
6
6
|
@runtime_checkable
|
|
7
7
|
class Storage(Protocol):
|
|
8
|
-
def put_file(self, local_path: str, path: str
|
|
8
|
+
def put_file(self, local_path: str, path: str, mode: str = "append",
|
|
9
|
+
key: str | list[str] | None = None) -> None: ...
|
|
9
10
|
def get_file(self, path: str, local_path: str) -> None: ...
|
|
10
|
-
def put_bytes(self, data: bytes, path: str
|
|
11
|
+
def put_bytes(self, data: bytes, path: str, mode: str = "append",
|
|
12
|
+
key: str | list[str] | None = None) -> None: ...
|
|
11
13
|
def get_bytes(self, path: str) -> bytes: ...
|
|
12
14
|
def list(self, prefix: str = "") -> List[str]: ...
|
|
13
15
|
def list_all(self) -> List[str]: ...
|
|
@@ -113,7 +113,10 @@ class S3Storage:
|
|
|
113
113
|
f"S3Storage: {operation} failed — {code} [s3://{self.bucket}/{key} at {self.endpoint}]"
|
|
114
114
|
)
|
|
115
115
|
|
|
116
|
-
def put_file(self, local_path: str, path: str
|
|
116
|
+
def put_file(self, local_path: str, path: str, mode: str = "append",
|
|
117
|
+
key: str | list[str] | None = None) -> None:
|
|
118
|
+
if mode == "upsert":
|
|
119
|
+
raise NotImplementedError("S3Storage does not support upsert — S3 objects have no row identity")
|
|
117
120
|
try:
|
|
118
121
|
self.client.upload_file(local_path, self.bucket, self._key(path))
|
|
119
122
|
logger.info("Uploaded %s", self._object_location(path))
|
|
@@ -132,7 +135,10 @@ class S3Storage:
|
|
|
132
135
|
except ClientError as e:
|
|
133
136
|
self._handle_client_error(e, path, "downloading")
|
|
134
137
|
|
|
135
|
-
def put_bytes(self, data: bytes, path: str
|
|
138
|
+
def put_bytes(self, data: bytes, path: str, mode: str = "append",
|
|
139
|
+
key: str | list[str] | None = None) -> None:
|
|
140
|
+
if mode == "upsert":
|
|
141
|
+
raise NotImplementedError("S3Storage does not support upsert — S3 objects have no row identity")
|
|
136
142
|
try:
|
|
137
143
|
self.client.put_object(Bucket=self.bucket, Key=self._key(path), Body=data)
|
|
138
144
|
logger.info("Stored %s", self._object_location(path))
|
|
@@ -392,13 +392,18 @@ class TrinoStorage:
|
|
|
392
392
|
except Exception as exc:
|
|
393
393
|
logger.warning("Could not ensure Trino schema %s.%s: %s", self.catalog, self.schema, exc)
|
|
394
394
|
|
|
395
|
-
def put_bytes(self, data: bytes, path: str
|
|
395
|
+
def put_bytes(self, data: bytes, path: str, mode: str = "append",
|
|
396
|
+
key: str | list[str] | None = None) -> None:
|
|
396
397
|
from pyiceberg.exceptions import NoSuchTableError, SignError
|
|
397
398
|
|
|
398
399
|
arrow_table = pq.read_table(io.BytesIO(data))
|
|
399
400
|
table_id = f"{self.schema}.{self._table_name(path)}"
|
|
400
401
|
catalog = self._iceberg_catalog
|
|
401
402
|
lakekeeper_base = self._lakekeeper_base
|
|
403
|
+
join_cols = ([key] if isinstance(key, str) else list(key)) if key else []
|
|
404
|
+
|
|
405
|
+
if mode == "upsert" and not join_cols:
|
|
406
|
+
raise ValueError("put_bytes(mode='upsert') requires key= to be set")
|
|
402
407
|
|
|
403
408
|
self._ensure_namespace()
|
|
404
409
|
|
|
@@ -501,7 +506,12 @@ class TrinoStorage:
|
|
|
501
506
|
|
|
502
507
|
try:
|
|
503
508
|
iceberg_table = catalog.load_table(table_id)
|
|
504
|
-
|
|
509
|
+
if mode == "replace":
|
|
510
|
+
iceberg_table.overwrite(arrow_table)
|
|
511
|
+
elif mode == "upsert":
|
|
512
|
+
iceberg_table.upsert(df=arrow_table, join_cols=join_cols)
|
|
513
|
+
else:
|
|
514
|
+
_append(iceberg_table, arrow_table, table_id)
|
|
505
515
|
except NoSuchTableError:
|
|
506
516
|
from pyiceberg.schema import Schema
|
|
507
517
|
from pyiceberg.types import (
|
|
@@ -537,9 +547,22 @@ class TrinoStorage:
|
|
|
537
547
|
iceberg_type = _PYARROW_TO_ICEBERG.get(field.type, StringType())
|
|
538
548
|
fields.append(NestedField(field_id=i + 1, name=field.name, field_type=iceberg_type, required=not field.nullable))
|
|
539
549
|
|
|
540
|
-
|
|
550
|
+
key_names = set(join_cols)
|
|
551
|
+
identifier_ids = [f.field_id for f in fields if f.name in key_names]
|
|
552
|
+
iceberg_schema = Schema(*fields, identifier_field_ids=identifier_ids)
|
|
553
|
+
|
|
554
|
+
cow_props: dict[str, str] = {}
|
|
555
|
+
if mode == "upsert":
|
|
556
|
+
cow_props = {
|
|
557
|
+
"format-version": "2",
|
|
558
|
+
"write.update.mode": "copy-on-write",
|
|
559
|
+
"write.delete.mode": "copy-on-write",
|
|
560
|
+
"write.merge.mode": "copy-on-write",
|
|
561
|
+
}
|
|
562
|
+
|
|
541
563
|
try:
|
|
542
|
-
iceberg_table = catalog.create_table(table_id, schema=iceberg_schema
|
|
564
|
+
iceberg_table = catalog.create_table(table_id, schema=iceberg_schema,
|
|
565
|
+
properties=cow_props or None)
|
|
543
566
|
except SignError as exc:
|
|
544
567
|
raise OSError(
|
|
545
568
|
f"TrinoStorage: Lakekeeper refused to sign table creation for '{table_id}' "
|
|
@@ -558,6 +581,7 @@ class TrinoStorage:
|
|
|
558
581
|
f"TrinoStorage: create table '{table_id}' failed via Lakekeeper [{lakekeeper_base}] — "
|
|
559
582
|
f"{type(exc).__name__}: {exc}"
|
|
560
583
|
) from exc
|
|
584
|
+
# first write on a new table: append is correct for all modes
|
|
561
585
|
_append(iceberg_table, arrow_table, table_id)
|
|
562
586
|
|
|
563
587
|
if os.getenv("STEVE_TRINO_TRACE") == "1":
|
|
@@ -574,5 +598,6 @@ class TrinoStorage:
|
|
|
574
598
|
)
|
|
575
599
|
logger.info("Written %d rows to %s", len(arrow_table), table_id)
|
|
576
600
|
|
|
577
|
-
def put_file(self, local_path: str, path: str
|
|
578
|
-
|
|
601
|
+
def put_file(self, local_path: str, path: str, mode: str = "append",
|
|
602
|
+
key: str | list[str] | None = None) -> None:
|
|
603
|
+
self.put_bytes(Path(local_path).read_bytes(), path, mode=mode, key=key)
|
|
@@ -12,9 +12,11 @@ class Storage(Protocol):
|
|
|
12
12
|
.. deprecated::
|
|
13
13
|
Use ``steve_cli.storage.protocol.Storage`` instead.
|
|
14
14
|
"""
|
|
15
|
-
def put_file(self, local_path: str, path: str
|
|
15
|
+
def put_file(self, local_path: str, path: str, mode: str = "append",
|
|
16
|
+
key: str | list[str] | None = None) -> None: ...
|
|
16
17
|
def get_file(self, path: str, local_path: str) -> None: ...
|
|
17
|
-
def put_bytes(self, data: bytes, path: str
|
|
18
|
+
def put_bytes(self, data: bytes, path: str, mode: str = "append",
|
|
19
|
+
key: str | list[str] | None = None) -> None: ...
|
|
18
20
|
def get_bytes(self, path: str) -> bytes: ...
|
|
19
21
|
def list(self, prefix: str = "") -> List[str]: ...
|
|
20
22
|
|
|
@@ -77,7 +79,10 @@ class S3Storage:
|
|
|
77
79
|
key = self._key(path)
|
|
78
80
|
return f"{self.endpoint}/{self.bucket}/{key}"
|
|
79
81
|
|
|
80
|
-
def put_file(self, local_path: str, path: str
|
|
82
|
+
def put_file(self, local_path: str, path: str, mode: str = "append",
|
|
83
|
+
key: str | list[str] | None = None) -> None:
|
|
84
|
+
if mode == "upsert":
|
|
85
|
+
raise NotImplementedError("S3Storage does not support upsert — S3 objects have no row identity")
|
|
81
86
|
absolute_url = self._get_absolute_url(path)
|
|
82
87
|
try:
|
|
83
88
|
self.client.upload_file(local_path, self.bucket, self._key(path))
|
|
@@ -114,7 +119,10 @@ class S3Storage:
|
|
|
114
119
|
print(f"❌ Storage error when reading: {path}: {e}", file=sys.stderr)
|
|
115
120
|
sys.exit(1)
|
|
116
121
|
|
|
117
|
-
def put_bytes(self, data: bytes, path: str
|
|
122
|
+
def put_bytes(self, data: bytes, path: str, mode: str = "append",
|
|
123
|
+
key: str | list[str] | None = None) -> None:
|
|
124
|
+
if mode == "upsert":
|
|
125
|
+
raise NotImplementedError("S3Storage does not support upsert — S3 objects have no row identity")
|
|
118
126
|
absolute_url = self._get_absolute_url(path)
|
|
119
127
|
try:
|
|
120
128
|
self.client.put_object(
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|