s3shift 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- s3shift/__init__.py +9 -0
- s3shift/redshift_connector.py +297 -0
- s3shift-0.1.0.dist-info/METADATA +49 -0
- s3shift-0.1.0.dist-info/RECORD +6 -0
- s3shift-0.1.0.dist-info/WHEEL +5 -0
- s3shift-0.1.0.dist-info/top_level.txt +1 -0
s3shift/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""s3shift — a truly serverless Redshift: Redshift SQL on DuckDB in a Lambda, over your own S3 and RDS data.
|
|
2
|
+
|
|
3
|
+
from s3shift import redshift_connector # later: `import redshift_connector` against Redshift
|
|
4
|
+
conn = redshift_connector.connect(host="xxx.lambda-url.us-east-2.on.aws", database="dev",
|
|
5
|
+
user="admin", password="...")
|
|
6
|
+
"""
|
|
7
|
+
from . import redshift_connector # noqa: F401
|
|
8
|
+
|
|
9
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
"""s3shift.redshift_connector — the redshift_connector API (connect, cursors, fetch_dataframe, DB-API errors)
|
|
2
|
+
over HTTP to the s3shift Lambda URL. Standard library only (pandas only for fetch_dataframe).
|
|
3
|
+
|
|
4
|
+
from s3shift import redshift_connector
|
|
5
|
+
conn = redshift_connector.connect(host="xxx.lambda-url.us-east-2.on.aws", database="dev",
|
|
6
|
+
user="admin", password="...")
|
|
7
|
+
cur = conn.cursor()
|
|
8
|
+
cur.execute("SELECT region, SUM(amount) FROM sales WHERE day >= %s GROUP BY 1 ORDER BY 2 DESC", ["2026-01-01"])
|
|
9
|
+
cur.fetchall() # ([region, Decimal(...)], ...)
|
|
10
|
+
cur.fetch_dataframe() # pandas, if installed
|
|
11
|
+
|
|
12
|
+
Every statement is committed when execute() returns (s3shift has no sessions); commit() is a no-op.
|
|
13
|
+
Large results are fetched page by page inside execute(), so fetchall() is always complete.
|
|
14
|
+
Retry policy: a request is re-sent once only when it provably never reached the server.
|
|
15
|
+
"""
|
|
16
|
+
import base64
|
|
17
|
+
import datetime
|
|
18
|
+
import decimal
|
|
19
|
+
import http.client
|
|
20
|
+
import json
|
|
21
|
+
import socket
|
|
22
|
+
import urllib.parse
|
|
23
|
+
import warnings
|
|
24
|
+
|
|
25
|
+
__version__ = "0.1.0"
|
|
26
|
+
apilevel = "2.0"
|
|
27
|
+
threadsafety = 1
|
|
28
|
+
paramstyle = "format"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
# ---------------- DB-API exceptions (redshift_connector's names) ----------------
|
|
32
|
+
|
|
33
|
+
class Warning(Exception): pass # noqa: A001
|
|
34
|
+
class Error(Exception): pass
|
|
35
|
+
class InterfaceError(Error): pass
|
|
36
|
+
class DatabaseError(Error): pass
|
|
37
|
+
class DataError(DatabaseError): pass
|
|
38
|
+
class OperationalError(DatabaseError): pass
|
|
39
|
+
class IntegrityError(DatabaseError): pass
|
|
40
|
+
class InternalError(DatabaseError): pass
|
|
41
|
+
class ProgrammingError(DatabaseError): pass
|
|
42
|
+
class NotSupportedError(DatabaseError): pass
|
|
43
|
+
|
|
44
|
+
_ERRORS = {c.__name__: c for c in (DataError, OperationalError, IntegrityError, InternalError, ProgrammingError,
|
|
45
|
+
NotSupportedError, DatabaseError)}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class _NotDelivered(Exception):
|
|
49
|
+
def __init__(self, cause, reused):
|
|
50
|
+
super().__init__(cause)
|
|
51
|
+
self.reused = reused
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# ---------------- values ----------------
|
|
55
|
+
|
|
56
|
+
def _encode(v, i):
|
|
57
|
+
if v is None or isinstance(v, (bool, int, float, str)):
|
|
58
|
+
return v
|
|
59
|
+
if isinstance(v, datetime.datetime):
|
|
60
|
+
return {"$ts": v.isoformat(sep=" ")}
|
|
61
|
+
if isinstance(v, datetime.date):
|
|
62
|
+
return {"$d": v.isoformat()}
|
|
63
|
+
if isinstance(v, decimal.Decimal):
|
|
64
|
+
return {"$n": str(v)}
|
|
65
|
+
if isinstance(v, (bytes, bytearray, memoryview)):
|
|
66
|
+
return {"$b": base64.b64encode(bytes(v)).decode()}
|
|
67
|
+
raise InterfaceError(f"parameter #{i}: type {type(v).__name__} cannot be sent")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _decode(v):
|
|
71
|
+
if not isinstance(v, dict) or len(v) != 1:
|
|
72
|
+
return v
|
|
73
|
+
(tag, text), = v.items()
|
|
74
|
+
if tag == "$n":
|
|
75
|
+
return decimal.Decimal(text)
|
|
76
|
+
if tag == "$d":
|
|
77
|
+
return datetime.date.fromisoformat(text)
|
|
78
|
+
if tag == "$ts":
|
|
79
|
+
return datetime.datetime.fromisoformat(text)
|
|
80
|
+
if tag == "$t":
|
|
81
|
+
return datetime.time.fromisoformat(text)
|
|
82
|
+
if tag == "$b":
|
|
83
|
+
return base64.b64decode(text)
|
|
84
|
+
if tag == "$f":
|
|
85
|
+
return float(text)
|
|
86
|
+
return v
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _bind(sql, params):
|
|
90
|
+
"""%s / %(name)s -> ?, %% -> %; quoted text is left alone. Returns (sql, list of params)."""
|
|
91
|
+
names, out, i, n, quote = [], [], 0, len(sql), None
|
|
92
|
+
while i < n:
|
|
93
|
+
ch = sql[i]
|
|
94
|
+
if quote:
|
|
95
|
+
out.append(ch)
|
|
96
|
+
if ch == quote:
|
|
97
|
+
quote = None
|
|
98
|
+
elif ch in ("'", '"'):
|
|
99
|
+
quote = ch
|
|
100
|
+
out.append(ch)
|
|
101
|
+
elif ch == "%" and i + 1 < n:
|
|
102
|
+
nxt = sql[i + 1]
|
|
103
|
+
if nxt == "s":
|
|
104
|
+
out.append("?"); names.append(None); i += 1
|
|
105
|
+
elif nxt == "%":
|
|
106
|
+
out.append("%"); i += 1
|
|
107
|
+
elif nxt == "(":
|
|
108
|
+
end = sql.find(")s", i)
|
|
109
|
+
if end == -1:
|
|
110
|
+
raise ProgrammingError(f"bad placeholder near {sql[i:i + 20]!r}")
|
|
111
|
+
out.append("?"); names.append(sql[i + 2:end]); i = end + 1
|
|
112
|
+
else:
|
|
113
|
+
out.append(ch)
|
|
114
|
+
else:
|
|
115
|
+
out.append(ch)
|
|
116
|
+
i += 1
|
|
117
|
+
if params is None:
|
|
118
|
+
if names:
|
|
119
|
+
raise ProgrammingError("the statement has placeholders but no parameters were given")
|
|
120
|
+
return sql, []
|
|
121
|
+
if isinstance(params, dict):
|
|
122
|
+
return "".join(out), [params[k] for k in names]
|
|
123
|
+
params = list(params)
|
|
124
|
+
if len(params) != len(names):
|
|
125
|
+
raise ProgrammingError(f"{len(names)} placeholders, {len(params)} parameters")
|
|
126
|
+
return "".join(out), params
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# ---------------- connection ----------------
|
|
130
|
+
|
|
131
|
+
class Connection:
|
|
132
|
+
def __init__(self, host, database="dev", user="", password="", timeout=900):
|
|
133
|
+
url = host if host.startswith("http") else "https://" + host
|
|
134
|
+
p = urllib.parse.urlparse(url.rstrip("/"))
|
|
135
|
+
self._host, self._base_path = p.netloc, p.path
|
|
136
|
+
self.database = database
|
|
137
|
+
self._auth = base64.b64encode(f"{user}:{password}".encode()).decode()
|
|
138
|
+
self.timeout = timeout
|
|
139
|
+
self._conn = None
|
|
140
|
+
self.autocommit = True
|
|
141
|
+
self._written = False
|
|
142
|
+
self.last = {} # {"ms": server time, "scanned": bytes read from S3} of the last statement
|
|
143
|
+
|
|
144
|
+
# ---- transport ----
|
|
145
|
+
|
|
146
|
+
def close(self):
|
|
147
|
+
try:
|
|
148
|
+
if self._conn:
|
|
149
|
+
self._conn.close()
|
|
150
|
+
finally:
|
|
151
|
+
self._conn = None
|
|
152
|
+
|
|
153
|
+
def __enter__(self):
|
|
154
|
+
return self
|
|
155
|
+
|
|
156
|
+
def __exit__(self, *a):
|
|
157
|
+
self.close()
|
|
158
|
+
|
|
159
|
+
def _send(self, path, body):
|
|
160
|
+
reused = self._conn is not None
|
|
161
|
+
if self._conn is None:
|
|
162
|
+
self._conn = http.client.HTTPSConnection(self._host, timeout=self.timeout)
|
|
163
|
+
headers = {"content-type": "application/json", "authorization": f"Basic {self._auth}"}
|
|
164
|
+
try:
|
|
165
|
+
self._conn.request("POST", self._base_path + path, body, headers)
|
|
166
|
+
except (socket.timeout, TimeoutError):
|
|
167
|
+
raise
|
|
168
|
+
except (OSError, http.client.HTTPException) as e:
|
|
169
|
+
raise _NotDelivered(e, reused) from None
|
|
170
|
+
try:
|
|
171
|
+
resp = self._conn.getresponse()
|
|
172
|
+
except (http.client.RemoteDisconnected, http.client.BadStatusLine) as e:
|
|
173
|
+
raise _NotDelivered(e, reused) from None
|
|
174
|
+
raw = resp.read()
|
|
175
|
+
try:
|
|
176
|
+
data = json.loads(raw)
|
|
177
|
+
except ValueError:
|
|
178
|
+
data = {"error": raw[:200].decode(errors="replace")}
|
|
179
|
+
return resp.status, data
|
|
180
|
+
|
|
181
|
+
def _post(self, path, payload):
|
|
182
|
+
body = json.dumps(payload)
|
|
183
|
+
try:
|
|
184
|
+
status, data = self._send(path, body)
|
|
185
|
+
except _NotDelivered as e:
|
|
186
|
+
self.close()
|
|
187
|
+
if not e.reused:
|
|
188
|
+
raise InterfaceError(f"cannot reach {self._host}: {e}") from None
|
|
189
|
+
status, data = self._send(path, body)
|
|
190
|
+
except (socket.timeout, TimeoutError) as e:
|
|
191
|
+
self.close()
|
|
192
|
+
raise OperationalError(f"timeout talking to {self._host}; result unknown ({e})") from None
|
|
193
|
+
if status == 401:
|
|
194
|
+
raise InterfaceError("authentication failed (user / password)")
|
|
195
|
+
if status >= 400:
|
|
196
|
+
raise _ERRORS.get(data.get("kind"), DatabaseError)(data.get("error", f"HTTP {status}"))
|
|
197
|
+
return data
|
|
198
|
+
|
|
199
|
+
def run(self, sql, params=None):
|
|
200
|
+
"""One statement; returns {columns, rows (decoded, complete), rowcount, scanned, ms}."""
|
|
201
|
+
qmark, args = _bind(sql, params)
|
|
202
|
+
data = self._post("/api/query", {"sql": qmark, "params": [_encode(a, i + 1) for i, a in enumerate(args)],
|
|
203
|
+
"database": self.database})
|
|
204
|
+
rows = data["rows"]
|
|
205
|
+
while data.get("next_token"):
|
|
206
|
+
data = self._post("/api/query", {"next_token": data["next_token"]})
|
|
207
|
+
rows += data["rows"]
|
|
208
|
+
first = {"columns": data["columns"], "rows": [[_decode(v) for v in r] for r in rows],
|
|
209
|
+
"rowcount": data["rowcount"], "scanned": data.get("scanned"), "ms": data.get("ms")}
|
|
210
|
+
self.last = {"ms": first["ms"], "scanned": first["scanned"]}
|
|
211
|
+
if not first["columns"] and first["rowcount"]:
|
|
212
|
+
self._written = True
|
|
213
|
+
return first
|
|
214
|
+
|
|
215
|
+
# ---- DB-API ----
|
|
216
|
+
|
|
217
|
+
def cursor(self):
|
|
218
|
+
return Cursor(self)
|
|
219
|
+
|
|
220
|
+
def commit(self):
|
|
221
|
+
self._written = False
|
|
222
|
+
|
|
223
|
+
def rollback(self):
|
|
224
|
+
if self._written:
|
|
225
|
+
warnings.warn("rollback() cannot undo statements that already ran: every statement is committed "
|
|
226
|
+
"when execute() returns", RuntimeWarning, stacklevel=2)
|
|
227
|
+
self._written = False
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
class Cursor:
|
|
231
|
+
def __init__(self, connection):
|
|
232
|
+
self.connection = connection
|
|
233
|
+
self.description = None
|
|
234
|
+
self.rowcount = -1
|
|
235
|
+
self.arraysize = 1
|
|
236
|
+
self._rows, self._pos = [], 0
|
|
237
|
+
|
|
238
|
+
def execute(self, operation, args=None, stream=None, merge_socket_read=False):
|
|
239
|
+
r = self.connection.run(operation, args)
|
|
240
|
+
cols = r["columns"]
|
|
241
|
+
self.description = [(c, None, None, None, None, None, None) for c in cols] or None
|
|
242
|
+
self.rowcount = r["rowcount"]
|
|
243
|
+
self._rows, self._pos = r["rows"], 0
|
|
244
|
+
return self
|
|
245
|
+
|
|
246
|
+
def executemany(self, operation, param_sets):
|
|
247
|
+
total = 0
|
|
248
|
+
for p in param_sets:
|
|
249
|
+
self.execute(operation, p)
|
|
250
|
+
total += max(self.rowcount, 0)
|
|
251
|
+
self.rowcount = total
|
|
252
|
+
return self
|
|
253
|
+
|
|
254
|
+
def fetchone(self):
|
|
255
|
+
if self._pos >= len(self._rows):
|
|
256
|
+
return None
|
|
257
|
+
row = self._rows[self._pos]
|
|
258
|
+
self._pos += 1
|
|
259
|
+
return row
|
|
260
|
+
|
|
261
|
+
def fetchmany(self, num=None):
|
|
262
|
+
n = num or self.arraysize
|
|
263
|
+
out = self._rows[self._pos:self._pos + n]
|
|
264
|
+
self._pos += len(out)
|
|
265
|
+
return tuple(out)
|
|
266
|
+
|
|
267
|
+
def fetchall(self):
|
|
268
|
+
out = self._rows[self._pos:]
|
|
269
|
+
self._pos = len(self._rows)
|
|
270
|
+
return tuple(out)
|
|
271
|
+
|
|
272
|
+
def fetch_dataframe(self, num=None):
|
|
273
|
+
import pandas
|
|
274
|
+
rows = self.fetchmany(num) if num else self.fetchall()
|
|
275
|
+
return pandas.DataFrame(list(rows), columns=[d[0] for d in self.description or []])
|
|
276
|
+
|
|
277
|
+
def __iter__(self):
|
|
278
|
+
while True:
|
|
279
|
+
row = self.fetchone()
|
|
280
|
+
if row is None:
|
|
281
|
+
return
|
|
282
|
+
yield row
|
|
283
|
+
|
|
284
|
+
def close(self):
|
|
285
|
+
self._rows = []
|
|
286
|
+
|
|
287
|
+
def __enter__(self):
|
|
288
|
+
return self
|
|
289
|
+
|
|
290
|
+
def __exit__(self, *a):
|
|
291
|
+
self.close()
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def connect(host, database="dev", user="", password="", port=None, timeout=900, **_ignored):
|
|
295
|
+
"""Same signature as redshift_connector.connect; host is the Lambda URL. Extra arguments
|
|
296
|
+
(port, ssl, iam, region, ...) are accepted and ignored."""
|
|
297
|
+
return Connection(host, database, user, password, timeout)
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: s3shift
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A truly serverless Redshift on your own AWS account: Redshift SQL on DuckDB in a Lambda, over your S3 and RDS data, ≈ $0 when idle. redshift_connector API: from s3shift import redshift_connector.
|
|
5
|
+
Author: Frank Zhang
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/cloudformer/s3shift
|
|
8
|
+
Project-URL: Repository, https://github.com/cloudformer/s3shift
|
|
9
|
+
Project-URL: Deploy, https://github.com/cloudformer/s3shift#quick-start
|
|
10
|
+
Project-URL: Compatibility, https://github.com/cloudformer/s3shift/blob/main/tests/compat/REPORT.md
|
|
11
|
+
Project-URL: Changelog, https://github.com/cloudformer/s3shift/blob/main/CHANGELOG.md
|
|
12
|
+
Project-URL: Issues, https://github.com/cloudformer/s3shift/issues
|
|
13
|
+
Keywords: serverless,aws,lambda,redshift,duckdb,s3,data-warehouse,sql,analytics
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Topic :: Database
|
|
19
|
+
Requires-Python: >=3.9
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
|
|
22
|
+
# s3shift
|
|
23
|
+
|
|
24
|
+
**Python client for [s3shift](https://github.com/cloudformer/s3shift) — a truly serverless Redshift: Redshift
|
|
25
|
+
SQL on DuckDB in a Lambda, over your own S3 and RDS data, ≈ $0 when idle.** The redshift_connector API,
|
|
26
|
+
standard library only. Part of [Redefine AWS Serverless](https://github.com/topics/redefine-aws-serverless).
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install s3shift
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from s3shift import redshift_connector # later: `import redshift_connector` against Redshift, nothing else changes
|
|
34
|
+
|
|
35
|
+
conn = redshift_connector.connect(host="xxx.lambda-url.us-east-2.on.aws", database="dev",
|
|
36
|
+
user="admin", password="...")
|
|
37
|
+
cur = conn.cursor()
|
|
38
|
+
cur.execute("CREATE EXTERNAL TABLE clicks (user_id INT, day DATE) STORED AS PARQUET LOCATION 's3://my-lake/clicks/'")
|
|
39
|
+
cur.execute("SELECT day, count(*) FROM clicks WHERE day >= %s GROUP BY 1 ORDER BY 1", ["2026-01-01"])
|
|
40
|
+
cur.fetchall() # ([datetime.date(2026, 1, 1), 1520], ...)
|
|
41
|
+
cur.fetch_dataframe() # pandas, if installed
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Every statement is committed when `execute()` returns; large results are fetched page by page inside
|
|
45
|
+
`execute()`. Not supported (raises `NotSupportedError`): stored procedures, UDFs, users / GRANT, Glue
|
|
46
|
+
Data Catalog schemas, materialized views.
|
|
47
|
+
|
|
48
|
+
The endpoint comes from deploying the [main repository](https://github.com/cloudformer/s3shift).
|
|
49
|
+
Compatibility: [REPORT.md](https://github.com/cloudformer/s3shift/blob/main/tests/compat/REPORT.md).
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
s3shift/__init__.py,sha256=guRFIAJC7PchT7jz8PMDGuIVh1P2963PL5mAGvPWtj4,448
|
|
2
|
+
s3shift/redshift_connector.py,sha256=bH-KQvAbYPr5VNZAJqr13mhgsXjwGOoMoUrtCbcq9ps,10518
|
|
3
|
+
s3shift-0.1.0.dist-info/METADATA,sha256=DWv-3YeQcyiYUfE0tpg3Cr2xtj8LjGzptgmdlY_gSIk,2564
|
|
4
|
+
s3shift-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
5
|
+
s3shift-0.1.0.dist-info/top_level.txt,sha256=ewbUxlxS9sV8Lx8oJGLBcGk0d_fBrUzFxjIy1PTAvWE,8
|
|
6
|
+
s3shift-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
s3shift
|