s3shift 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
s3shift-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,49 @@
1
+ Metadata-Version: 2.4
2
+ Name: s3shift
3
+ Version: 0.1.0
4
+ Summary: A truly serverless Redshift on your own AWS account: Redshift SQL on DuckDB in a Lambda, over your S3 and RDS data, ≈ $0 when idle. redshift_connector API: from s3shift import redshift_connector.
5
+ Author: Frank Zhang
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/cloudformer/s3shift
8
+ Project-URL: Repository, https://github.com/cloudformer/s3shift
9
+ Project-URL: Deploy, https://github.com/cloudformer/s3shift#quick-start
10
+ Project-URL: Compatibility, https://github.com/cloudformer/s3shift/blob/main/tests/compat/REPORT.md
11
+ Project-URL: Changelog, https://github.com/cloudformer/s3shift/blob/main/CHANGELOG.md
12
+ Project-URL: Issues, https://github.com/cloudformer/s3shift/issues
13
+ Keywords: serverless,aws,lambda,redshift,duckdb,s3,data-warehouse,sql,analytics
14
+ Classifier: Development Status :: 3 - Alpha
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: License :: OSI Approved :: MIT License
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Topic :: Database
19
+ Requires-Python: >=3.9
20
+ Description-Content-Type: text/markdown
21
+
22
+ # s3shift
23
+
24
+ **Python client for [s3shift](https://github.com/cloudformer/s3shift) — a truly serverless Redshift: Redshift
25
+ SQL on DuckDB in a Lambda, over your own S3 and RDS data, ≈ $0 when idle.** The redshift_connector API,
26
+ standard library only. Part of [Redefine AWS Serverless](https://github.com/topics/redefine-aws-serverless).
27
+
28
+ ```bash
29
+ pip install s3shift
30
+ ```
31
+
32
+ ```python
33
+ from s3shift import redshift_connector # later: `import redshift_connector` against Redshift, nothing else changes
34
+
35
+ conn = redshift_connector.connect(host="xxx.lambda-url.us-east-2.on.aws", database="dev",
36
+ user="admin", password="...")
37
+ cur = conn.cursor()
38
+ cur.execute("CREATE EXTERNAL TABLE clicks (user_id INT, day DATE) STORED AS PARQUET LOCATION 's3://my-lake/clicks/'")
39
+ cur.execute("SELECT day, count(*) FROM clicks WHERE day >= %s GROUP BY 1 ORDER BY 1", ["2026-01-01"])
40
+ cur.fetchall() # ([datetime.date(2026, 1, 1), 1520], ...)
41
+ cur.fetch_dataframe() # pandas, if installed
42
+ ```
43
+
44
+ Every statement is committed when `execute()` returns; large results are fetched page by page inside
45
+ `execute()`. Not supported (raises `NotSupportedError`): stored procedures, UDFs, users / GRANT, Glue
46
+ Data Catalog schemas, materialized views.
47
+
48
+ The endpoint comes from deploying the [main repository](https://github.com/cloudformer/s3shift).
49
+ Compatibility: [REPORT.md](https://github.com/cloudformer/s3shift/blob/main/tests/compat/REPORT.md).
@@ -0,0 +1,28 @@
1
+ # s3shift
2
+
3
+ **Python client for [s3shift](https://github.com/cloudformer/s3shift) — a truly serverless Redshift: Redshift
4
+ SQL on DuckDB in a Lambda, over your own S3 and RDS data, ≈ $0 when idle.** The redshift_connector API,
5
+ standard library only. Part of [Redefine AWS Serverless](https://github.com/topics/redefine-aws-serverless).
6
+
7
+ ```bash
8
+ pip install s3shift
9
+ ```
10
+
11
+ ```python
12
+ from s3shift import redshift_connector # later: `import redshift_connector` against Redshift, nothing else changes
13
+
14
+ conn = redshift_connector.connect(host="xxx.lambda-url.us-east-2.on.aws", database="dev",
15
+ user="admin", password="...")
16
+ cur = conn.cursor()
17
+ cur.execute("CREATE EXTERNAL TABLE clicks (user_id INT, day DATE) STORED AS PARQUET LOCATION 's3://my-lake/clicks/'")
18
+ cur.execute("SELECT day, count(*) FROM clicks WHERE day >= %s GROUP BY 1 ORDER BY 1", ["2026-01-01"])
19
+ cur.fetchall() # ([datetime.date(2026, 1, 1), 1520], ...)
20
+ cur.fetch_dataframe() # pandas, if installed
21
+ ```
22
+
23
+ Every statement is committed when `execute()` returns; large results are fetched page by page inside
24
+ `execute()`. Not supported (raises `NotSupportedError`): stored procedures, UDFs, users / GRANT, Glue
25
+ Data Catalog schemas, materialized views.
26
+
27
+ The endpoint comes from deploying the [main repository](https://github.com/cloudformer/s3shift).
28
+ Compatibility: [REPORT.md](https://github.com/cloudformer/s3shift/blob/main/tests/compat/REPORT.md).
@@ -0,0 +1,32 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "s3shift"
7
+ version = "0.1.0"
8
+ description = "A truly serverless Redshift on your own AWS account: Redshift SQL on DuckDB in a Lambda, over your S3 and RDS data, ≈ $0 when idle. redshift_connector API: from s3shift import redshift_connector."
9
+ readme = "README.md"
10
+ requires-python = ">=3.9"
11
+ license = { text = "MIT" }
12
+ authors = [{ name = "Frank Zhang" }]
13
+ keywords = ["serverless", "aws", "lambda", "redshift", "duckdb", "s3", "data-warehouse", "sql", "analytics"]
14
+ classifiers = [
15
+ "Development Status :: 3 - Alpha",
16
+ "Intended Audience :: Developers",
17
+ "License :: OSI Approved :: MIT License",
18
+ "Programming Language :: Python :: 3",
19
+ "Topic :: Database",
20
+ ]
21
+ dependencies = [] # standard library only (pandas only for fetch_dataframe)
22
+
23
+ [project.urls]
24
+ Homepage = "https://github.com/cloudformer/s3shift"
25
+ Repository = "https://github.com/cloudformer/s3shift"
26
+ Deploy = "https://github.com/cloudformer/s3shift#quick-start"
27
+ Compatibility = "https://github.com/cloudformer/s3shift/blob/main/tests/compat/REPORT.md"
28
+ Changelog = "https://github.com/cloudformer/s3shift/blob/main/CHANGELOG.md"
29
+ Issues = "https://github.com/cloudformer/s3shift/issues"
30
+
31
+ [tool.setuptools]
32
+ packages = ["s3shift"]
@@ -0,0 +1,9 @@
1
+ """s3shift — a truly serverless Redshift: Redshift SQL on DuckDB in a Lambda, over your own S3 and RDS data.
2
+
3
+ from s3shift import redshift_connector # later: `import redshift_connector` against Redshift
4
+ conn = redshift_connector.connect(host="xxx.lambda-url.us-east-2.on.aws", database="dev",
5
+ user="admin", password="...")
6
+ """
7
+ from . import redshift_connector # noqa: F401
8
+
9
+ __version__ = "0.1.0"
@@ -0,0 +1,297 @@
1
+ """s3shift.redshift_connector — the redshift_connector API (connect, cursors, fetch_dataframe, DB-API errors)
2
+ over HTTP to the s3shift Lambda URL. Standard library only (pandas only for fetch_dataframe).
3
+
4
+ from s3shift import redshift_connector
5
+ conn = redshift_connector.connect(host="xxx.lambda-url.us-east-2.on.aws", database="dev",
6
+ user="admin", password="...")
7
+ cur = conn.cursor()
8
+ cur.execute("SELECT region, SUM(amount) FROM sales WHERE day >= %s GROUP BY 1 ORDER BY 2 DESC", ["2026-01-01"])
9
+ cur.fetchall() # ([region, Decimal(...)], ...)
10
+ cur.fetch_dataframe() # pandas, if installed
11
+
12
+ Every statement is committed when execute() returns (s3shift has no sessions); commit() is a no-op.
13
+ Large results are fetched page by page inside execute(), so fetchall() is always complete.
14
+ Retry policy: a request is re-sent once only when it provably never reached the server.
15
+ """
16
+ import base64
17
+ import datetime
18
+ import decimal
19
+ import http.client
20
+ import json
21
+ import socket
22
+ import urllib.parse
23
+ import warnings
24
+
25
+ __version__ = "0.1.0"
26
+ apilevel = "2.0"
27
+ threadsafety = 1
28
+ paramstyle = "format"
29
+
30
+
31
+ # ---------------- DB-API exceptions (redshift_connector's names) ----------------
32
+
33
+ class Warning(Exception): pass # noqa: A001
34
+ class Error(Exception): pass
35
+ class InterfaceError(Error): pass
36
+ class DatabaseError(Error): pass
37
+ class DataError(DatabaseError): pass
38
+ class OperationalError(DatabaseError): pass
39
+ class IntegrityError(DatabaseError): pass
40
+ class InternalError(DatabaseError): pass
41
+ class ProgrammingError(DatabaseError): pass
42
+ class NotSupportedError(DatabaseError): pass
43
+
44
+ _ERRORS = {c.__name__: c for c in (DataError, OperationalError, IntegrityError, InternalError, ProgrammingError,
45
+ NotSupportedError, DatabaseError)}
46
+
47
+
48
+ class _NotDelivered(Exception):
49
+ def __init__(self, cause, reused):
50
+ super().__init__(cause)
51
+ self.reused = reused
52
+
53
+
54
+ # ---------------- values ----------------
55
+
56
+ def _encode(v, i):
57
+ if v is None or isinstance(v, (bool, int, float, str)):
58
+ return v
59
+ if isinstance(v, datetime.datetime):
60
+ return {"$ts": v.isoformat(sep=" ")}
61
+ if isinstance(v, datetime.date):
62
+ return {"$d": v.isoformat()}
63
+ if isinstance(v, decimal.Decimal):
64
+ return {"$n": str(v)}
65
+ if isinstance(v, (bytes, bytearray, memoryview)):
66
+ return {"$b": base64.b64encode(bytes(v)).decode()}
67
+ raise InterfaceError(f"parameter #{i}: type {type(v).__name__} cannot be sent")
68
+
69
+
70
+ def _decode(v):
71
+ if not isinstance(v, dict) or len(v) != 1:
72
+ return v
73
+ (tag, text), = v.items()
74
+ if tag == "$n":
75
+ return decimal.Decimal(text)
76
+ if tag == "$d":
77
+ return datetime.date.fromisoformat(text)
78
+ if tag == "$ts":
79
+ return datetime.datetime.fromisoformat(text)
80
+ if tag == "$t":
81
+ return datetime.time.fromisoformat(text)
82
+ if tag == "$b":
83
+ return base64.b64decode(text)
84
+ if tag == "$f":
85
+ return float(text)
86
+ return v
87
+
88
+
89
+ def _bind(sql, params):
90
+ """%s / %(name)s -> ?, %% -> %; quoted text is left alone. Returns (sql, list of params)."""
91
+ names, out, i, n, quote = [], [], 0, len(sql), None
92
+ while i < n:
93
+ ch = sql[i]
94
+ if quote:
95
+ out.append(ch)
96
+ if ch == quote:
97
+ quote = None
98
+ elif ch in ("'", '"'):
99
+ quote = ch
100
+ out.append(ch)
101
+ elif ch == "%" and i + 1 < n:
102
+ nxt = sql[i + 1]
103
+ if nxt == "s":
104
+ out.append("?"); names.append(None); i += 1
105
+ elif nxt == "%":
106
+ out.append("%"); i += 1
107
+ elif nxt == "(":
108
+ end = sql.find(")s", i)
109
+ if end == -1:
110
+ raise ProgrammingError(f"bad placeholder near {sql[i:i + 20]!r}")
111
+ out.append("?"); names.append(sql[i + 2:end]); i = end + 1
112
+ else:
113
+ out.append(ch)
114
+ else:
115
+ out.append(ch)
116
+ i += 1
117
+ if params is None:
118
+ if names:
119
+ raise ProgrammingError("the statement has placeholders but no parameters were given")
120
+ return sql, []
121
+ if isinstance(params, dict):
122
+ return "".join(out), [params[k] for k in names]
123
+ params = list(params)
124
+ if len(params) != len(names):
125
+ raise ProgrammingError(f"{len(names)} placeholders, {len(params)} parameters")
126
+ return "".join(out), params
127
+
128
+
129
+ # ---------------- connection ----------------
130
+
131
+ class Connection:
132
+ def __init__(self, host, database="dev", user="", password="", timeout=900):
133
+ url = host if host.startswith("http") else "https://" + host
134
+ p = urllib.parse.urlparse(url.rstrip("/"))
135
+ self._host, self._base_path = p.netloc, p.path
136
+ self.database = database
137
+ self._auth = base64.b64encode(f"{user}:{password}".encode()).decode()
138
+ self.timeout = timeout
139
+ self._conn = None
140
+ self.autocommit = True
141
+ self._written = False
142
+ self.last = {} # {"ms": server time, "scanned": bytes read from S3} of the last statement
143
+
144
+ # ---- transport ----
145
+
146
+ def close(self):
147
+ try:
148
+ if self._conn:
149
+ self._conn.close()
150
+ finally:
151
+ self._conn = None
152
+
153
+ def __enter__(self):
154
+ return self
155
+
156
+ def __exit__(self, *a):
157
+ self.close()
158
+
159
+ def _send(self, path, body):
160
+ reused = self._conn is not None
161
+ if self._conn is None:
162
+ self._conn = http.client.HTTPSConnection(self._host, timeout=self.timeout)
163
+ headers = {"content-type": "application/json", "authorization": f"Basic {self._auth}"}
164
+ try:
165
+ self._conn.request("POST", self._base_path + path, body, headers)
166
+ except (socket.timeout, TimeoutError):
167
+ raise
168
+ except (OSError, http.client.HTTPException) as e:
169
+ raise _NotDelivered(e, reused) from None
170
+ try:
171
+ resp = self._conn.getresponse()
172
+ except (http.client.RemoteDisconnected, http.client.BadStatusLine) as e:
173
+ raise _NotDelivered(e, reused) from None
174
+ raw = resp.read()
175
+ try:
176
+ data = json.loads(raw)
177
+ except ValueError:
178
+ data = {"error": raw[:200].decode(errors="replace")}
179
+ return resp.status, data
180
+
181
+ def _post(self, path, payload):
182
+ body = json.dumps(payload)
183
+ try:
184
+ status, data = self._send(path, body)
185
+ except _NotDelivered as e:
186
+ self.close()
187
+ if not e.reused:
188
+ raise InterfaceError(f"cannot reach {self._host}: {e}") from None
189
+ status, data = self._send(path, body)
190
+ except (socket.timeout, TimeoutError) as e:
191
+ self.close()
192
+ raise OperationalError(f"timeout talking to {self._host}; result unknown ({e})") from None
193
+ if status == 401:
194
+ raise InterfaceError("authentication failed (user / password)")
195
+ if status >= 400:
196
+ raise _ERRORS.get(data.get("kind"), DatabaseError)(data.get("error", f"HTTP {status}"))
197
+ return data
198
+
199
+ def run(self, sql, params=None):
200
+ """One statement; returns {columns, rows (decoded, complete), rowcount, scanned, ms}."""
201
+ qmark, args = _bind(sql, params)
202
+ data = self._post("/api/query", {"sql": qmark, "params": [_encode(a, i + 1) for i, a in enumerate(args)],
203
+ "database": self.database})
204
+ rows = data["rows"]
205
+ while data.get("next_token"):
206
+ data = self._post("/api/query", {"next_token": data["next_token"]})
207
+ rows += data["rows"]
208
+ first = {"columns": data["columns"], "rows": [[_decode(v) for v in r] for r in rows],
209
+ "rowcount": data["rowcount"], "scanned": data.get("scanned"), "ms": data.get("ms")}
210
+ self.last = {"ms": first["ms"], "scanned": first["scanned"]}
211
+ if not first["columns"] and first["rowcount"]:
212
+ self._written = True
213
+ return first
214
+
215
+ # ---- DB-API ----
216
+
217
+ def cursor(self):
218
+ return Cursor(self)
219
+
220
+ def commit(self):
221
+ self._written = False
222
+
223
+ def rollback(self):
224
+ if self._written:
225
+ warnings.warn("rollback() cannot undo statements that already ran: every statement is committed "
226
+ "when execute() returns", RuntimeWarning, stacklevel=2)
227
+ self._written = False
228
+
229
+
230
+ class Cursor:
231
+ def __init__(self, connection):
232
+ self.connection = connection
233
+ self.description = None
234
+ self.rowcount = -1
235
+ self.arraysize = 1
236
+ self._rows, self._pos = [], 0
237
+
238
+ def execute(self, operation, args=None, stream=None, merge_socket_read=False):
239
+ r = self.connection.run(operation, args)
240
+ cols = r["columns"]
241
+ self.description = [(c, None, None, None, None, None, None) for c in cols] or None
242
+ self.rowcount = r["rowcount"]
243
+ self._rows, self._pos = r["rows"], 0
244
+ return self
245
+
246
+ def executemany(self, operation, param_sets):
247
+ total = 0
248
+ for p in param_sets:
249
+ self.execute(operation, p)
250
+ total += max(self.rowcount, 0)
251
+ self.rowcount = total
252
+ return self
253
+
254
+ def fetchone(self):
255
+ if self._pos >= len(self._rows):
256
+ return None
257
+ row = self._rows[self._pos]
258
+ self._pos += 1
259
+ return row
260
+
261
+ def fetchmany(self, num=None):
262
+ n = num or self.arraysize
263
+ out = self._rows[self._pos:self._pos + n]
264
+ self._pos += len(out)
265
+ return tuple(out)
266
+
267
+ def fetchall(self):
268
+ out = self._rows[self._pos:]
269
+ self._pos = len(self._rows)
270
+ return tuple(out)
271
+
272
+ def fetch_dataframe(self, num=None):
273
+ import pandas
274
+ rows = self.fetchmany(num) if num else self.fetchall()
275
+ return pandas.DataFrame(list(rows), columns=[d[0] for d in self.description or []])
276
+
277
+ def __iter__(self):
278
+ while True:
279
+ row = self.fetchone()
280
+ if row is None:
281
+ return
282
+ yield row
283
+
284
+ def close(self):
285
+ self._rows = []
286
+
287
+ def __enter__(self):
288
+ return self
289
+
290
+ def __exit__(self, *a):
291
+ self.close()
292
+
293
+
294
+ def connect(host, database="dev", user="", password="", port=None, timeout=900, **_ignored):
295
+ """Same signature as redshift_connector.connect; host is the Lambda URL. Extra arguments
296
+ (port, ssl, iam, region, ...) are accepted and ignored."""
297
+ return Connection(host, database, user, password, timeout)
@@ -0,0 +1,49 @@
1
+ Metadata-Version: 2.4
2
+ Name: s3shift
3
+ Version: 0.1.0
4
+ Summary: A truly serverless Redshift on your own AWS account: Redshift SQL on DuckDB in a Lambda, over your S3 and RDS data, ≈ $0 when idle. redshift_connector API: from s3shift import redshift_connector.
5
+ Author: Frank Zhang
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/cloudformer/s3shift
8
+ Project-URL: Repository, https://github.com/cloudformer/s3shift
9
+ Project-URL: Deploy, https://github.com/cloudformer/s3shift#quick-start
10
+ Project-URL: Compatibility, https://github.com/cloudformer/s3shift/blob/main/tests/compat/REPORT.md
11
+ Project-URL: Changelog, https://github.com/cloudformer/s3shift/blob/main/CHANGELOG.md
12
+ Project-URL: Issues, https://github.com/cloudformer/s3shift/issues
13
+ Keywords: serverless,aws,lambda,redshift,duckdb,s3,data-warehouse,sql,analytics
14
+ Classifier: Development Status :: 3 - Alpha
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: License :: OSI Approved :: MIT License
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Topic :: Database
19
+ Requires-Python: >=3.9
20
+ Description-Content-Type: text/markdown
21
+
22
+ # s3shift
23
+
24
+ **Python client for [s3shift](https://github.com/cloudformer/s3shift) — a truly serverless Redshift: Redshift
25
+ SQL on DuckDB in a Lambda, over your own S3 and RDS data, ≈ $0 when idle.** The redshift_connector API,
26
+ standard library only. Part of [Redefine AWS Serverless](https://github.com/topics/redefine-aws-serverless).
27
+
28
+ ```bash
29
+ pip install s3shift
30
+ ```
31
+
32
+ ```python
33
+ from s3shift import redshift_connector # later: `import redshift_connector` against Redshift, nothing else changes
34
+
35
+ conn = redshift_connector.connect(host="xxx.lambda-url.us-east-2.on.aws", database="dev",
36
+ user="admin", password="...")
37
+ cur = conn.cursor()
38
+ cur.execute("CREATE EXTERNAL TABLE clicks (user_id INT, day DATE) STORED AS PARQUET LOCATION 's3://my-lake/clicks/'")
39
+ cur.execute("SELECT day, count(*) FROM clicks WHERE day >= %s GROUP BY 1 ORDER BY 1", ["2026-01-01"])
40
+ cur.fetchall() # ([datetime.date(2026, 1, 1), 1520], ...)
41
+ cur.fetch_dataframe() # pandas, if installed
42
+ ```
43
+
44
+ Every statement is committed when `execute()` returns; large results are fetched page by page inside
45
+ `execute()`. Not supported (raises `NotSupportedError`): stored procedures, UDFs, users / GRANT, Glue
46
+ Data Catalog schemas, materialized views.
47
+
48
+ The endpoint comes from deploying the [main repository](https://github.com/cloudformer/s3shift).
49
+ Compatibility: [REPORT.md](https://github.com/cloudformer/s3shift/blob/main/tests/compat/REPORT.md).
@@ -0,0 +1,8 @@
1
+ README.md
2
+ pyproject.toml
3
+ s3shift/__init__.py
4
+ s3shift/redshift_connector.py
5
+ s3shift.egg-info/PKG-INFO
6
+ s3shift.egg-info/SOURCES.txt
7
+ s3shift.egg-info/dependency_links.txt
8
+ s3shift.egg-info/top_level.txt
@@ -0,0 +1 @@
1
+ s3shift
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+