periplus-python-sdk 0.7.0__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- periplus_python_sdk-0.8.0/PKG-INFO +70 -0
- periplus_python_sdk-0.8.0/README.md +52 -0
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/pyproject.toml +1 -1
- periplus_python_sdk-0.8.0/src/periplus_python_sdk.egg-info/PKG-INFO +70 -0
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/src/periplus_sdk/client.py +20 -18
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/src/periplus_sdk/dbapi.py +36 -43
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/src/periplus_sdk/sql_api.py +2 -2
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/src/periplus_sdk/sqlalchemy.py +31 -24
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/src/periplus_sdk/stream.py +1 -1
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/src/periplus_sdk/types.py +4 -2
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/tests/test_client.py +6 -23
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/tests/test_dbapi.py +7 -7
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/tests/test_notebook.py +44 -23
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/tests/test_sql_api.py +6 -5
- periplus_python_sdk-0.7.0/PKG-INFO +0 -301
- periplus_python_sdk-0.7.0/README.md +0 -283
- periplus_python_sdk-0.7.0/src/periplus_python_sdk.egg-info/PKG-INFO +0 -301
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/LICENSE +0 -0
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/NOTICE +0 -0
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/setup.cfg +0 -0
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/src/periplus_python_sdk.egg-info/SOURCES.txt +0 -0
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/src/periplus_python_sdk.egg-info/dependency_links.txt +0 -0
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/src/periplus_python_sdk.egg-info/entry_points.txt +0 -0
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/src/periplus_python_sdk.egg-info/requires.txt +0 -0
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/src/periplus_python_sdk.egg-info/top_level.txt +0 -0
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/src/periplus_sdk/__init__.py +0 -0
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/src/periplus_sdk/errors.py +0 -0
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/src/periplus_sdk/py.typed +0 -0
- {periplus_python_sdk-0.7.0 → periplus_python_sdk-0.8.0}/tests/test_stream.py +0 -0
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: periplus-python-sdk
|
|
3
|
+
Version: 0.8.0
|
|
4
|
+
Summary: Read-only Python client for the public Periplus query API
|
|
5
|
+
License-Expression: Apache-2.0
|
|
6
|
+
Project-URL: Repository, https://github.com/elei-io/periplus
|
|
7
|
+
Project-URL: Issues, https://github.com/elei-io/periplus/issues
|
|
8
|
+
Requires-Python: >=3.11
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
License-File: NOTICE
|
|
12
|
+
Requires-Dist: httpx>=0.28
|
|
13
|
+
Requires-Dist: pydantic<3,>=2.12
|
|
14
|
+
Requires-Dist: sqlalchemy<3,>=2.0
|
|
15
|
+
Provides-Extra: notebook
|
|
16
|
+
Requires-Dist: marimo[sql]>=0.24.1; extra == "notebook"
|
|
17
|
+
Dynamic: license-file
|
|
18
|
+
|
|
19
|
+
# Periplus Python SDK
|
|
20
|
+
|
|
21
|
+
Read-only access to the public ClickHouse corpus through the Periplus HTTP API.
|
|
22
|
+
Install the SDK from the same checkout as your deployment:
|
|
23
|
+
|
|
24
|
+
```sh
|
|
25
|
+
python -m pip install ./packages/periplus-python-sdk
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
```python
|
|
29
|
+
from periplus_sdk import Client
|
|
30
|
+
|
|
31
|
+
with Client("http://localhost:8080") as client:
|
|
32
|
+
result = client.execute(
|
|
33
|
+
"SELECT capture_id, url FROM public_v1.captures LIMIT ?", [10]
|
|
34
|
+
)
|
|
35
|
+
print(result.columns, result.rows)
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Use `PERIPLUS_PUBLIC_URL` to omit the URL argument. No database credentials or
|
|
39
|
+
service token are needed for the public gateway. `AsyncClient` provides async
|
|
40
|
+
methods. `prepare` validates/explains SQL; `execute` returns typed columns, rows,
|
|
41
|
+
truncation and query metadata; `helpers` describes the installed public views.
|
|
42
|
+
|
|
43
|
+
The public schema is `public_v1`. HTML joins use `document_id` plus node index;
|
|
44
|
+
`document_id` identifies retained bytes and their HTML interpretation. Raw-byte hashes stay internal.
|
|
45
|
+
There is one query endpoint, with no experimental fallback. Client errors preserve
|
|
46
|
+
server categories and do not automatically retry executed queries.
|
|
47
|
+
|
|
48
|
+
For notebook/SQLAlchemy integration:
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
from periplus_sdk import sql_api
|
|
52
|
+
from sqlalchemy import text
|
|
53
|
+
|
|
54
|
+
engine = sql_api.create_engine(base_url="http://localhost:8080")
|
|
55
|
+
with engine.connect() as connection:
|
|
56
|
+
print(connection.execute(text("SELECT url FROM public_v1.captures LIMIT 5")).all())
|
|
57
|
+
engine.dispose()
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Marimo discovers the five public views through the helper catalogue. Column
|
|
61
|
+
reflection uses `SELECT * ... LIMIT 0` to retrieve native types without reading
|
|
62
|
+
corpus rows; no `SHOW`, `DESCRIBE`, or system-table access is required.
|
|
63
|
+
|
|
64
|
+
The DB-API connection advertises the ClickHouse dialect and converts native
|
|
65
|
+
nullable integer, decimal, date and datetime types. Nested types retain JSON wire
|
|
66
|
+
values. It is read-only: there are no client transactions or writable sessions.
|
|
67
|
+
Streaming cursors expose incomplete/truncated results explicitly; configure
|
|
68
|
+
`allow_partial` only when partial results suit the application.
|
|
69
|
+
|
|
70
|
+
See [the public schema](../../docs/SCHEMA.md) and [query boundary](../../docs/QUERY.md).
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Periplus Python SDK
|
|
2
|
+
|
|
3
|
+
Read-only access to the public ClickHouse corpus through the Periplus HTTP API.
|
|
4
|
+
Install the SDK from the same checkout as your deployment:
|
|
5
|
+
|
|
6
|
+
```sh
|
|
7
|
+
python -m pip install ./packages/periplus-python-sdk
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
```python
|
|
11
|
+
from periplus_sdk import Client
|
|
12
|
+
|
|
13
|
+
with Client("http://localhost:8080") as client:
|
|
14
|
+
result = client.execute(
|
|
15
|
+
"SELECT capture_id, url FROM public_v1.captures LIMIT ?", [10]
|
|
16
|
+
)
|
|
17
|
+
print(result.columns, result.rows)
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
Use `PERIPLUS_PUBLIC_URL` to omit the URL argument. No database credentials or
|
|
21
|
+
service token are needed for the public gateway. `AsyncClient` provides async
|
|
22
|
+
methods. `prepare` validates/explains SQL; `execute` returns typed columns, rows,
|
|
23
|
+
truncation and query metadata; `helpers` describes the installed public views.
|
|
24
|
+
|
|
25
|
+
The public schema is `public_v1`. HTML joins use `document_id` plus node index;
|
|
26
|
+
`document_id` identifies retained bytes and their HTML interpretation. Raw-byte hashes stay internal.
|
|
27
|
+
There is one query endpoint, with no experimental fallback. Client errors preserve
|
|
28
|
+
server categories and do not automatically retry executed queries.
|
|
29
|
+
|
|
30
|
+
For notebook/SQLAlchemy integration:
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from periplus_sdk import sql_api
|
|
34
|
+
from sqlalchemy import text
|
|
35
|
+
|
|
36
|
+
engine = sql_api.create_engine(base_url="http://localhost:8080")
|
|
37
|
+
with engine.connect() as connection:
|
|
38
|
+
print(connection.execute(text("SELECT url FROM public_v1.captures LIMIT 5")).all())
|
|
39
|
+
engine.dispose()
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Marimo discovers the five public views through the helper catalogue. Column
|
|
43
|
+
reflection uses `SELECT * ... LIMIT 0` to retrieve native types without reading
|
|
44
|
+
corpus rows; no `SHOW`, `DESCRIBE`, or system-table access is required.
|
|
45
|
+
|
|
46
|
+
The DB-API connection advertises the ClickHouse dialect and converts native
|
|
47
|
+
nullable integer, decimal, date and datetime types. Nested types retain JSON wire
|
|
48
|
+
values. It is read-only: there are no client transactions or writable sessions.
|
|
49
|
+
Streaming cursors expose incomplete/truncated results explicitly; configure
|
|
50
|
+
`allow_partial` only when partial results suit the application.
|
|
51
|
+
|
|
52
|
+
See [the public schema](../../docs/SCHEMA.md) and [query boundary](../../docs/QUERY.md).
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: periplus-python-sdk
|
|
3
|
+
Version: 0.8.0
|
|
4
|
+
Summary: Read-only Python client for the public Periplus query API
|
|
5
|
+
License-Expression: Apache-2.0
|
|
6
|
+
Project-URL: Repository, https://github.com/elei-io/periplus
|
|
7
|
+
Project-URL: Issues, https://github.com/elei-io/periplus/issues
|
|
8
|
+
Requires-Python: >=3.11
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
License-File: NOTICE
|
|
12
|
+
Requires-Dist: httpx>=0.28
|
|
13
|
+
Requires-Dist: pydantic<3,>=2.12
|
|
14
|
+
Requires-Dist: sqlalchemy<3,>=2.0
|
|
15
|
+
Provides-Extra: notebook
|
|
16
|
+
Requires-Dist: marimo[sql]>=0.24.1; extra == "notebook"
|
|
17
|
+
Dynamic: license-file
|
|
18
|
+
|
|
19
|
+
# Periplus Python SDK
|
|
20
|
+
|
|
21
|
+
Read-only access to the public ClickHouse corpus through the Periplus HTTP API.
|
|
22
|
+
Install the SDK from the same checkout as your deployment:
|
|
23
|
+
|
|
24
|
+
```sh
|
|
25
|
+
python -m pip install ./packages/periplus-python-sdk
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
```python
|
|
29
|
+
from periplus_sdk import Client
|
|
30
|
+
|
|
31
|
+
with Client("http://localhost:8080") as client:
|
|
32
|
+
result = client.execute(
|
|
33
|
+
"SELECT capture_id, url FROM public_v1.captures LIMIT ?", [10]
|
|
34
|
+
)
|
|
35
|
+
print(result.columns, result.rows)
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Use `PERIPLUS_PUBLIC_URL` to omit the URL argument. No database credentials or
|
|
39
|
+
service token are needed for the public gateway. `AsyncClient` provides async
|
|
40
|
+
methods. `prepare` validates/explains SQL; `execute` returns typed columns, rows,
|
|
41
|
+
truncation and query metadata; `helpers` describes the installed public views.
|
|
42
|
+
|
|
43
|
+
The public schema is `public_v1`. HTML joins use `document_id` plus node index;
|
|
44
|
+
`document_id` identifies retained bytes and their HTML interpretation. Raw-byte hashes stay internal.
|
|
45
|
+
There is one query endpoint, with no experimental fallback. Client errors preserve
|
|
46
|
+
server categories and do not automatically retry executed queries.
|
|
47
|
+
|
|
48
|
+
For notebook/SQLAlchemy integration:
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
from periplus_sdk import sql_api
|
|
52
|
+
from sqlalchemy import text
|
|
53
|
+
|
|
54
|
+
engine = sql_api.create_engine(base_url="http://localhost:8080")
|
|
55
|
+
with engine.connect() as connection:
|
|
56
|
+
print(connection.execute(text("SELECT url FROM public_v1.captures LIMIT 5")).all())
|
|
57
|
+
engine.dispose()
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Marimo discovers the five public views through the helper catalogue. Column
|
|
61
|
+
reflection uses `SELECT * ... LIMIT 0` to retrieve native types without reading
|
|
62
|
+
corpus rows; no `SHOW`, `DESCRIBE`, or system-table access is required.
|
|
63
|
+
|
|
64
|
+
The DB-API connection advertises the ClickHouse dialect and converts native
|
|
65
|
+
nullable integer, decimal, date and datetime types. Nested types retain JSON wire
|
|
66
|
+
values. It is read-only: there are no client transactions or writable sessions.
|
|
67
|
+
Streaming cursors expose incomplete/truncated results explicitly; configure
|
|
68
|
+
`allow_partial` only when partial results suit the application.
|
|
69
|
+
|
|
70
|
+
See [the public schema](../../docs/SCHEMA.md) and [query boundary](../../docs/QUERY.md).
|
|
@@ -79,10 +79,11 @@ def _payload(sql: str, parameters: Sequence[JsonValue] | None, schema_version: s
|
|
|
79
79
|
class Client:
|
|
80
80
|
"""Reusable synchronous public query client. Close it or use a with block."""
|
|
81
81
|
|
|
82
|
-
def __init__(self, base_url: str | None = None, *, timeout: float = 620, mode: Literal["stable"
|
|
83
|
-
if mode not in {"stable"
|
|
84
|
-
raise ConfigurationError("
|
|
85
|
-
self.
|
|
82
|
+
def __init__(self, base_url: str | None = None, *, timeout: float = 620, mode: Literal["stable"] = "stable"):
|
|
83
|
+
if mode not in {"stable"}:
|
|
84
|
+
raise ConfigurationError("Only the stable public catalogue is available.")
|
|
85
|
+
self.schema_version = "public_v1"
|
|
86
|
+
self._query_path = "api/query/"
|
|
86
87
|
self._http = httpx.Client(**_options(base_url, timeout))
|
|
87
88
|
|
|
88
89
|
def __enter__(self) -> Client:
|
|
@@ -101,19 +102,19 @@ class Client:
|
|
|
101
102
|
raise TransportError("Could not complete the public query request.") from None
|
|
102
103
|
return _decode(response, model)
|
|
103
104
|
|
|
104
|
-
def prepare(self, sql: str, parameters: Sequence[JsonValue] | None = None, *, schema_version: str =
|
|
105
|
-
return self._request("POST", "prep", PreparedQuery, json=_payload(sql, parameters, schema_version))
|
|
105
|
+
def prepare(self, sql: str, parameters: Sequence[JsonValue] | None = None, *, schema_version: str | None = None) -> PreparedQuery:
|
|
106
|
+
return self._request("POST", "prep", PreparedQuery, json=_payload(sql, parameters, schema_version if schema_version is not None else self.schema_version))
|
|
106
107
|
|
|
107
|
-
def execute(self, sql: str, parameters: Sequence[JsonValue] | None = None, *, schema_version: str =
|
|
108
|
-
return self._request("POST", "exec", QueryResult, json=_payload(sql, parameters, schema_version))
|
|
108
|
+
def execute(self, sql: str, parameters: Sequence[JsonValue] | None = None, *, schema_version: str | None = None) -> QueryResult:
|
|
109
|
+
return self._request("POST", "exec", QueryResult, json=_payload(sql, parameters, schema_version if schema_version is not None else self.schema_version))
|
|
109
110
|
|
|
110
111
|
def stream(self, sql: str, parameters: Sequence[JsonValue] | None = None, *,
|
|
111
|
-
schema_version: str =
|
|
112
|
+
schema_version: str | None = None, allow_partial: bool = False):
|
|
112
113
|
"""Stream batches from one snapshot; use as a context manager for early exit."""
|
|
113
114
|
from .stream import MEDIA_TYPE, QueryStream
|
|
114
115
|
try:
|
|
115
116
|
request = self._http.build_request("POST", self._query_path + "exec",
|
|
116
|
-
headers={"accept": MEDIA_TYPE}, json=_payload(sql, parameters, schema_version))
|
|
117
|
+
headers={"accept": MEDIA_TYPE}, json=_payload(sql, parameters, schema_version if schema_version is not None else self.schema_version))
|
|
117
118
|
response = self._http.send(request, stream=True)
|
|
118
119
|
try:
|
|
119
120
|
if not response.is_success:
|
|
@@ -133,10 +134,11 @@ class Client:
|
|
|
133
134
|
class AsyncClient:
|
|
134
135
|
"""Reusable asynchronous public query client. Use an async with block."""
|
|
135
136
|
|
|
136
|
-
def __init__(self, base_url: str | None = None, *, timeout: float = 620, mode: Literal["stable"
|
|
137
|
-
if mode not in {"stable"
|
|
138
|
-
raise ConfigurationError("
|
|
139
|
-
self.
|
|
137
|
+
def __init__(self, base_url: str | None = None, *, timeout: float = 620, mode: Literal["stable"] = "stable"):
|
|
138
|
+
if mode not in {"stable"}:
|
|
139
|
+
raise ConfigurationError("Only the stable public catalogue is available.")
|
|
140
|
+
self.schema_version = "public_v1"
|
|
141
|
+
self._query_path = "api/query/"
|
|
140
142
|
self._http = httpx.AsyncClient(**_options(base_url, timeout))
|
|
141
143
|
|
|
142
144
|
async def __aenter__(self) -> AsyncClient:
|
|
@@ -155,11 +157,11 @@ class AsyncClient:
|
|
|
155
157
|
raise TransportError("Could not complete the public query request.") from None
|
|
156
158
|
return _decode(response, model)
|
|
157
159
|
|
|
158
|
-
async def prepare(self, sql: str, parameters: Sequence[JsonValue] | None = None, *, schema_version: str =
|
|
159
|
-
return await self._request("POST", "prep", PreparedQuery, json=_payload(sql, parameters, schema_version))
|
|
160
|
+
async def prepare(self, sql: str, parameters: Sequence[JsonValue] | None = None, *, schema_version: str | None = None) -> PreparedQuery:
|
|
161
|
+
return await self._request("POST", "prep", PreparedQuery, json=_payload(sql, parameters, schema_version if schema_version is not None else self.schema_version))
|
|
160
162
|
|
|
161
|
-
async def execute(self, sql: str, parameters: Sequence[JsonValue] | None = None, *, schema_version: str =
|
|
162
|
-
return await self._request("POST", "exec", QueryResult, json=_payload(sql, parameters, schema_version))
|
|
163
|
+
async def execute(self, sql: str, parameters: Sequence[JsonValue] | None = None, *, schema_version: str | None = None) -> QueryResult:
|
|
164
|
+
return await self._request("POST", "exec", QueryResult, json=_payload(sql, parameters, schema_version if schema_version is not None else self.schema_version))
|
|
163
165
|
|
|
164
166
|
async def helpers(self) -> QueryHelpers:
|
|
165
167
|
return await self._request("GET", "helpers", QueryHelpers)
|
|
@@ -86,54 +86,47 @@ def TimestampFromTicks(ticks: float) -> datetime:
|
|
|
86
86
|
return datetime.fromtimestamp(ticks)
|
|
87
87
|
|
|
88
88
|
|
|
89
|
-
_INTEGER_TYPES = {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
89
|
+
_INTEGER_TYPES = {f'{prefix}Int{bits}' for prefix in ('', 'U') for bits in (8,16,32,64,128,256)}
|
|
90
|
+
_FLOAT_TYPES = {'Float32', 'Float64'}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _base_type(value: str) -> str:
|
|
94
|
+
while value.startswith(('Nullable(', 'LowCardinality(')):
|
|
95
|
+
value = value[value.index('(')+1:-1]
|
|
96
|
+
return value
|
|
95
97
|
|
|
96
98
|
|
|
97
99
|
class _TypeCategory:
|
|
98
|
-
def __init__(self, names: set[str],
|
|
99
|
-
self.names, self.
|
|
100
|
+
def __init__(self, names: set[str], prefixes: tuple[str, ...] = ()):
|
|
101
|
+
self.names, self.prefixes = names, prefixes
|
|
100
102
|
|
|
101
103
|
def __eq__(self, other: object) -> bool:
|
|
102
|
-
|
|
104
|
+
if not isinstance(other, str):return False
|
|
105
|
+
value = _base_type(other)
|
|
106
|
+
return value in self.names or value.startswith(self.prefixes)
|
|
103
107
|
|
|
104
108
|
|
|
105
|
-
STRING = _TypeCategory({
|
|
106
|
-
BINARY = _TypeCategory(
|
|
107
|
-
NUMBER = _TypeCategory(_INTEGER_TYPES | _FLOAT_TYPES | {
|
|
108
|
-
DATETIME = _TypeCategory({
|
|
109
|
+
STRING = _TypeCategory({'String','UUID','JSON'}, ('FixedString(', 'Enum'))
|
|
110
|
+
BINARY = _TypeCategory(set())
|
|
111
|
+
NUMBER = _TypeCategory(_INTEGER_TYPES | _FLOAT_TYPES | {'Bool'}, ('Decimal',))
|
|
112
|
+
DATETIME = _TypeCategory({'Date','Date32'}, ('DateTime','Time'))
|
|
109
113
|
ROWID = _TypeCategory(set())
|
|
110
114
|
|
|
111
115
|
|
|
112
116
|
def _value(value: Any, sql_type: str) -> Any:
|
|
113
|
-
if value is None:
|
|
114
|
-
|
|
115
|
-
if sql_type in _INTEGER_TYPES:
|
|
116
|
-
|
|
117
|
-
if sql_type
|
|
118
|
-
|
|
119
|
-
if sql_type
|
|
120
|
-
return
|
|
121
|
-
|
|
122
|
-
if sql_type
|
|
123
|
-
try:
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
return value
|
|
127
|
-
if sql_type in _TIME_TYPES:
|
|
128
|
-
return time.fromisoformat(value)
|
|
129
|
-
if sql_type in _TIMESTAMP_TYPES:
|
|
130
|
-
try:
|
|
131
|
-
return datetime.fromisoformat(value)
|
|
132
|
-
except ValueError:
|
|
133
|
-
return value
|
|
134
|
-
if sql_type == "BLOB":
|
|
135
|
-
return base64.b64decode(value, validate=True)
|
|
136
|
-
# UUIDs remain strings, and nested/other types retain their JSON wire values.
|
|
117
|
+
if value is None:return None
|
|
118
|
+
sql_type = _base_type(sql_type)
|
|
119
|
+
if sql_type in _INTEGER_TYPES:return int(value)
|
|
120
|
+
if sql_type in _FLOAT_TYPES:return float(value)
|
|
121
|
+
if sql_type.startswith('Decimal'):return Decimal(str(value))
|
|
122
|
+
if sql_type == 'Bool':return value in (True, 1, '1', 'true')
|
|
123
|
+
if sql_type in ('Date','Date32'):
|
|
124
|
+
try:return date.fromisoformat(value)
|
|
125
|
+
except ValueError:return value
|
|
126
|
+
if sql_type.startswith('DateTime'):
|
|
127
|
+
try:return datetime.fromisoformat(value)
|
|
128
|
+
except ValueError:return value
|
|
129
|
+
if sql_type.startswith('Time'):return time.fromisoformat(value)
|
|
137
130
|
return value
|
|
138
131
|
|
|
139
132
|
|
|
@@ -152,16 +145,16 @@ def _parameter(value: Any) -> Any:
|
|
|
152
145
|
class Connection:
|
|
153
146
|
"""Marimo-discoverable, read-only connection; commit is a no-op."""
|
|
154
147
|
|
|
155
|
-
dialect = "
|
|
148
|
+
dialect = "clickhouse"
|
|
156
149
|
|
|
157
150
|
def __init__(self, base_url: str | None = None, *, timeout: float = 620,
|
|
158
|
-
mode: Literal["stable"
|
|
159
|
-
schema_version: str =
|
|
151
|
+
mode: Literal["stable"] = "stable",
|
|
152
|
+
schema_version: str | None = None, allow_partial: bool = False):
|
|
160
153
|
try:
|
|
161
154
|
self._client = Client(base_url, timeout=timeout, mode=mode)
|
|
162
155
|
except ConfigurationError as exc:
|
|
163
156
|
raise InterfaceError(str(exc)) from exc
|
|
164
|
-
self.schema_version = schema_version
|
|
157
|
+
self.schema_version = schema_version if schema_version is not None else ("public_v1")
|
|
165
158
|
self.allow_partial = allow_partial
|
|
166
159
|
self._cursors = set()
|
|
167
160
|
self.closed = False
|
|
@@ -210,8 +203,8 @@ class Connection:
|
|
|
210
203
|
|
|
211
204
|
|
|
212
205
|
def connect(base_url: str | None = None, *, timeout: float = 620,
|
|
213
|
-
mode: Literal["stable"
|
|
214
|
-
schema_version: str =
|
|
206
|
+
mode: Literal["stable"] = "stable",
|
|
207
|
+
schema_version: str | None = None, allow_partial: bool = False) -> Connection:
|
|
215
208
|
return Connection(base_url, timeout=timeout, mode=mode, schema_version=schema_version, allow_partial=allow_partial)
|
|
216
209
|
|
|
217
210
|
|
|
@@ -10,9 +10,9 @@ from sqlalchemy.engine import Engine, URL
|
|
|
10
10
|
def create_engine(
|
|
11
11
|
base_url: str | None = None,
|
|
12
12
|
*,
|
|
13
|
-
mode: Literal["stable"
|
|
13
|
+
mode: Literal["stable"] = "stable",
|
|
14
14
|
timeout: float = 620,
|
|
15
|
-
schema_version: str =
|
|
15
|
+
schema_version: str | None = None,
|
|
16
16
|
allow_partial: bool = False,
|
|
17
17
|
) -> Engine:
|
|
18
18
|
"""Create a SQLAlchemy engine recognized by marimo and other SQL tools.
|
|
@@ -9,10 +9,11 @@ from sqlalchemy.engine.reflection import cache
|
|
|
9
9
|
from sqlalchemy.sql.compiler import IdentifierPreparer
|
|
10
10
|
|
|
11
11
|
from . import dbapi
|
|
12
|
+
from .errors import ApiError, ResponseError, TransportError
|
|
12
13
|
|
|
13
14
|
|
|
14
15
|
class SQLType(types.UserDefinedType):
|
|
15
|
-
"""Preserve
|
|
16
|
+
"""Preserve ClickHouse type names, including nested types, during reflection."""
|
|
16
17
|
|
|
17
18
|
cache_ok = True
|
|
18
19
|
|
|
@@ -24,28 +25,21 @@ class SQLType(types.UserDefinedType):
|
|
|
24
25
|
|
|
25
26
|
@property
|
|
26
27
|
def python_type(self) -> type:
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
if
|
|
30
|
-
|
|
31
|
-
if
|
|
32
|
-
|
|
33
|
-
if
|
|
34
|
-
|
|
35
|
-
if self.name == "DATE":
|
|
36
|
-
return dbapi.date
|
|
37
|
-
if self.name in dbapi._TIMESTAMP_TYPES:
|
|
38
|
-
return dbapi.datetime
|
|
39
|
-
if self.name in dbapi._TIME_TYPES:
|
|
40
|
-
return dbapi.time
|
|
41
|
-
if self.name == "BLOB":
|
|
42
|
-
return bytes
|
|
28
|
+
name = dbapi._base_type(self.name)
|
|
29
|
+
if name in dbapi._INTEGER_TYPES:return int
|
|
30
|
+
if name in dbapi._FLOAT_TYPES:return float
|
|
31
|
+
if name == 'Bool':return bool
|
|
32
|
+
if name.startswith('Decimal'):return dbapi.Decimal
|
|
33
|
+
if name in ('Date','Date32'):return dbapi.date
|
|
34
|
+
if name.startswith('DateTime'):return dbapi.datetime
|
|
35
|
+
if name.startswith('Time'):return dbapi.time
|
|
43
36
|
return str
|
|
44
37
|
|
|
45
38
|
|
|
39
|
+
|
|
46
40
|
class PeriplusDialect(default.DefaultDialect):
|
|
47
|
-
# The server speaks
|
|
48
|
-
name = "
|
|
41
|
+
# The server speaks ClickHouse SQL; this enables the correct notebook SQL dialect.
|
|
42
|
+
name = "clickhouse"
|
|
49
43
|
driver = "periplus"
|
|
50
44
|
supports_statement_cache = False
|
|
51
45
|
supports_sane_rowcount = False
|
|
@@ -102,8 +96,15 @@ class PeriplusDialect(default.DefaultDialect):
|
|
|
102
96
|
@cache
|
|
103
97
|
def get_view_names(self, connection, schema=None, **kw):
|
|
104
98
|
schema = self._schema(connection, schema)
|
|
105
|
-
|
|
106
|
-
|
|
99
|
+
try:
|
|
100
|
+
catalogue = connection.connection.dbapi_connection._client.helpers()
|
|
101
|
+
except (ApiError, ResponseError, TransportError) as error:
|
|
102
|
+
raise exc.InvalidRequestError(f"Public catalogue discovery failed: {error}") from error
|
|
103
|
+
if catalogue.schema_version != schema:
|
|
104
|
+
raise exc.InvalidRequestError("Public catalogue does not match the configured schema.")
|
|
105
|
+
prefix = f"{schema}."
|
|
106
|
+
return [relation.name.removeprefix(prefix) for relation in catalogue.relations
|
|
107
|
+
if relation.kind == "view" and relation.name.startswith(prefix)]
|
|
107
108
|
|
|
108
109
|
@cache
|
|
109
110
|
def get_table_names(self, connection, schema=None, **kw):
|
|
@@ -119,9 +120,15 @@ class PeriplusDialect(default.DefaultDialect):
|
|
|
119
120
|
def get_columns(self, connection, table_name, schema=None, **kw):
|
|
120
121
|
schema = self._schema(connection, schema)
|
|
121
122
|
quote = self.identifier_preparer.quote_identifier
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
123
|
+
# A zero-row SELECT obtains native types through the supported public
|
|
124
|
+
# query contract, without scanning the view or requiring system access.
|
|
125
|
+
result = connection.exec_driver_sql(f"SELECT * FROM {quote(schema)}.{quote(table_name)} LIMIT 0")
|
|
126
|
+
metadata = result.cursor.result
|
|
127
|
+
self._complete(result)
|
|
128
|
+
return [{"name": name, "type": SQLType(kind),
|
|
129
|
+
"nullable": kind.startswith("Nullable(") or kind.startswith("LowCardinality(Nullable("),
|
|
130
|
+
"default": None}
|
|
131
|
+
for name, kind in zip(metadata.columns, metadata.types, strict=True)]
|
|
125
132
|
|
|
126
133
|
@cache
|
|
127
134
|
def get_pk_constraint(self, connection, table_name, schema=None, **kw):
|
|
@@ -16,7 +16,7 @@ MEDIA_TYPE = "application/x-ndjson"
|
|
|
16
16
|
class StreamResult(PreparedQuery):
|
|
17
17
|
columns: list[str]
|
|
18
18
|
types: list[str]
|
|
19
|
-
source_snapshot: int = Field(ge=0)
|
|
19
|
+
source_snapshot: int | None = Field(default=None, ge=0)
|
|
20
20
|
limits: dict[str, int]
|
|
21
21
|
complete: bool = False
|
|
22
22
|
truncated: bool = False
|
|
@@ -11,7 +11,7 @@ class Diagnostic(BaseModel):
|
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
class PreparedQuery(BaseModel):
|
|
14
|
-
query_mode: Literal["stable"
|
|
14
|
+
query_mode: Literal["stable"]
|
|
15
15
|
compiler_version: str
|
|
16
16
|
optimizations: list[str]
|
|
17
17
|
schema_version: str
|
|
@@ -28,7 +28,7 @@ class QueryResult(PreparedQuery):
|
|
|
28
28
|
rows: list[list[JsonValue]]
|
|
29
29
|
truncated: bool
|
|
30
30
|
elapsed_ms: float
|
|
31
|
-
source_snapshot: int = Field(ge=0)
|
|
31
|
+
source_snapshot: int | None = Field(default=None, ge=0)
|
|
32
32
|
row_count: int = Field(ge=0)
|
|
33
33
|
result_bytes: int = Field(ge=0)
|
|
34
34
|
truncation_reason: Literal["max_rows", "max_result_bytes"] | None = None
|
|
@@ -51,4 +51,6 @@ class QueryHelper(BaseModel):
|
|
|
51
51
|
|
|
52
52
|
class QueryHelpers(BaseModel):
|
|
53
53
|
catalogue_version: str
|
|
54
|
+
schema_version: str
|
|
55
|
+
relations: list[QueryHelper]
|
|
54
56
|
helpers: list[QueryHelper]
|
|
@@ -8,7 +8,7 @@ import httpx
|
|
|
8
8
|
from periplus_sdk import AsyncClient, Client, ApiError, ConfigurationError, ResponseError, TransportError
|
|
9
9
|
|
|
10
10
|
PREP = dict(query_mode='stable', compiler_version='public-query-v8:stable', optimizations=[], schema_version='public_v1', query_id='q', sql='SELECT ? AS n', parameters=[1], diagnostics=[], plan='plan')
|
|
11
|
-
RESULT = dict(**PREP, columns=['n', 'n'], types=['
|
|
11
|
+
RESULT = dict(**PREP, columns=['n', 'n'], types=['Int64', 'Decimal(20,2)'],
|
|
12
12
|
rows=[['9007199254740993', '123.45']], truncated=True, elapsed_ms=1.2, source_snapshot=4, row_count=1, result_bytes=32)
|
|
13
13
|
|
|
14
14
|
|
|
@@ -40,7 +40,7 @@ class ClientTests(unittest.TestCase):
|
|
|
40
40
|
self.assertNotIn('authorization', request.headers)
|
|
41
41
|
self.assertEqual(request.headers['x-periplus-query-source'], 'sdk')
|
|
42
42
|
if request.url.path.endswith('helpers'):
|
|
43
|
-
return httpx.Response(200, json={'catalogue_version': '1.0.0', 'helpers': []})
|
|
43
|
+
return httpx.Response(200, json={'catalogue_version': '1.0.0', 'helpers': [], 'relations': [], 'schema_version': 'public_v1'})
|
|
44
44
|
self.assertEqual(json.loads(request.content), {'sql': PREP['sql'], 'parameters': [1], 'schema_version': 'public_v1'})
|
|
45
45
|
return httpx.Response(200, json=RESULT if request.url.path.endswith('exec') else PREP)
|
|
46
46
|
with patch.dict(os.environ, {'PERIPLUS_QUERY_API_TOKEN': 'secret', 'PERIPLUS_API_TOKEN': 'admin'}):
|
|
@@ -54,24 +54,6 @@ class ClientTests(unittest.TestCase):
|
|
|
54
54
|
self.assertEqual(client.helpers().catalogue_version, '1.0.0')
|
|
55
55
|
self.assertEqual([r.url.path for r in requests], ['/prefix/api/query/prep', '/prefix/api/query/exec', '/prefix/api/query/helpers'])
|
|
56
56
|
|
|
57
|
-
def test_experimental_routes_and_metadata(self):
|
|
58
|
-
paths = []
|
|
59
|
-
def handler(request):
|
|
60
|
-
paths.append(request.url.path)
|
|
61
|
-
if request.url.path.endswith('helpers'):
|
|
62
|
-
return httpx.Response(200, json={'catalogue_version': '1.0.0', 'helpers': []})
|
|
63
|
-
payload = dict(RESULT if request.url.path.endswith('exec') else PREP,
|
|
64
|
-
query_mode='experimental', compiler_version='public-query-v8:experimental',
|
|
65
|
-
optimizations=['content_scope'])
|
|
66
|
-
return httpx.Response(200, json=payload)
|
|
67
|
-
with self.client(handler, mode='experimental') as client:
|
|
68
|
-
prepared = client.prepare('SELECT 1')
|
|
69
|
-
self.assertEqual(prepared.query_mode, 'experimental')
|
|
70
|
-
self.assertEqual(prepared.compiler_version, 'public-query-v8:experimental')
|
|
71
|
-
self.assertEqual(client.execute('SELECT 1').optimizations, ['content_scope'])
|
|
72
|
-
client.helpers()
|
|
73
|
-
self.assertEqual(paths, ['/prefix/api/query/experimental/' + p for p in ['prep', 'exec', 'helpers']])
|
|
74
|
-
|
|
75
57
|
def test_invalid_mode(self):
|
|
76
58
|
for factory in (Client, AsyncClient):
|
|
77
59
|
with self.assertRaises(ConfigurationError):
|
|
@@ -132,15 +114,16 @@ class AsyncClientTests(unittest.IsolatedAsyncioTestCase):
|
|
|
132
114
|
def handler(request):
|
|
133
115
|
calls.append(request)
|
|
134
116
|
if request.url.path.endswith('helpers'):
|
|
135
|
-
return httpx.Response(200, json={'catalogue_version': '1.0.0', 'helpers': []})
|
|
117
|
+
return httpx.Response(200, json={'catalogue_version': '1.0.0', 'helpers': [], 'relations': [], 'schema_version': 'public_v1'})
|
|
136
118
|
return httpx.Response(200, json=RESULT if request.url.path.endswith('exec') else PREP)
|
|
137
119
|
with patch('periplus_sdk.client.httpx.AsyncClient', side_effect=lambda **kw:
|
|
138
120
|
factory(**kw, transport=httpx.MockTransport(handler))):
|
|
139
|
-
async with AsyncClient('https://public.example/prefix/', mode='
|
|
121
|
+
async with AsyncClient('https://public.example/prefix/', mode='stable') as client:
|
|
140
122
|
self.assertEqual((await client.prepare('SELECT ?', [1])).parameters, [1])
|
|
141
123
|
self.assertEqual((await client.execute('SELECT ?', [1])).rows, RESULT['rows'])
|
|
142
124
|
self.assertEqual((await client.helpers()).catalogue_version, '1.0.0')
|
|
143
125
|
self.assertTrue(client._http.is_closed)
|
|
144
126
|
self.assertEqual(len(calls), 3)
|
|
145
|
-
self.assertEqual([r.url.path for r in calls], ['/prefix/api/query/
|
|
127
|
+
self.assertEqual([r.url.path for r in calls], ['/prefix/api/query/' + p for p in ['prep', 'exec', 'helpers']])
|
|
146
128
|
self.assertTrue(all(r.headers['x-periplus-query-source'] == 'sdk' for r in calls))
|
|
129
|
+
self.assertTrue(all(json.loads(r.content)['schema_version'] == 'public_v1' for r in calls if r.method == 'POST'))
|
|
@@ -34,7 +34,7 @@ class DBAPITests(unittest.TestCase):
|
|
|
34
34
|
cur.execute('SELECT ? AS n', [1])
|
|
35
35
|
self.assertEqual(cur.rowcount, -1)
|
|
36
36
|
self.assertEqual([d[0] for d in cur.description], ['n', 'n'])
|
|
37
|
-
self.assertEqual(cur.description[1][1], '
|
|
37
|
+
self.assertEqual(cur.description[1][1], 'Decimal(20,2)')
|
|
38
38
|
self.assertEqual(cur.fetchmany(0), [])
|
|
39
39
|
self.assertEqual(cur.fetchone(), (9007199254740993, Decimal('123.45')))
|
|
40
40
|
self.assertIsNone(cur.fetchone())
|
|
@@ -48,7 +48,7 @@ class DBAPITests(unittest.TestCase):
|
|
|
48
48
|
self.assertEqual(cur.fetchall(), [])
|
|
49
49
|
|
|
50
50
|
def test_empty_and_fetchmany(self):
|
|
51
|
-
c = self.connection(lambda r: httpx.Response(200, json=response(columns=['n'], types=['
|
|
51
|
+
c = self.connection(lambda r: httpx.Response(200, json=response(columns=['n'], types=['Int32'], rows=[[1],[2],[3]])))
|
|
52
52
|
cur = c.execute('SELECT 1')
|
|
53
53
|
cur.arraysize = 2
|
|
54
54
|
self.assertEqual(cur.fetchmany(), [(1,), (2,)])
|
|
@@ -66,12 +66,12 @@ class DBAPITests(unittest.TestCase):
|
|
|
66
66
|
def handler(r):
|
|
67
67
|
requests.append(r)
|
|
68
68
|
return httpx.Response(200, json=response(columns=['d','t','ts','b','f','n','s'],
|
|
69
|
-
types=['
|
|
69
|
+
types=['Date','Time','DateTime64(6)','String','Float64','Int64','String'],
|
|
70
70
|
rows=[['2026-09-11','12:34:56','2026-09-11T12:34:56+00:00','aGk=','inf',None,'9007199254740993']]))
|
|
71
|
-
c = self.connection(handler, mode='
|
|
72
|
-
row = c.execute('SELECT CAST(? AS
|
|
73
|
-
self.assertEqual(row, (date(2026,9,11),time(12,34,56),datetime.fromisoformat('2026-09-11T12:34:56+00:00'),
|
|
74
|
-
self.assertEqual(requests[0].url.path, '/prefix/api/query/
|
|
71
|
+
c = self.connection(handler, mode='stable')
|
|
72
|
+
row = c.execute('SELECT CAST(? AS Date)', [date(2026,9,11)]).fetchone()
|
|
73
|
+
self.assertEqual(row, (date(2026,9,11),time(12,34,56),datetime.fromisoformat('2026-09-11T12:34:56+00:00'),'aGk=',float('inf'),None,'9007199254740993'))
|
|
74
|
+
self.assertEqual(requests[0].url.path, '/prefix/api/query/exec')
|
|
75
75
|
self.assertEqual(requests[0].headers['x-periplus-query-source'], 'sdk')
|
|
76
76
|
|
|
77
77
|
def test_truncation_and_lifecycle(self):
|