jusi-clickhouse 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jusi_clickhouse-0.2.0/.gitignore +24 -0
- jusi_clickhouse-0.2.0/Dockerfile +6 -0
- jusi_clickhouse-0.2.0/LICENSE +21 -0
- jusi_clickhouse-0.2.0/PKG-INFO +79 -0
- jusi_clickhouse-0.2.0/README.md +43 -0
- jusi_clickhouse-0.2.0/docker/clickhouse/initdb.d/001_jusi_demo.sql +74 -0
- jusi_clickhouse-0.2.0/examples/jusi.toml +8 -0
- jusi_clickhouse-0.2.0/pyproject.toml +33 -0
- jusi_clickhouse-0.2.0/scripts/run-clickhouse.sh +78 -0
- jusi_clickhouse-0.2.0/src/jusi_clickhouse/__init__.py +5 -0
- jusi_clickhouse-0.2.0/src/jusi_clickhouse/catalog.py +19 -0
- jusi_clickhouse-0.2.0/src/jusi_clickhouse/config.py +25 -0
- jusi_clickhouse-0.2.0/src/jusi_clickhouse/constants.py +3 -0
- jusi_clickhouse-0.2.0/src/jusi_clickhouse/ipc.py +129 -0
- jusi_clickhouse-0.2.0/src/jusi_clickhouse/kernel.py +19 -0
- jusi_clickhouse-0.2.0/src/jusi_clickhouse/metadata.py +55 -0
- jusi_clickhouse-0.2.0/src/jusi_clickhouse/runner.py +590 -0
- jusi_clickhouse-0.2.0/src/jusi_clickhouse/worker.py +108 -0
- jusi_clickhouse-0.2.0/tests/test_plugin.py +248 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Python bytecode and caches
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
.pytest_cache/
|
|
6
|
+
.mypy_cache/
|
|
7
|
+
.ruff_cache/
|
|
8
|
+
.coverage
|
|
9
|
+
htmlcov/
|
|
10
|
+
|
|
11
|
+
# Build artifacts
|
|
12
|
+
build/
|
|
13
|
+
dist/
|
|
14
|
+
*.egg-info/
|
|
15
|
+
|
|
16
|
+
# Virtual environments
|
|
17
|
+
.venv/
|
|
18
|
+
venv/
|
|
19
|
+
venv*/
|
|
20
|
+
|
|
21
|
+
# Local runtime state
|
|
22
|
+
.env
|
|
23
|
+
.env.*
|
|
24
|
+
.DS_Store
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
FROM clickhouse/clickhouse-server:25.4-alpine
|
|
2
|
+
|
|
3
|
+
LABEL org.opencontainers.image.title="jusi-clickhouse-dev"
|
|
4
|
+
LABEL org.opencontainers.image.description="Local ClickHouse fixture for jusi-clickhouse development"
|
|
5
|
+
|
|
6
|
+
COPY docker/clickhouse/initdb.d/ /docker-entrypoint-initdb.d/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 notawhaleble
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: jusi-clickhouse
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: ClickHouse exact-provider plugin for Jusi 1.0 SQL
|
|
5
|
+
Author: Jusi contributors
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2026 notawhaleble
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
Requires-Python: >=3.9
|
|
29
|
+
Requires-Dist: clickhouse-connect>=0.7
|
|
30
|
+
Requires-Dist: jusi-sql<0.3,>=0.2
|
|
31
|
+
Requires-Dist: jusi<2,>=1.0
|
|
32
|
+
Requires-Dist: visidata<4,>=3
|
|
33
|
+
Provides-Extra: test
|
|
34
|
+
Requires-Dist: pytest<9,>=8; extra == 'test'
|
|
35
|
+
Description-Content-Type: text/markdown
|
|
36
|
+
|
|
37
|
+
# jusi-clickhouse
|
|
38
|
+
|
|
39
|
+
`jusi-clickhouse` is the ClickHouse exact-provider plugin for Jusi 1.0 and the
|
|
40
|
+
shared `jusi-sql` family. It exposes the `clickhouse` provider for `%%sql`
|
|
41
|
+
cells. The family owns target selection, magic dispatch, completion ranges,
|
|
42
|
+
metadata caching, operation routing, and common VisiData SQL commands.
|
|
43
|
+
|
|
44
|
+
Configure a SQL target in the Jusi session config:
|
|
45
|
+
|
|
46
|
+
```toml
|
|
47
|
+
[sql.targets.analytics]
|
|
48
|
+
provider = "clickhouse"
|
|
49
|
+
host = "localhost"
|
|
50
|
+
port = 8123
|
|
51
|
+
username = "default"
|
|
52
|
+
password = ""
|
|
53
|
+
database = "default"
|
|
54
|
+
initial_fetch = 100
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Use it from a cell:
|
|
58
|
+
|
|
59
|
+
```sql
|
|
60
|
+
%%sql analytics
|
|
61
|
+
SELECT *
|
|
62
|
+
FROM system.numbers
|
|
63
|
+
LIMIT 1000
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Results open in VisiData. Press `1`-`9` to fetch more rows, `gf` to prompt for a fetch count (`0` fetches the rest), and `gb` to open the selected raw value through VisiData. `gc` and `gr` are present for SQL-family parity, but ClickHouse transaction control is reported as unsupported.
|
|
67
|
+
|
|
68
|
+
Follow-up cells reuse the same VisiData application, ClickHouse connection, and
|
|
69
|
+
open result streams. `JusiInterrupt` requests cancellation of active queries.
|
|
70
|
+
Closing the client closes its streams, connection, private control socket, and
|
|
71
|
+
staged launch data.
|
|
72
|
+
|
|
73
|
+
For a local manual-test database:
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
./scripts/run-clickhouse.sh
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
The script builds `jusi-clickhouse-dev`, starts a local ClickHouse container, loads demo tables under `demo`, waits until it is ready, and prints a matching `jusi.toml` block.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# jusi-clickhouse
|
|
2
|
+
|
|
3
|
+
`jusi-clickhouse` is the ClickHouse exact-provider plugin for Jusi 1.0 and the
|
|
4
|
+
shared `jusi-sql` family. It exposes the `clickhouse` provider for `%%sql`
|
|
5
|
+
cells. The family owns target selection, magic dispatch, completion ranges,
|
|
6
|
+
metadata caching, operation routing, and common VisiData SQL commands.
|
|
7
|
+
|
|
8
|
+
Configure a SQL target in the Jusi session config:
|
|
9
|
+
|
|
10
|
+
```toml
|
|
11
|
+
[sql.targets.analytics]
|
|
12
|
+
provider = "clickhouse"
|
|
13
|
+
host = "localhost"
|
|
14
|
+
port = 8123
|
|
15
|
+
username = "default"
|
|
16
|
+
password = ""
|
|
17
|
+
database = "default"
|
|
18
|
+
initial_fetch = 100
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Use it from a cell:
|
|
22
|
+
|
|
23
|
+
```sql
|
|
24
|
+
%%sql analytics
|
|
25
|
+
SELECT *
|
|
26
|
+
FROM system.numbers
|
|
27
|
+
LIMIT 1000
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Results open in VisiData. Press `1`-`9` to fetch more rows, `gf` to prompt for a fetch count (`0` fetches the rest), and `gb` to open the selected raw value through VisiData. `gc` and `gr` are present for SQL-family parity, but ClickHouse transaction control is reported as unsupported.
|
|
31
|
+
|
|
32
|
+
Follow-up cells reuse the same VisiData application, ClickHouse connection, and
|
|
33
|
+
open result streams. `JusiInterrupt` requests cancellation of active queries.
|
|
34
|
+
Closing the client closes its streams, connection, private control socket, and
|
|
35
|
+
staged launch data.
|
|
36
|
+
|
|
37
|
+
For a local manual-test database:
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
./scripts/run-clickhouse.sh
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
The script builds `jusi-clickhouse-dev`, starts a local ClickHouse container, loads demo tables under `demo`, waits until it is ready, and prints a matching `jusi.toml` block.
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
CREATE DATABASE IF NOT EXISTS demo;
|
|
2
|
+
|
|
3
|
+
CREATE TABLE IF NOT EXISTS demo.accounts
|
|
4
|
+
(
|
|
5
|
+
id UInt64,
|
|
6
|
+
email String,
|
|
7
|
+
display_name String,
|
|
8
|
+
active Bool,
|
|
9
|
+
created_at DateTime
|
|
10
|
+
)
|
|
11
|
+
ENGINE = MergeTree
|
|
12
|
+
ORDER BY id;
|
|
13
|
+
|
|
14
|
+
CREATE TABLE IF NOT EXISTS demo.events
|
|
15
|
+
(
|
|
16
|
+
id UInt64,
|
|
17
|
+
account_id UInt64,
|
|
18
|
+
event_type LowCardinality(String),
|
|
19
|
+
payload String,
|
|
20
|
+
created_at DateTime
|
|
21
|
+
)
|
|
22
|
+
ENGINE = MergeTree
|
|
23
|
+
ORDER BY (account_id, id);
|
|
24
|
+
|
|
25
|
+
CREATE TABLE IF NOT EXISTS demo.blob_files
|
|
26
|
+
(
|
|
27
|
+
id UInt64,
|
|
28
|
+
filename String,
|
|
29
|
+
content_type String,
|
|
30
|
+
payload String,
|
|
31
|
+
created_at DateTime
|
|
32
|
+
)
|
|
33
|
+
ENGINE = MergeTree
|
|
34
|
+
ORDER BY id;
|
|
35
|
+
|
|
36
|
+
TRUNCATE TABLE demo.accounts;
|
|
37
|
+
TRUNCATE TABLE demo.events;
|
|
38
|
+
TRUNCATE TABLE demo.blob_files;
|
|
39
|
+
|
|
40
|
+
INSERT INTO demo.accounts (id, email, display_name, active, created_at) VALUES
|
|
41
|
+
(1, 'ada@example.test', 'Ada Lovelace', true, now() - INTERVAL 3 DAY),
|
|
42
|
+
(2, 'grace@example.test', 'Grace Hopper', true, now() - INTERVAL 2 DAY),
|
|
43
|
+
(3, 'alan@example.test', 'Alan Turing', false, now() - INTERVAL 1 DAY);
|
|
44
|
+
|
|
45
|
+
INSERT INTO demo.events (id, account_id, event_type, payload, created_at) VALUES
|
|
46
|
+
(1, 1, 'login', '{"ip":"127.0.0.1"}', now() - INTERVAL 3 HOUR),
|
|
47
|
+
(2, 1, 'query', '{"rows":42}', now() - INTERVAL 2 HOUR),
|
|
48
|
+
(3, 2, 'login', '{"ip":"127.0.0.2"}', now() - INTERVAL 1 HOUR),
|
|
49
|
+
(4, 3, 'disabled_login', '{"reason":"inactive"}', now());
|
|
50
|
+
|
|
51
|
+
INSERT INTO demo.blob_files (id, filename, content_type, payload, created_at) VALUES
|
|
52
|
+
(
|
|
53
|
+
1,
|
|
54
|
+
'test-blob.zip',
|
|
55
|
+
'application/zip',
|
|
56
|
+
unhex('504b03040a0000000000231ab65c2909ab311b0000001b0000000f001c00626c6f622d726561646d652e747874555409000382a00f6a82a00f6a75780b000104f501000004140000004a75736920506f73746772657320626c6f6220666978747572650a504b01021e030a0000000000231ab65c2909ab311b0000001b0000000f0018000000000001000000a48100000000626c6f622d726561646d652e747874555405000382a00f6a75780b000104f50100000414000000504b0506000000000100010055000000640000000000'),
|
|
57
|
+
now()
|
|
58
|
+
);
|
|
59
|
+
|
|
60
|
+
CREATE OR REPLACE VIEW demo.account_summary AS
|
|
61
|
+
SELECT
|
|
62
|
+
a.id,
|
|
63
|
+
a.email,
|
|
64
|
+
a.display_name,
|
|
65
|
+
a.active,
|
|
66
|
+
count(e.id) AS event_count,
|
|
67
|
+
max(e.created_at) AS last_event_at
|
|
68
|
+
FROM demo.accounts AS a
|
|
69
|
+
LEFT JOIN demo.events AS e ON e.account_id = a.id
|
|
70
|
+
GROUP BY
|
|
71
|
+
a.id,
|
|
72
|
+
a.email,
|
|
73
|
+
a.display_name,
|
|
74
|
+
a.active;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "jusi-clickhouse"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "ClickHouse exact-provider plugin for Jusi 1.0 SQL"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { file = "LICENSE" }
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "Jusi contributors" }
|
|
14
|
+
]
|
|
15
|
+
dependencies = [
|
|
16
|
+
"clickhouse-connect>=0.7",
|
|
17
|
+
"jusi>=1.0,<2",
|
|
18
|
+
"jusi-sql>=0.2,<0.3",
|
|
19
|
+
"visidata>=3,<4",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[project.optional-dependencies]
|
|
23
|
+
test = ["pytest>=8,<9"]
|
|
24
|
+
|
|
25
|
+
[project.entry-points."jusi.plugins.v1"]
|
|
26
|
+
clickhouse = "jusi_clickhouse.catalog:catalog_entry"
|
|
27
|
+
|
|
28
|
+
[tool.hatch.build.targets.wheel]
|
|
29
|
+
packages = ["src/jusi_clickhouse"]
|
|
30
|
+
|
|
31
|
+
[tool.pytest.ini_options]
|
|
32
|
+
pythonpath = ["src"]
|
|
33
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
IMAGE_NAME="${JUSI_CLICKHOUSE_IMAGE:-jusi-clickhouse-dev}"
|
|
5
|
+
CONTAINER_NAME="${JUSI_CLICKHOUSE_CONTAINER:-jusi-clickhouse-dev}"
|
|
6
|
+
HTTP_PORT="${JUSI_CLICKHOUSE_HTTP_PORT:-58123}"
|
|
7
|
+
NATIVE_PORT="${JUSI_CLICKHOUSE_NATIVE_PORT:-59000}"
|
|
8
|
+
CLICKHOUSE_USER="${JUSI_CLICKHOUSE_USER:-jusi}"
|
|
9
|
+
CLICKHOUSE_PASSWORD="${JUSI_CLICKHOUSE_PASSWORD:-jusi}"
|
|
10
|
+
CLICKHOUSE_DB="${JUSI_CLICKHOUSE_DB:-demo}"
|
|
11
|
+
RECREATE="${JUSI_CLICKHOUSE_RECREATE:-0}"
|
|
12
|
+
|
|
13
|
+
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
14
|
+
|
|
15
|
+
docker build -t "${IMAGE_NAME}" "${ROOT_DIR}"
|
|
16
|
+
|
|
17
|
+
if [[ "${RECREATE}" == "1" ]] && docker ps -a --format '{{.Names}}' | grep -Fxq "${CONTAINER_NAME}"; then
|
|
18
|
+
docker rm -f "${CONTAINER_NAME}" >/dev/null
|
|
19
|
+
fi
|
|
20
|
+
|
|
21
|
+
if docker ps --format '{{.Names}}' | grep -Fxq "${CONTAINER_NAME}"; then
|
|
22
|
+
echo "ClickHouse container '${CONTAINER_NAME}' is already running on HTTP port ${HTTP_PORT}."
|
|
23
|
+
else
|
|
24
|
+
if docker ps -a --format '{{.Names}}' | grep -Fxq "${CONTAINER_NAME}"; then
|
|
25
|
+
docker rm "${CONTAINER_NAME}" >/dev/null
|
|
26
|
+
fi
|
|
27
|
+
|
|
28
|
+
docker run \
|
|
29
|
+
--detach \
|
|
30
|
+
--name "${CONTAINER_NAME}" \
|
|
31
|
+
--publish "127.0.0.1:${HTTP_PORT}:8123" \
|
|
32
|
+
--publish "127.0.0.1:${NATIVE_PORT}:9000" \
|
|
33
|
+
--env "CLICKHOUSE_USER=${CLICKHOUSE_USER}" \
|
|
34
|
+
--env "CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD}" \
|
|
35
|
+
--env "CLICKHOUSE_DB=${CLICKHOUSE_DB}" \
|
|
36
|
+
"${IMAGE_NAME}" >/dev/null
|
|
37
|
+
fi
|
|
38
|
+
|
|
39
|
+
echo "Waiting for ClickHouse to accept connections..."
|
|
40
|
+
for _ in $(seq 1 90); do
|
|
41
|
+
if docker exec "${CONTAINER_NAME}" clickhouse-client \
|
|
42
|
+
--user "${CLICKHOUSE_USER}" \
|
|
43
|
+
--password "${CLICKHOUSE_PASSWORD}" \
|
|
44
|
+
--database "${CLICKHOUSE_DB}" \
|
|
45
|
+
--query "SELECT 1" >/dev/null 2>&1; then
|
|
46
|
+
cat <<EOF
|
|
47
|
+
ClickHouse is running.
|
|
48
|
+
|
|
49
|
+
jusi.toml:
|
|
50
|
+
|
|
51
|
+
[sql.targets.local_clickhouse]
|
|
52
|
+
provider = "clickhouse"
|
|
53
|
+
host = "127.0.0.1"
|
|
54
|
+
port = ${HTTP_PORT}
|
|
55
|
+
username = "${CLICKHOUSE_USER}"
|
|
56
|
+
password = "${CLICKHOUSE_PASSWORD}"
|
|
57
|
+
database = "${CLICKHOUSE_DB}"
|
|
58
|
+
initial_fetch = 25
|
|
59
|
+
|
|
60
|
+
Manual test queries:
|
|
61
|
+
|
|
62
|
+
%%sql local_clickhouse
|
|
63
|
+
SELECT * FROM demo.account_summary ORDER BY id
|
|
64
|
+
|
|
65
|
+
%%sql local_clickhouse
|
|
66
|
+
SELECT number FROM system.numbers LIMIT 50
|
|
67
|
+
|
|
68
|
+
%%sql local_clickhouse
|
|
69
|
+
SELECT filename, content_type, payload FROM demo.blob_files
|
|
70
|
+
EOF
|
|
71
|
+
exit 0
|
|
72
|
+
fi
|
|
73
|
+
sleep 1
|
|
74
|
+
done
|
|
75
|
+
|
|
76
|
+
echo "ClickHouse did not become ready in time." >&2
|
|
77
|
+
docker logs "${CONTAINER_NAME}" >&2 || true
|
|
78
|
+
exit 1
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Lightweight Jusi 1.0 catalog provider; imports no runtime dependencies."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from jusi_sql import sql_catalog_entry
|
|
7
|
+
|
|
8
|
+
from . import __version__
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def catalog_entry() -> dict[str, Any]:
|
|
12
|
+
return sql_catalog_entry(
|
|
13
|
+
plugin_id="clickhouse",
|
|
14
|
+
plugin_version=__version__,
|
|
15
|
+
distribution="jusi-clickhouse",
|
|
16
|
+
kernel_extension="jusi_clickhouse.kernel",
|
|
17
|
+
worker_entry_point="jusi_clickhouse.worker:create_worker",
|
|
18
|
+
provider_presentation={"syntax": "clickhouse", "indent": "sql"},
|
|
19
|
+
)
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Any, Mapping
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
PLUGIN_OPTIONS = {"provider", "initial_fetch"}
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class ClickHouseOptions:
|
|
12
|
+
connect: dict[str, Any]
|
|
13
|
+
initial_fetch: int
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def parse_clickhouse_options(options: Mapping[str, Any]) -> ClickHouseOptions:
|
|
17
|
+
raw_initial = options.get("initial_fetch", 100)
|
|
18
|
+
try:
|
|
19
|
+
initial_fetch = int(raw_initial)
|
|
20
|
+
except (TypeError, ValueError):
|
|
21
|
+
initial_fetch = 100
|
|
22
|
+
if initial_fetch < 0:
|
|
23
|
+
initial_fetch = 0
|
|
24
|
+
connect = {str(key): value for key, value in options.items() if str(key) not in PLUGIN_OPTIONS}
|
|
25
|
+
return ClickHouseOptions(connect=connect, initial_fetch=initial_fetch)
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""Private worker/application control channel for one ClickHouse client."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import socket
|
|
7
|
+
import threading
|
|
8
|
+
from typing import Any, Callable
|
|
9
|
+
from uuid import uuid4
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class WorkerApplicationBridge:
|
|
13
|
+
def __init__(self, socket_path: str) -> None:
|
|
14
|
+
self.socket_path = socket_path
|
|
15
|
+
self._listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
16
|
+
self._listener.bind(socket_path)
|
|
17
|
+
self._listener.listen(1)
|
|
18
|
+
self._connection: socket.socket | None = None
|
|
19
|
+
self._connected = threading.Event()
|
|
20
|
+
self._closed = threading.Event()
|
|
21
|
+
self._write_lock = threading.Lock()
|
|
22
|
+
self._condition = threading.Condition()
|
|
23
|
+
self._responses: dict[str, dict[str, Any]] = {}
|
|
24
|
+
threading.Thread(target=self._accept, name="jusi-clickhouse-bridge", daemon=True).start()
|
|
25
|
+
|
|
26
|
+
def _accept(self) -> None:
|
|
27
|
+
try:
|
|
28
|
+
connection, _ = self._listener.accept()
|
|
29
|
+
self._connection = connection
|
|
30
|
+
self._connected.set()
|
|
31
|
+
with connection.makefile("r", encoding="utf-8") as stream:
|
|
32
|
+
for line in stream:
|
|
33
|
+
response = json.loads(line)
|
|
34
|
+
request_id = str(response.get("id", ""))
|
|
35
|
+
if request_id:
|
|
36
|
+
with self._condition:
|
|
37
|
+
self._responses[request_id] = response
|
|
38
|
+
self._condition.notify_all()
|
|
39
|
+
except (OSError, ValueError, json.JSONDecodeError):
|
|
40
|
+
pass
|
|
41
|
+
finally:
|
|
42
|
+
self._closed.set()
|
|
43
|
+
self._connected.set()
|
|
44
|
+
with self._condition:
|
|
45
|
+
self._condition.notify_all()
|
|
46
|
+
|
|
47
|
+
def request(self, operation: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|
48
|
+
if not self._connected.wait(10) or self._connection is None:
|
|
49
|
+
raise RuntimeError("ClickHouse terminal application did not connect")
|
|
50
|
+
request_id = uuid4().hex
|
|
51
|
+
self._send({"id": request_id, "operation": operation, "payload": payload})
|
|
52
|
+
with self._condition:
|
|
53
|
+
while request_id not in self._responses and not self._closed.is_set():
|
|
54
|
+
self._condition.wait()
|
|
55
|
+
response = self._responses.pop(request_id, None)
|
|
56
|
+
if response is None:
|
|
57
|
+
raise RuntimeError("ClickHouse terminal application disconnected")
|
|
58
|
+
return response
|
|
59
|
+
|
|
60
|
+
def interrupt(self) -> None:
|
|
61
|
+
if self._connection is not None and not self._closed.is_set():
|
|
62
|
+
self._send({"id": uuid4().hex, "operation": "interrupt", "payload": {}})
|
|
63
|
+
|
|
64
|
+
def _send(self, message: dict[str, Any]) -> None:
|
|
65
|
+
connection = self._connection
|
|
66
|
+
if connection is None:
|
|
67
|
+
raise RuntimeError("ClickHouse terminal application is unavailable")
|
|
68
|
+
data = (json.dumps(message, ensure_ascii=False) + "\n").encode("utf-8")
|
|
69
|
+
with self._write_lock:
|
|
70
|
+
connection.sendall(data)
|
|
71
|
+
|
|
72
|
+
def close(self) -> None:
|
|
73
|
+
self._closed.set()
|
|
74
|
+
try:
|
|
75
|
+
self._listener.close()
|
|
76
|
+
except OSError:
|
|
77
|
+
pass
|
|
78
|
+
if self._connection is not None:
|
|
79
|
+
try:
|
|
80
|
+
self._connection.shutdown(socket.SHUT_RDWR)
|
|
81
|
+
except OSError:
|
|
82
|
+
pass
|
|
83
|
+
self._connection.close()
|
|
84
|
+
try:
|
|
85
|
+
os.unlink(self.socket_path)
|
|
86
|
+
except FileNotFoundError:
|
|
87
|
+
pass
|
|
88
|
+
with self._condition:
|
|
89
|
+
self._condition.notify_all()
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class ApplicationController:
|
|
93
|
+
def __init__(self, socket_path: str, handler: Callable[[str, dict[str, Any]], dict[str, Any]]) -> None:
|
|
94
|
+
self._handler = handler
|
|
95
|
+
self._connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
96
|
+
self._connection.connect(socket_path)
|
|
97
|
+
self._write_lock = threading.Lock()
|
|
98
|
+
|
|
99
|
+
def start(self) -> None:
|
|
100
|
+
threading.Thread(target=self._read, name="jusi-clickhouse-control", daemon=True).start()
|
|
101
|
+
|
|
102
|
+
def _read(self) -> None:
|
|
103
|
+
try:
|
|
104
|
+
with self._connection.makefile("r", encoding="utf-8") as stream:
|
|
105
|
+
for line in stream:
|
|
106
|
+
message = json.loads(line)
|
|
107
|
+
operation = str(message.get("operation", ""))
|
|
108
|
+
if operation == "interrupt":
|
|
109
|
+
self._handle(message)
|
|
110
|
+
else:
|
|
111
|
+
threading.Thread(target=self._handle, args=(message,), daemon=True).start()
|
|
112
|
+
except (OSError, ValueError, json.JSONDecodeError):
|
|
113
|
+
return
|
|
114
|
+
|
|
115
|
+
def _handle(self, message: dict[str, Any]) -> None:
|
|
116
|
+
payload = message.get("payload", {})
|
|
117
|
+
if not isinstance(payload, dict):
|
|
118
|
+
payload = {}
|
|
119
|
+
try:
|
|
120
|
+
result = self._handler(str(message.get("operation", "")), payload)
|
|
121
|
+
response = {"ok": True, "result": result}
|
|
122
|
+
except BaseException as exc:
|
|
123
|
+
response = {"ok": False, "error": "fatal", "message": f"{type(exc).__name__}: {exc}"}
|
|
124
|
+
response = {"id": str(message.get("id", "")), **response}
|
|
125
|
+
try:
|
|
126
|
+
with self._write_lock:
|
|
127
|
+
self._connection.sendall((json.dumps(response, ensure_ascii=False) + "\n").encode("utf-8"))
|
|
128
|
+
except OSError:
|
|
129
|
+
return
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Thin exact-provider adapter for the shared Jusi SQL kernel dispatcher."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from jusi_sql.kernel import SqlKernelAdapter
|
|
5
|
+
|
|
6
|
+
from . import __version__
|
|
7
|
+
from .constants import CLICKHOUSE_BOOTSTRAP_SQL
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
_adapter = SqlKernelAdapter(
|
|
11
|
+
plugin_id="clickhouse",
|
|
12
|
+
plugin_version=__version__,
|
|
13
|
+
selectors=("clickhouse",),
|
|
14
|
+
empty_sql=CLICKHOUSE_BOOTSTRAP_SQL,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
jusi_kernel_adapter_v1 = _adapter.manifest
|
|
18
|
+
configure_jusi_runtime_v1 = _adapter.configure
|
|
19
|
+
load_ipython_extension = _adapter.load
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from jusi_sql import CompletionColumn, CompletionObject, MetadataSnapshot
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def load_clickhouse_metadata(client: Any) -> MetadataSnapshot:
|
|
9
|
+
schemas = [str(row[0]) for row in _query_rows(client, "SELECT name FROM system.databases ORDER BY name")]
|
|
10
|
+
object_rows = _query_rows(
|
|
11
|
+
client,
|
|
12
|
+
"""
|
|
13
|
+
SELECT database, name, engine
|
|
14
|
+
FROM system.tables
|
|
15
|
+
WHERE database NOT IN ('INFORMATION_SCHEMA', 'information_schema')
|
|
16
|
+
ORDER BY database, name
|
|
17
|
+
""",
|
|
18
|
+
)
|
|
19
|
+
objects = [
|
|
20
|
+
CompletionObject(schema=str(row[0]), name=str(row[1]), kind=_table_kind(str(row[2])), detail=str(row[2]))
|
|
21
|
+
for row in object_rows
|
|
22
|
+
]
|
|
23
|
+
column_rows = _query_rows(
|
|
24
|
+
client,
|
|
25
|
+
"""
|
|
26
|
+
SELECT database, table, name, type
|
|
27
|
+
FROM system.columns
|
|
28
|
+
WHERE database NOT IN ('INFORMATION_SCHEMA', 'information_schema')
|
|
29
|
+
ORDER BY database, table, position
|
|
30
|
+
""",
|
|
31
|
+
)
|
|
32
|
+
columns = [
|
|
33
|
+
CompletionColumn(schema=str(row[0]), table=str(row[1]), name=str(row[2]), data_type=str(row[3]))
|
|
34
|
+
for row in column_rows
|
|
35
|
+
]
|
|
36
|
+
function_rows = _query_rows(client, "SELECT name, origin FROM system.functions ORDER BY name")
|
|
37
|
+
functions = [
|
|
38
|
+
CompletionObject(schema="", name=str(row[0]), kind="function", detail=str(row[1]))
|
|
39
|
+
for row in function_rows
|
|
40
|
+
]
|
|
41
|
+
return MetadataSnapshot(schemas=schemas, objects=objects, columns=columns, functions=functions)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _query_rows(client: Any, sql: str) -> list[Any]:
|
|
45
|
+
result = client.query(sql)
|
|
46
|
+
return list(getattr(result, "result_rows", []) or [])
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _table_kind(engine: str) -> str:
|
|
50
|
+
normalized = engine.lower()
|
|
51
|
+
if normalized == "view":
|
|
52
|
+
return "view"
|
|
53
|
+
if "dictionary" in normalized:
|
|
54
|
+
return "dictionary"
|
|
55
|
+
return "table"
|