spark-connect-cli 0.2.0__tar.gz

spark_connect_cli-0.2.0/.github/workflows/publish.yml

@@ -0,0 +1,43 @@
+name: Publish to PyPI
+
+# Publishes to PyPI via OIDC Trusted Publishing (no API token stored).
+# Triggered when you publish a GitHub Release. The version comes from
+# pyproject.toml — bump it before tagging, since PyPI rejects re-uploads.
+on:
+  release:
+    types: [published]
+  workflow_dispatch: {}
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+      - name: Build sdist + wheel
+        run: |
+          python -m pip install --upgrade build
+          python -m build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/spark-connect-cli
+    permissions:
+      id-token: write   # required for OIDC trusted publishing
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

spark_connect_cli-0.2.0/.gitignore

@@ -0,0 +1,9 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+build/
+dist/
+.venv/
+venv/
+.pytest_cache/
+.spark-connect-cli/

spark_connect_cli-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 dengshu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

spark_connect_cli-0.2.0/PKG-INFO

@@ -0,0 +1,156 @@
+Metadata-Version: 2.4
+Name: spark-connect-cli
+Version: 0.2.0
+Summary: Agent-friendly Spark Connect CLI: read-only querying + async long-job control. No JVM, no Kerberos on the client.
+Project-URL: Homepage, https://github.com/dengshu2/spark-connect-cli
+Project-URL: Issues, https://github.com/dengshu2/spark-connect-cli/issues
+Author: dengshu
+License: MIT
+License-File: LICENSE
+Keywords: agent,cli,clickhouse,hive,llm,spark,spark-connect
+Classifier: Environment :: Console
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database :: Front-Ends
+Requires-Python: >=3.9
+Requires-Dist: pyspark[connect]<4,>=3.5
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# spark-connect-cli (`scq`)
+
+An agent-friendly [Spark Connect](https://spark.apache.org/spark-connect/) CLI —
+**read-only querying** plus **async control for long-running jobs**.
+
+Built for LLM agents and humans who live in a shell. Unlike `spark-sql` /
+`spark-submit`, the client is a thin **pure-Python gRPC client**: no JVM, and
+**no Kerberos on the client side** — the Spark Connect server authenticates with
+its own keytab, so you just point at `sc://host:15002` and go.
+
+## Why
+
+- **JSON-first, read-only by default.** Safe for an agent to call for
+  exploration; writes/DDL are blocked unless you opt in (`--allow-ddl`).
+- **Long jobs don't block you.** A multi-minute Spark job shouldn't trap an agent
+  in a 30-minute tool call. `scq` submits the job, hands back a durable **job
+  id**, and returns immediately. Poll it whenever you like; the handle survives a
+  client/container restart because it lives in an on-disk registry.
+- **Stable exit codes** so a caller can branch without scraping text.
+
+## Install
+
+```bash
+pip install spark-connect-cli         # once published
+# or, from source:
+pip install -e .
+```
+
+## Quick start
+
+```bash
+export SPARK_REMOTE=sc://localhost:15002   # your Spark Connect endpoint
+
+scq databases
+scq tables mydb --like '%orders%'
+scq describe mydb.orders
+scq query "SELECT id, name FROM mydb.orders LIMIT 10"
+```
+
+Output is **JSONEachRow** (one JSON object per line) by default; pick another with
+`--format json|csv|tsv|table`.
+
+### Read-only guard
+
+`scq query` allows only `SELECT/SHOW/DESCRIBE/EXPLAIN/WITH`. Anything else exits
+with code **3** unless you pass `--allow-ddl`.
+
+| exit | meaning |
+|------|---------|
+| 0 | success |
+| 1 | query error (bad SQL) |
+| 2 | connection error |
+| 3 | blocked by the read-only guard |
+| 4 | job-control error (no such job, …) |
+
+## Async jobs (Layer A)
+
+Long work runs detached and is tracked by a file-based registry under
+`$SCQ_JOBS_DIR` (default `~/.spark-connect-cli/jobs`).
+
+```bash
+# submit — returns a job id immediately, does NOT block
+scq sync ods.orders --to clickhouse
+# {"job_id": "j-20260625-...", "state": "running", "message": "... poll with ..."}
+
+scq jobs list                       # all jobs + state
+scq jobs status j-20260625-...      # full status (rows, timings, pid, exit code)
+scq jobs logs   j-20260625-... --tail 40
+scq jobs cancel j-20260625-...      # kills the whole process group
+```
+
+Design: each job is a directory with `meta.json` (state machine:
+`submitted → running → succeeded|failed|cancelled`) and `out.log`. The worker
+runs in its **own process group**, so cancel kills the entire tree (no orphans).
+A `running` job whose process has vanished is reconciled to `failed` on the next
+status read, so status never lies.
+
+## Hive → ClickHouse sync
+
+`scq sync` is one job kind built on the async subsystem. It uses **Spark direct
+write**: a Spark Connect job reads the Hive table and writes to ClickHouse over
+JDBC. The write runs on the executors, so rows never pass through this process or
+the agent.
+
+Modes control write parallelism — `single` (one connection, small tables),
+`parallel` (N partitions, large tables), `auto` (picks by row count).
+
+Requires:
+- `clickhouse-jdbc` on the Spark Connect server classpath (`/opt/spark/jars/`),
+- cluster→ClickHouse network egress,
+- a JDBC URL with credentials via `--ch-jdbc` / `$SCQ_CH_JDBC`,
+- the **target ClickHouse table created beforehand** with a suitable engine
+  (Spark `append` won't build a usable MergeTree table for you — create it first,
+  e.g. with the `chsql` skill).
+
+## Introspection
+
+```bash
+scq meta db.table            # one JSON: schema, created time, location,
+                             # partitions, file count/size, mtime range
+scq meta db.table --count    # also run an exact count(*)
+
+scq exec stages?status=active            # read-only Spark REST passthrough
+scq exec executors
+scq exec stages/<id>/<attempt>/taskSummary?quantiles=0.5,0.95,1.0   # skew: max/median
+```
+
+`scq exec` auto-discovers the running Spark app via the YARN ResourceManager and
+proxies its monitoring REST API (GET-only). Set the RM base with `$SCQ_YARN_RM`.
+
+**Reading `scq exec executors`** — the `maxMemory` field is Spark's
+**storage/cache pool** (`(heap − 300 MB reserved) × 0.6`), *not* the executor's
+total memory: a 512 MB executor reports ~93 MB, a 1536 MB driver ~741 MB. The
+real heap is `spark.executor.memory` (+ off-heap overhead). The `driver` row has
+0 cores and runs no tasks. With dynamic allocation, idle executors are released —
+so the list may show only the driver when nothing is running.
+
+## Configuration
+
+| env | default | meaning |
+|-----|---------|---------|
+| `SPARK_REMOTE` | `sc://localhost:15002` | Spark Connect endpoint |
+| `SCQ_JOBS_DIR` | `~/.spark-connect-cli/jobs` | job registry (put on a persistent volume) |
+| `SCQ_MAX_ROWS` | `10000` | default row cap for `query` |
+| `SCQ_CH_JDBC` | — | ClickHouse JDBC URL for `sync` path A |
+| `SCQ_YARN_RM` | `http://namenode.hive-net:8088` | YARN RM base for `scq exec` |
+
+## Use with an LLM agent
+
+`SKILL.md` ships a ready-made skill (discover-before-query workflow, async-job
+etiquette, type-mapping table). Drop it into your agent's skills directory and
+the agent drives `scq` through a shell/Bash tool.
+
+## License
+
+MIT

spark_connect_cli-0.2.0/README.md

@@ -0,0 +1,136 @@
+# spark-connect-cli (`scq`)
+
+An agent-friendly [Spark Connect](https://spark.apache.org/spark-connect/) CLI —
+**read-only querying** plus **async control for long-running jobs**.
+
+Built for LLM agents and humans who live in a shell. Unlike `spark-sql` /
+`spark-submit`, the client is a thin **pure-Python gRPC client**: no JVM, and
+**no Kerberos on the client side** — the Spark Connect server authenticates with
+its own keytab, so you just point at `sc://host:15002` and go.
+
+## Why
+
+- **JSON-first, read-only by default.** Safe for an agent to call for
+  exploration; writes/DDL are blocked unless you opt in (`--allow-ddl`).
+- **Long jobs don't block you.** A multi-minute Spark job shouldn't trap an agent
+  in a 30-minute tool call. `scq` submits the job, hands back a durable **job
+  id**, and returns immediately. Poll it whenever you like; the handle survives a
+  client/container restart because it lives in an on-disk registry.
+- **Stable exit codes** so a caller can branch without scraping text.
+
+## Install
+
+```bash
+pip install spark-connect-cli         # once published
+# or, from source:
+pip install -e .
+```
+
+## Quick start
+
+```bash
+export SPARK_REMOTE=sc://localhost:15002   # your Spark Connect endpoint
+
+scq databases
+scq tables mydb --like '%orders%'
+scq describe mydb.orders
+scq query "SELECT id, name FROM mydb.orders LIMIT 10"
+```
+
+Output is **JSONEachRow** (one JSON object per line) by default; pick another with
+`--format json|csv|tsv|table`.
+
+### Read-only guard
+
+`scq query` allows only `SELECT/SHOW/DESCRIBE/EXPLAIN/WITH`. Anything else exits
+with code **3** unless you pass `--allow-ddl`.
+
+| exit | meaning |
+|------|---------|
+| 0 | success |
+| 1 | query error (bad SQL) |
+| 2 | connection error |
+| 3 | blocked by the read-only guard |
+| 4 | job-control error (no such job, …) |
+
+## Async jobs (Layer A)
+
+Long work runs detached and is tracked by a file-based registry under
+`$SCQ_JOBS_DIR` (default `~/.spark-connect-cli/jobs`).
+
+```bash
+# submit — returns a job id immediately, does NOT block
+scq sync ods.orders --to clickhouse
+# {"job_id": "j-20260625-...", "state": "running", "message": "... poll with ..."}
+
+scq jobs list                       # all jobs + state
+scq jobs status j-20260625-...      # full status (rows, timings, pid, exit code)
+scq jobs logs   j-20260625-... --tail 40
+scq jobs cancel j-20260625-...      # kills the whole process group
+```
+
+Design: each job is a directory with `meta.json` (state machine:
+`submitted → running → succeeded|failed|cancelled`) and `out.log`. The worker
+runs in its **own process group**, so cancel kills the entire tree (no orphans).
+A `running` job whose process has vanished is reconciled to `failed` on the next
+status read, so status never lies.
+
+## Hive → ClickHouse sync
+
+`scq sync` is one job kind built on the async subsystem. It uses **Spark direct
+write**: a Spark Connect job reads the Hive table and writes to ClickHouse over
+JDBC. The write runs on the executors, so rows never pass through this process or
+the agent.
+
+Modes control write parallelism — `single` (one connection, small tables),
+`parallel` (N partitions, large tables), `auto` (picks by row count).
+
+Requires:
+- `clickhouse-jdbc` on the Spark Connect server classpath (`/opt/spark/jars/`),
+- cluster→ClickHouse network egress,
+- a JDBC URL with credentials via `--ch-jdbc` / `$SCQ_CH_JDBC`,
+- the **target ClickHouse table created beforehand** with a suitable engine
+  (Spark `append` won't build a usable MergeTree table for you — create it first,
+  e.g. with the `chsql` skill).
+
+## Introspection
+
+```bash
+scq meta db.table            # one JSON: schema, created time, location,
+                             # partitions, file count/size, mtime range
+scq meta db.table --count    # also run an exact count(*)
+
+scq exec stages?status=active            # read-only Spark REST passthrough
+scq exec executors
+scq exec stages/<id>/<attempt>/taskSummary?quantiles=0.5,0.95,1.0   # skew: max/median
+```
+
+`scq exec` auto-discovers the running Spark app via the YARN ResourceManager and
+proxies its monitoring REST API (GET-only). Set the RM base with `$SCQ_YARN_RM`.
+
+**Reading `scq exec executors`** — the `maxMemory` field is Spark's
+**storage/cache pool** (`(heap − 300 MB reserved) × 0.6`), *not* the executor's
+total memory: a 512 MB executor reports ~93 MB, a 1536 MB driver ~741 MB. The
+real heap is `spark.executor.memory` (+ off-heap overhead). The `driver` row has
+0 cores and runs no tasks. With dynamic allocation, idle executors are released —
+so the list may show only the driver when nothing is running.
+
+## Configuration
+
+| env | default | meaning |
+|-----|---------|---------|
+| `SPARK_REMOTE` | `sc://localhost:15002` | Spark Connect endpoint |
+| `SCQ_JOBS_DIR` | `~/.spark-connect-cli/jobs` | job registry (put on a persistent volume) |
+| `SCQ_MAX_ROWS` | `10000` | default row cap for `query` |
+| `SCQ_CH_JDBC` | — | ClickHouse JDBC URL for `sync` path A |
+| `SCQ_YARN_RM` | `http://namenode.hive-net:8088` | YARN RM base for `scq exec` |
+
+## Use with an LLM agent
+
+`SKILL.md` ships a ready-made skill (discover-before-query workflow, async-job
+etiquette, type-mapping table). Drop it into your agent's skills directory and
+the agent drives `scq` through a shell/Bash tool.
+
+## License
+
+MIT

spark_connect_cli-0.2.0/SKILL.md

@@ -0,0 +1,151 @@
+---
+name: spark-connect-cli
+description: >-
+  Query Spark / Hive from the shell with the `scq` CLI over Spark Connect, and
+  run long Spark jobs (e.g. Hive->ClickHouse syncs) without blocking. Use
+  whenever the user wants to read Hive/Spark data, explore databases/tables/
+  schema, run a Spark SQL analysis, or sync a Hive table somewhere. Triggers:
+  Hive, Spark, Spark SQL, 查 Hive, 跑个 Spark SQL, 看下这个表, 同步到 ClickHouse,
+  sync table.
+---
+
+# scq — Spark Connect from the shell
+
+`scq` queries a Spark Connect server (JSON-first, **read-only by default**) and
+manages **async long jobs** so you never sit in a blocking tool call.
+
+## Discover before you query
+
+Don't guess names. Discover them first:
+
+1. `scq databases` — list databases.
+2. `scq tables [DB] --like '%keyword%'` — list tables.
+3. `scq describe <db.table>` — list columns (name, type, comment).
+4. `scq query "SELECT ..."` — run it once you know the schema.
+
+## Reading output
+
+- **stdout** carries data. Default is **JSONEachRow** (NDJSON — one JSON object
+  per line). Other formats: `--format json|csv|tsv|table`.
+- **stderr** carries errors as one JSON object `{"error": ..., "code": ...}`.
+- `query` caps at `SCQ_MAX_ROWS` (default 10k); a `{"warning": ...}` on stderr
+  means it was truncated — add `LIMIT`/filters or raise `--max-rows`.
+
+## Branch on the exit code
+
+`0` ok · `1` query error (fix the SQL) · `2` connection error (check
+`$SPARK_REMOTE`) · `3` read-only guard blocked it · `4` job-control error.
+
+## Read-only by default
+
+`scq query` allows only SELECT/SHOW/DESCRIBE/EXPLAIN/WITH. Writes and DDL exit
+with code `3` unless you pass `--allow-ddl`. **Only** add `--allow-ddl` when the
+user explicitly asked to modify data or schema.
+
+## Long jobs — submit, then poll. NEVER block.
+
+A full-table sync is a multi-minute Spark job. **Do not** run it in the
+foreground and wait. Submit it, tell the user the job id, and hand control back:
+
+```bash
+scq sync ods.orders --to clickhouse      # prints {"job_id": "...", "state": "running"}
+```
+
+Then, *only when the user asks* "how's it going" / after a natural pause:
+
+```bash
+scq jobs status j-20260625-...           # state, source_rows, written_rows, exit_code
+scq jobs logs   j-20260625-... --tail 40 # recent progress
+scq jobs cancel j-20260625-...           # stop it
+```
+
+Etiquette:
+- After submitting, reply with the job id and a one-line "I'll check when you
+  want." Don't loop on `status` in a tight wait — let the user drive, or poll on
+  a relaxed cadence.
+- Report terminal state plainly: `succeeded` with `written_rows`, or `failed`
+  with the tail of the log.
+
+## Hive → ClickHouse sync workflow
+
+When the user says "同步 X 表到 ClickHouse":
+
+1. `scq describe <src>` — get the Hive schema.
+2. Decide the **target database and table**. `--target` takes `db.table`:
+   - If the user names a database (e.g. `class_db`), pass it **qualified**:
+     `--target class_db.class`. A **bare table name lands in the connection's
+     default database** (`default`) — don't let data silently go there.
+   - The **database must already exist** (auto-create makes the table, not the
+     database). Ensure it first: `chsql query --allow-ddl "CREATE DATABASE IF NOT
+     EXISTS class_db"`.
+3. Make sure the target table is good:
+   - For a quick/one-off sync, let `scq sync` auto-create it — but pass
+     `--order-by <key>` so it gets a real sort key (otherwise it is created with
+     `ORDER BY tuple()`, no primary index).
+   - For a production table, **pre-create it** with `chsql query --allow-ddl
+     "CREATE TABLE class_db.class (...) ENGINE = MergeTree ORDER BY (...)"` (full
+     control over engine, keys, partitioning), then sync.
+4. Submit: `scq sync <src> --target db.table [--order-by key] [--where ...]`.
+5. Hand back the job id. Verify with row counts when it finishes.
+
+The ClickHouse JDBC connection (`$SCQ_CH_JDBC`) is preconfigured — you do **not**
+pass credentials; just choose the `db.table` with `--target`.
+
+### Spark/Hive → ClickHouse type mapping
+
+| Spark/Hive | ClickHouse |
+|------------|------------|
+| boolean | Bool |
+| tinyint / smallint / int / bigint | Int8 / Int16 / Int32 / Int64 |
+| float / double | Float32 / Float64 |
+| decimal(p,s) | Decimal(p,s) |
+| string / varchar / char / binary | String |
+| date | Date32 |
+| timestamp | DateTime64(3) |
+
+Nullable columns map to `Nullable(T)`. Nested/complex types default to `String`
+(JSON) — confirm with the user before relying on them.
+
+## Metadata & execution introspection
+
+Two general primitives — don't hand-stitch many queries.
+
+**Table metadata → `scq meta db.table`** — one JSON: schema (+ ClickHouse type
+mapping), created time, owner, format, HDFS location, partition columns +
+partition list/count, file count/total size, and min/max file modification time
+(i.e. "when did the data arrive"). Add `--count` for an exact row count (runs a
+`count(*)`, so only when asked). For ad-hoc bits you can still use
+`scq query "DESCRIBE EXTENDED t"` / `"SHOW PARTITIONS t"`.
+
+**Execution metadata → `scq exec <path>`** — read-only passthrough to the Spark
+REST API (auto-discovers the app, GET-only). The model reads the JSON, so any
+runtime question is the same command with a different path:
+
+```bash
+scq exec stages?status=active          # what's running now
+scq exec sql                           # each query's plan + metrics
+scq exec executors                     # cores / memory / GC / shuffle
+scq exec jobs
+scq exec stages/<id>/<attempt>/taskSummary?quantiles=0.5,0.95,1.0
+```
+
+- **Data skew**: pull a stage's `taskSummary` and compare a metric's **max vs
+  median** (`executorRunTime`, `shuffleReadBytes`, `shuffleReadRecords`). A large
+  `max/median` ratio = a straggler / skewed partition. `…?details=true` on a
+  stage lists every task to find the hot one.
+- Stage/job lists can be long — filter (`?status=active`) or fetch one id.
+- For the *plan before running*, use `scq query "EXPLAIN FORMATTED SELECT ..."`.
+
+## Connection
+
+`scq --remote sc://host:15002 ...` or set `$SPARK_REMOTE`. No Kerberos or JVM is
+needed on this side — the Spark Connect server does the auth.
+
+## Recipes
+
+```bash
+scq --format table tables analytics --like '%event%'
+scq query --format table "SELECT count(*) FROM analytics.events"
+scq query --max-rows 0 "SELECT * FROM small_dim"        # no cap
+scq sync analytics.events --to clickhouse --where "dt='2026-06-25'"
+```

spark_connect_cli-0.2.0/pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "spark-connect-cli"
+version = "0.2.0"
+description = "Agent-friendly Spark Connect CLI: read-only querying + async long-job control. No JVM, no Kerberos on the client."
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+authors = [{ name = "dengshu" }]
+keywords = ["spark", "spark-connect", "cli", "hive", "clickhouse", "llm", "agent"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Topic :: Database :: Front-Ends",
+    "Environment :: Console",
+]
+dependencies = [
+    "pyspark[connect]>=3.5,<4",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=7"]
+
+[project.urls]
+Homepage = "https://github.com/dengshu2/spark-connect-cli"
+Issues = "https://github.com/dengshu2/spark-connect-cli/issues"
+
+[project.scripts]
+scq = "spark_connect_cli.cli:main"
+spark-connect-cli = "spark_connect_cli.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/spark_connect_cli"]
+
+# Ship SKILL.md inside the package so `scq skill install` can write it out.
+[tool.hatch.build.targets.wheel.force-include]
+"SKILL.md" = "spark_connect_cli/SKILL.md"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

spark_connect_cli-0.2.0/src/spark_connect_cli/__init__.py

@@ -0,0 +1,2 @@
+"""spark-connect-cli — an agent-friendly Spark Connect CLI."""
+__version__ = "0.2.0"

spark_connect_cli-0.2.0/src/spark_connect_cli/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    main()